diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2021-08-11 14:29:27 +1000 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-08-11 04:42:04 +0000 |
| commit | 08bd2e4a83e711cba3938684a32c3544b56a61dd (patch) | |
| tree | 4e6802a3105d03dd794ebe22b2b76ec8cb553ed0 /src/third_party/wiredtiger | |
| parent | 7c6b85895f41c914873198fbc0510fe44eea3ae7 (diff) | |
| download | mongo-08bd2e4a83e711cba3938684a32c3544b56a61dd.tar.gz | |
Import wiredtiger: 03eb45ea7fd94d6f241f8668a58d0119c39bbf52 from branch mongodb-4.4
ref: 2b73914cd8..03eb45ea7f
for: 4.4.9
WT-6755 Documentation: populate developer glossary
WT-6902 Metadata subpage for Architecture Guide
WT-6905 Write row-store and column-store subpage for Architecture Guide
WT-6910 Write "history store" subpage for Architecture Guide
WT-6914 Write "database files" subpage for Architecture Guide
WT-7007 Backup architecture guide page
WT-7198 Fix test_backup15 failure with backup mismatch
WT-7352 Fix test_hs01 conflict between concurrent operations in cursor modify
WT-7521 Remove excess ckplist invalidations
WT-7592 Remove log_flush("sync=background") support
WT-7599 Update the CONFIG file based on the release that is about to run for compatibility tests
WT-7607 Add the possibility to use an existing database for many-coll-test
WT-7673 Investigate and fix manydbs test failure on Windows
WT-7703 Fix timeout in test_checkpoint_snapshot04
WT-7732 Add a timeout configuration for flush_tier
WT-7758 Force evict a page when the update chain is too long
WT-7787 Don't read pages for checkpoint cleanup when the cache is in aggressive mode
WT-7789 Change tiered python tests to fail without extension library
WT-7817 Make tiered storage address cookies backward compatible
WT-7830 Migrate the python setup scripts to use cmake
WT-7838 Ability for ordered timestamp assertion to do more than a log message
WT-7842 Remove explicit ulimit -n call in many-collection-test
WT-7860 Improve code coverage reporting
WT-7864 Add support to run.py for running lists/ranges of scenarios in a test
WT-7865 Disable timeout assert while waiting for eviction to quiesce prior to RTS and test
WT-7870 Fix measurement of cyclomatic code complexity
WT-7871 Remove comment that is no longer true
WT-7874 Remove two other stale comments
WT-7876 Update rollback to stable test to use correct boolean values and update statistic checking logic
WT-7882 Fix discrepancy for wiredtiger.in on mongodb-4.4 branch
WT-7883 Remove incorrect wt_free statement
WT-7889 Find/replace existing uses of WT with WiredTiger in reference guide
WT-7890 Fix CMake syntax error in config_lib
WT-7893 Remove ignored message from wiredtiger_open in test_encrypt08
WT-7895 Fix arch-data-file.dox documentation build failure
WT-7897 Enable verbose logging for test_backup15 to aid debugging
WT-7898 Upload many-coll-test artifacts
WT-7900 Fix insertion of new records in test format for column-store
WT-7901 test suite cleanup
WT-7908 Make variable-length column store work again with the static tests
WT-7935 Add arm64 implementation of rdtsc equivalent instruction
WT-7937 Fix s_docs to use sh, not bash syntax
WT-7938 Fix rollback-to-stable memory leak on error
Diffstat (limited to 'src/third_party/wiredtiger')
249 files changed, 5529 insertions, 3667 deletions
diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/ycsb-f.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/ycsb-f.wtperf index c997b7c1231..4fe3587ac0a 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/ycsb-f.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/ycsb-f.wtperf @@ -5,7 +5,8 @@ key_sz=100 value_sz=1024 icount=120000000 run_time=3600 -threads=((count=10,reads=1),(count=10,modifies=1)) +sess_config="isolation=snapshot" +threads=((count=10,reads=1),(count=10,modify=1,ops_per_txn=1)) warmup=120 sample_interval=5 populate_threads=8 diff --git a/src/third_party/wiredtiger/build_cmake/helpers.cmake b/src/third_party/wiredtiger/build_cmake/helpers.cmake index 5593018284a..4dc449c0748 100644 --- a/src/third_party/wiredtiger/build_cmake/helpers.cmake +++ b/src/third_party/wiredtiger/build_cmake/helpers.cmake @@ -434,7 +434,7 @@ function(config_lib config_name description) message("-- Looking for library ${CONFIG_LIB_LIB}: found ${has_lib_${config_name}}, include path ${include_path_${config_name}}") include_directories(${include_path_${config_name}}) else() - message("-- Looking for library ${CONFIG_LIB_LIB}: found ${has_lib$_{config_name}}") + message("-- Looking for library ${CONFIG_LIB_LIB}: found ${has_lib_${config_name}}") endif() unset(include_path_${config_name} CACHE) else() diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index c560a28eb11..9663188789b 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -1385,12 +1385,10 @@ methods = { 'WT_SESSION.log_flush' : Method([ Config('sync', 'on', r''' forcibly flush the log and wait for it to achieve the synchronization - level specified. The \c background setting initiates a background - synchronization intended to be used with a later call to - WT_SESSION::transaction_sync. The \c off setting forces any + level specified. The \c off setting forces any buffered log records to be written to the file system. The \c on setting forces log records to be written to the storage device''', - choices=['background', 'off', 'on']), + choices=['off', 'on']), ]), 'WT_SESSION.log_printf' : Method([]), @@ -1573,17 +1571,14 @@ methods = { \c on setting causes the caller to wait until all work queued for this call to be completely processed before returning''', choices=['off', 'on']), + Config('timeout', '0', r''' + maximum amount of time to allow for waiting for previous flushing + of objects, in seconds. The actual amount of time spent waiting may + exceed the configured value. A value of zero disables the timeout''', + type='int'), ]), 'WT_SESSION.strerror' : Method([]), -'WT_SESSION.transaction_sync' : Method([ - Config('timeout_ms', '1200000', # !!! Must match WT_SESSION_BG_SYNC_MSEC - r''' - maximum amount of time to wait for background sync to complete in - milliseconds. A value of zero disables the timeout and returns - immediately''', - type='int'), -]), 'WT_SESSION.truncate' : Method([]), 'WT_SESSION.upgrade' : Method([]), @@ -1701,13 +1696,10 @@ methods = { min=1), Config('sync', '', r''' override whether to sync log records when the transaction commits, - inherited from ::wiredtiger_open \c transaction_sync. - The \c background setting initiates a background - synchronization intended to be used with a later call to - WT_SESSION::transaction_sync. The \c off setting does not + inherited from ::wiredtiger_open \c transaction_sync. The \c off setting does not wait for record to be written or synchronized. The \c on setting forces log records to be written to the storage device''', - choices=['background', 'off', 'on']), + choices=['off', 'on']), ]), 'WT_SESSION.prepare_transaction' : Method([ diff --git a/src/third_party/wiredtiger/dist/docs_data.py b/src/third_party/wiredtiger/dist/docs_data.py index 0aa93464bef..831cdc61f30 100644 --- a/src/third_party/wiredtiger/dist/docs_data.py +++ b/src/third_party/wiredtiger/dist/docs_data.py @@ -11,6 +11,9 @@ class ArchDocPage: # List of all architecture subsections ########################################## arch_doc_pages = [ + ArchDocPage('arch-backup', + ['WT_CURSOR_BACKUP'], + ['src/cursor/cur_backup.c', 'src/cursor/cur_backup_incr.c']), ArchDocPage('arch-block', ['WT_BLOCK', 'WT_BLOCK_CKPT', 'WT_BLOCK_DESC', 'WT_BLOCK_HEADER', 'WT_BM', 'WT_EXTLIST'], @@ -24,9 +27,6 @@ arch_doc_pages = [ ['src/block/block_ckpt.c', 'src/block/block_ckpt_scan.c', 'src/conn/conn_ckpt.c', 'src/meta/meta_ckpt.c', 'src/txn/txn_ckpt.c']), - ArchDocPage('arch-column', - ['WT_BTREE'], - ['src/include/btree.h']), ArchDocPage('arch-cursor', ['WT_CURSOR', 'WT_CURSOR_BACKUP', 'WT_CURSOR_BTREE', 'WT_CURSOR_BULK', 'WT_CURSOR_DATA_SOURCE', 'WT_CURSOR_DUMP', 'WT_CURSOR_INDEX', @@ -67,7 +67,7 @@ arch_doc_pages = [ ArchDocPage('arch-python', [], ['lang/python/']), - ArchDocPage('arch-row', + ArchDocPage('arch-row-column', ['WT_BTREE'], ['src/include/btree.h']), ArchDocPage('arch-rts', diff --git a/src/third_party/wiredtiger/dist/s_docs b/src/third_party/wiredtiger/dist/s_docs index 1077ce5f332..83b31aa1a09 100755 --- a/src/third_party/wiredtiger/dist/s_docs +++ b/src/third_party/wiredtiger/dist/s_docs @@ -1,7 +1,8 @@ #! /bin/bash t=__wt.$$ -trap 'rm -f $t' 0 1 2 3 13 15 +t2=__wt.$$.2 +trap 'rm -f $t $t2' 0 1 2 3 13 15 # Skip this when building release packages: docs are built separately test -n "$WT_RELEASE_BUILD" && exit 0 @@ -53,6 +54,18 @@ wtperf_config() echo 'q') | ed ../src/docs/wtperf.dox 1>/dev/null 2>/dev/null } +glossarychk() +{ + (cd ../src/docs && grep '^<tr><td>' arch-glossary.dox | cut -d'>' -f3 | sed -e 's/<.*//') > $t + sort --ignore-case $t > $t2 + if ! diff $t $t2; then + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + echo "arch-glossary.dox entries out of order" + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + e=1 + fi +} + structurechk() { # @page names should match the source file name @@ -116,7 +129,7 @@ spellchk() (cd ../src/docs && # Separate quoted newlines "line\nline" so "nline" is not reported. - sed -e 's/\("[^"]*\)\\n\([^"]*"\)/\1 \2/' *.dox | \ + sed -e 's/\("[^"]*\)\\n\([^"]*"\)/\1 \2/' -e 's/[@\\]sic *\([^ ]*\) //g' *.dox | \ aspell --encoding=iso-8859-1 --lang=en_US --personal=./spell.ok list) | sort -u > $t test -s $t && { @@ -177,6 +190,9 @@ EOF e=1 } + # Copy any needed javascript + cp ../src/docs/js/* ../docs/ + # Fix up bad links doxygen generates in navtree.js (cd ../docs && sed -i~ -e 's,/\.html,/,' -e 's,\.html\.html,.html,' navtree.js && @@ -243,6 +259,7 @@ wtperf_config asciichk spellchk structurechk +glossarychk # Check the docs data input file. check_docs_data diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index eb797977de4..d0b2ab55adf 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -52,7 +52,9 @@ Btree Buf Bxxx Bzip +CACHEABLE CAS +CBT CCCCCCCCCCCCCCCCCC CELL's CELLs @@ -573,6 +575,7 @@ bzDecompressInit bzalloc bzfree bzip +cacheable calc call's calloc @@ -619,6 +622,7 @@ clsm cmd cmp cnt +cntvct colcheck colgroup colgroups @@ -1062,6 +1066,7 @@ metadata metadata's metafile mfence +minSnapshotHistoryWindowInSeconds minorp mips mkdir @@ -1069,6 +1074,7 @@ mmap mmrand mnt movemask +mrs msecs msg msvc @@ -1416,6 +1422,7 @@ uint uintmax unbare unbuffered +uncacheable uncompressed uncompressing uncustomized diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index aca6c6b2298..32d3d237858 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -213,6 +213,7 @@ conn_stats = [ CacheStat('cache_eviction_fail_in_reconciliation', 'pages selected for eviction unable to be evicted because of failure in reconciliation'), CacheStat('cache_eviction_fail_parent_has_overflow_items', 'pages selected for eviction unable to be evicted as the parent page has overflow items'), CacheStat('cache_eviction_force', 'forced eviction - pages selected count'), + CacheStat('cache_eviction_force_long_update_list', 'forced eviction - pages selected because of a large number of updates to a single item'), CacheStat('cache_eviction_force_clean', 'forced eviction - pages evicted that were clean count'), CacheStat('cache_eviction_force_clean_time', 'forced eviction - pages evicted that were clean time (usecs)'), CacheStat('cache_eviction_force_delete', 'forced eviction - pages selected because of too many deleted items count'), @@ -572,7 +573,6 @@ conn_stats = [ TxnStat('txn_set_ts_oldest_upd', 'set timestamp oldest updates'), TxnStat('txn_set_ts_stable', 'set timestamp stable calls'), TxnStat('txn_set_ts_stable_upd', 'set timestamp stable updates'), - TxnStat('txn_sync', 'transaction sync calls'), TxnStat('txn_timestamp_oldest_active_read', 'transaction read timestamp of the oldest active reader', 'no_clear,no_scale'), TxnStat('txn_walk_sessions', 'transaction walk of concurrent sessions'), @@ -584,7 +584,6 @@ conn_stats = [ YieldStat('child_modify_blocked_page', 'page reconciliation yielded due to child modification'), YieldStat('conn_close_blocked_lsm', 'connection close yielded for lsm manager shutdown'), YieldStat('dhandle_lock_blocked', 'data handle lock yielded'), - YieldStat('log_server_sync_blocked', 'log server sync yielded for log write'), YieldStat('page_busy_blocked', 'page acquire busy blocked'), YieldStat('page_del_rollback_blocked', 'page delete rollback time sleeping for state change (usecs)'), YieldStat('page_forcible_evict_blocked', 'page acquire eviction blocked'), diff --git a/src/third_party/wiredtiger/examples/c/Makefile.am b/src/third_party/wiredtiger/examples/c/Makefile.am index 6c20b31a146..e83bac63f2c 100644 --- a/src/third_party/wiredtiger/examples/c/Makefile.am +++ b/src/third_party/wiredtiger/examples/c/Makefile.am @@ -25,7 +25,6 @@ noinst_PROGRAMS = \ ex_schema \ ex_smoke \ ex_stat \ - ex_sync \ ex_thread ex_encrypt_LDFLAGS = -rdynamic diff --git a/src/third_party/wiredtiger/examples/c/ex_all.c b/src/third_party/wiredtiger/examples/c/ex_all.c index 9742a4f59a0..c769ffff4b4 100644 --- a/src/third_party/wiredtiger/examples/c/ex_all.c +++ b/src/third_party/wiredtiger/examples/c/ex_all.c @@ -700,10 +700,6 @@ session_ops(WT_SESSION *session) error_check(session->truncate(session, "table:mytable", NULL, NULL, NULL)); /*! [Truncate a table] */ - /*! [Transaction sync] */ - error_check(session->transaction_sync(session, NULL)); - /*! [Transaction sync] */ - /*! [Reset the session] */ error_check(session->reset(session)); /*! [Reset the session] */ diff --git a/src/third_party/wiredtiger/examples/c/ex_sync.c b/src/third_party/wiredtiger/examples/c/ex_sync.c deleted file mode 100644 index 0ee99e6bc8d..00000000000 --- a/src/third_party/wiredtiger/examples/c/ex_sync.c +++ /dev/null @@ -1,130 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * ex_sync.c - * demonstrates how to use the transaction sync configuration. - */ -#include <test_util.h> - -static const char *home; -static const char *const uri = "table:test"; - -#define CONN_CONFIG "create,cache_size=100MB,log=(archive=false,enabled=true)" -#define MAX_KEYS 100 - -int -main(int argc, char *argv[]) -{ - WT_CONNECTION *wt_conn; - WT_CURSOR *cursor; - WT_SESSION *session; - int i, record_count, ret; - char k[32], v[32]; - const char *conf; - - home = example_setup(argc, argv); - error_check(wiredtiger_open(home, NULL, CONN_CONFIG, &wt_conn)); - - error_check(wt_conn->open_session(wt_conn, NULL, NULL, &session)); - error_check(session->create(session, uri, "key_format=S,value_format=S")); - - error_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); - /* - * Perform some operations with individual auto-commit transactions. - */ - error_check(session->begin_transaction(session, NULL)); - for (record_count = 0, i = 0; i < MAX_KEYS; i++, record_count++) { - if (i == MAX_KEYS / 2) { - error_check(session->commit_transaction(session, "sync=background")); - ret = session->transaction_sync(session, "timeout_ms=0"); - if (ret == ETIMEDOUT) - printf("Transactions not yet stable\n"); - else if (ret != 0) { - fprintf( - stderr, "session.transaction_sync: error %s\n", session->strerror(session, ret)); - exit(1); - } - error_check(session->begin_transaction(session, NULL)); - } else { - if ((record_count % 3) == 0) - conf = "sync=background"; - else - conf = "sync=off"; - error_check(session->commit_transaction(session, conf)); - error_check(session->begin_transaction(session, NULL)); - } - (void)snprintf(k, sizeof(k), "key%d", i); - (void)snprintf(v, sizeof(v), "value%d", i); - cursor->set_key(cursor, k); - cursor->set_value(cursor, v); - error_check(cursor->insert(cursor)); - } - error_check(session->commit_transaction(session, "sync=background")); - printf("Wait forever until stable\n"); - error_check(session->transaction_sync(session, NULL)); - printf("Transactions now stable\n"); - error_check(session->begin_transaction(session, NULL)); - /* - * Perform some operations within a single transaction. - */ - for (i = MAX_KEYS; i < MAX_KEYS + 5; i++, record_count++) { - (void)snprintf(k, sizeof(k), "key%d", i); - (void)snprintf(v, sizeof(v), "value%d", i); - cursor->set_key(cursor, k); - cursor->set_value(cursor, v); - error_check(cursor->insert(cursor)); - } - error_check(session->commit_transaction(session, "sync=on")); - error_check(session->transaction_sync(session, "timeout_ms=0")); - - /* - * Demonstrate using log_flush to force the log to disk. - */ - for (i = 0; i < MAX_KEYS; i++, record_count++) { - (void)snprintf(k, sizeof(k), "key%d", record_count); - (void)snprintf(v, sizeof(v), "value%d", record_count); - cursor->set_key(cursor, k); - cursor->set_value(cursor, v); - error_check(cursor->insert(cursor)); - } - error_check(session->log_flush(session, "sync=on")); - - for (i = 0; i < MAX_KEYS; i++, record_count++) { - (void)snprintf(k, sizeof(k), "key%d", record_count); - (void)snprintf(v, sizeof(v), "value%d", record_count); - cursor->set_key(cursor, k); - cursor->set_value(cursor, v); - error_check(cursor->insert(cursor)); - } - error_check(cursor->close(cursor)); - error_check(session->log_flush(session, "sync=off")); - error_check(session->log_flush(session, "sync=on")); - - error_check(wt_conn->close(wt_conn, NULL)); - - return (EXIT_SUCCESS); -} diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 3a9e0ea16cc..9e2b467faaf 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "2b73914cd8912fab0e01ebd67cd0106de45442cd" + "commit": "03eb45ea7fd94d6f241f8668a58d0119c39bbf52" } diff --git a/src/third_party/wiredtiger/lang/python/setup.py b/src/third_party/wiredtiger/lang/python/setup.py index 22261bc8272..d6cedb3ab3d 100644 --- a/src/third_party/wiredtiger/lang/python/setup.py +++ b/src/third_party/wiredtiger/lang/python/setup.py @@ -27,8 +27,8 @@ # OTHER DEALINGS IN THE SOFTWARE. # -import re, os, sys -from distutils.core import setup, Extension +import re, os +from setuptools import setup, Extension # OS X hack: turn off the Universal binary support that is built into the # Python build machinery, just build for the default CPU architecture. diff --git a/src/third_party/wiredtiger/lang/python/setup_pip.py b/src/third_party/wiredtiger/lang/python/setup_pip.py index 0dac330389e..ede4f472548 100755 --- a/src/third_party/wiredtiger/lang/python/setup_pip.py +++ b/src/third_party/wiredtiger/lang/python/setup_pip.py @@ -34,11 +34,9 @@ # "python setup_pip.py sdist", this creates a tar.gz file under ./dist . from __future__ import print_function import os, os.path, re, shutil, site, sys -from setuptools import setup, Distribution -from distutils.extension import Extension +from setuptools import setup, Distribution, Extension import distutils.sysconfig import distutils.ccompiler -from distutils.errors import CompileError, LinkError import subprocess from subprocess import call import setuptools.command.install @@ -75,12 +73,12 @@ def check_needed_dependencies(builtins, inc_paths, lib_paths): distutils.sysconfig.customize_compiler(compiler) compiler.set_library_dirs(library_dirs) missing = [] - for name, libname, instructions in builtins: + for _, libname, instructions in builtins: found = compiler.find_library_file(library_dirs, libname) if found is None: msg(libname + ": missing") msg(instructions) - msg("after installing it, set LD_LIBRARY_PATH or DYLD_LIBRARY_PATH") + msg("after installing it, set CMAKE_LIBRARY_PATH") missing.append(libname) else: package_top = os.path.dirname(os.path.dirname(found)) @@ -94,41 +92,12 @@ def check_needed_dependencies(builtins, inc_paths, lib_paths): #if len(missing) > 0: # die("install packages for: " + str(missing)) -# find_executable -- -# Locate an executable in the PATH. -def find_executable(exename, path): - p = subprocess.Popen(['which', exename ], stdout=subprocess.PIPE, - stderr=subprocess.PIPE, universal_newlines=True) - out, err = p.communicate('') - out = str(out) # needed for Python3 - if out == '': - if err != '': - err = ': "' + err + '"' - die('"' + exename + '": not found in path' + err) - dirname = os.path.dirname(out) - if not dirname in path: - path.append(dirname) - -# get_build_path -- -# Create a PATH that can be used for installation. Apparently, -# installation commands are run with a restricted PATH, and -# autoreconf/aclocal will not normally be found. -def get_build_path(): - build_paths = [] - find_executable('autoreconf', build_paths) - find_executable('aclocal', build_paths) - build_path = os.environ['PATH'] + ':' + ':'.join(build_paths) - return build_path - # get_compile_flags -- # Get system specific compile flags. Return a triple: C preprocessor # flags, C compilation flags and linker flags. def get_compile_flags(inc_paths, lib_paths): # Suppress warnings building SWIG generated code - if sys.platform == 'win32' and cc == 'msvc': - cflags = ['/arch:SSE2', '/EHsc'] - cppflags = [] - ldflags = [] + if sys.platform == 'win32': # Windows untested and incomplete, don't claim that it works. die('Windows is not supported by this setup script') else: @@ -181,44 +150,12 @@ def get_library_dirs(): dirs.append("/opt/local/lib") dirs.append("/usr/lib") dirs.append("/usr/lib64") - for path in ['LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH', 'LIBRARY_PATH']: + for path in ['CMAKE_LIBRARY_PATH', 'LIBRARY_PATH']: if path in os.environ: dirs.extend(os.environ[path].split(':')) dirs = list(set(filter(os.path.isdir, dirs))) return dirs -# source_filter -# Make any needed changes to the original sources list and return a manifest, -# a list of source file names relative to the new staging root. Any entry -# that needs to be renamed returned in a dictionary where the entry's key -# is the new name and the value is the old source name. -def source_filter(sources): - result = [] - movers = dict() - py_dir = os.path.join('lang', 'python') - pywt_dir = os.path.join(py_dir, 'wiredtiger') - pywt_build_dir = os.path.join('build_posix', py_dir, 'wiredtiger') - pywt_prefix = pywt_dir + os.path.sep - for f in sources: - if not re.match(source_regex, f): - continue - src = f - dest = f - # move all lang/python files to the top level. - if dest.startswith(pywt_prefix): - dest = os.path.basename(dest) - if dest == 'init.py': - dest = '__init__.py' - if dest != src: - movers[dest] = src - result.append(dest) - # Add SWIG generated files - result.append(os.path.join(py_dir, 'wiredtiger_wrap.c')) - wiredtiger_py = 'swig_wiredtiger.py' - result.append('swig_wiredtiger.py') - movers['swig_wiredtiger.py'] = os.path.join(py_dir, 'wiredtiger.py') - return result, movers - ################################################################ # Do some initial setup and checks. this_abs_script = os.path.abspath(__file__) @@ -243,10 +180,10 @@ if sys.maxsize < 2**32: die('need to be running on a 64 bit system, and have a 64 bit Python') python_rel_dir = os.path.join('lang', 'python') -build_dir = os.path.join(wt_dir, 'build_posix') -makefile = os.path.join(build_dir, 'Makefile') +build_dir = os.path.join(wt_dir, 'cmake_pip_build') +makefile = os.path.join(build_dir, 'build.ninja') built_sentinal = os.path.join(build_dir, 'built.txt') -conf_make_dir = 'build_posix' +conf_make_dir = 'cmake_pip_build' wt_swig_lib_name = os.path.join(python_rel_dir, '_wiredtiger.so') ################################################################ @@ -257,11 +194,6 @@ long_description = 'WiredTiger is a ' + short_description + '.\n\n' + \ open(os.path.join(wt_dir, 'README')).read() wt_ver, wt_full_ver = get_wiredtiger_versions(wt_dir) -build_path = get_build_path() - -# We only need a small set of directories to build a WT library, -# we also include any files at the top level. -source_regex = r'^(?:(?:api|build_posix|ext|lang/python|src|dist)/|[^/]*$)' # The builtins that we include in this distribution. builtins = [ @@ -277,7 +209,7 @@ builtins = [ 'It can be installed via: apt-get install zlib1g' ], [ 'zstd', 'zstd', 'Need to install zstd\n' + \ - 'It can be installed via: apt-get install zstd' ] + 'It can be installed via: apt-get install libzstd-dev' ] ] builtin_names = [b[0] for b in builtins] builtin_libraries = [b[1] for b in builtins] @@ -285,22 +217,19 @@ builtin_libraries = [b[1] for b in builtins] # Here's the configure/make operations we perform before the python extension # is linked. configure_cmds = [ - './makemake --clean-and-make', - './reconf', - # force building a position independent library; it will be linked - # into a single shared library with the SWIG interface code. - 'CFLAGS="${CFLAGS:-} -fPIC -DPIC" ' + \ - '../configure --enable-python --with-builtins=' + ','.join(builtin_names) + 'cmake -B cmake_pip_build -G Ninja -DENABLE_STATIC=1 -DCMAKE_C_FLAGS="${CFLAGS:-}" -DENABLE_PYTHON=1 ' + \ + ' '.join(map(lambda name: '-DHAVE_BUILTIN_EXTENSION_' + name.upper() + '=1', builtin_names)), ] # build all the builtins, at the moment they are all compressors. make_cmds = [] for name in builtin_names: - make_cmds.append('(cd ext/compressors/' + name + '/; make)') -make_cmds.append('make libwiredtiger.la') + make_cmds.append('ninja -C ' + build_dir + ' ext/compressors/' + name + '/all') +make_cmds.append('ninja -C ' + build_dir + ' libwiredtiger.a') +make_cmds.append('ninja -C ' + build_dir + ' lang/python/all') -inc_paths = [ os.path.join(build_dir, 'src', 'include'), build_dir, '.' ] -lib_paths = [ '.' ] # wiredtiger.so is moved into the top level directory +inc_paths = [ os.path.join(build_dir, 'include'), os.path.join(build_dir, 'config'), build_dir, '.' ] +lib_paths = [ '.' ] check_needed_dependencies(builtins, inc_paths, lib_paths) @@ -320,7 +249,7 @@ if pip_command == 'sdist': if python2: die('Python3 should be used to create a source distribution') - sources, movers = source_filter(get_sources_curdir()) + sources = get_sources_curdir() stage_dir = os.path.join(python_rel_dir, 'stage') shutil.rmtree(stage_dir, True) os.makedirs(stage_dir) @@ -329,23 +258,34 @@ if pip_command == 'sdist': d = os.path.join(stage_dir, os.path.dirname(f)) if not os.path.isdir(d): os.makedirs(d) - if f in movers: - src = movers[f] - else: - src = f # Symlinks are not followed in setup, we need to use real files. - shutil.copy2(src, os.path.join(stage_dir, f)) + shutil.copy2(f, os.path.join(stage_dir, f)) + # Copy files in lang/python/wiredtiger to the root folder. + pywt_path = os.path.join(stage_dir, "lang", "python", "wiredtiger") + pywt_files = os.listdir(pywt_path) + for f in pywt_files: + basename = os.path.basename(f) + src = os.path.join(pywt_path, f) + if basename == 'init.py': + shutil.copy2(src, os.path.join(stage_dir, '__init__.py')) + else: + shutil.copy2(src, os.path.join(stage_dir, basename)) os.chdir(stage_dir) sys.argv.append('--dist-dir=' + os.path.join('..', 'dist')) else: - sources = [ os.path.join(python_rel_dir, 'wiredtiger_wrap.c') ] + sources = [ os.path.join(conf_make_dir, python_rel_dir, 'CMakeFiles', '__wiredtiger.dir', 'wiredtigerPYTHON_wrap.c') ] wt_ext = Extension('_wiredtiger', sources = sources, extra_compile_args = cflags + cppflags, extra_link_args = ldflags, libraries = builtin_libraries, - extra_objects = [ os.path.join(build_dir, '.libs', 'libwiredtiger.a') ], + # FIXME-WT-7905: Remove manual linking of static extension libraries. + # Unfortunately CMake's use of the builtin doesn't currently support linking in the extension + # objects into a static libwiredtiger archive. As a workaround, we need to manually link + # the ext/compressor libraries. + extra_objects = [ os.path.join(build_dir, 'libwiredtiger.a') ] + \ + list(map(lambda name: os.path.join(build_dir, 'ext', 'compressors', name) + '/libwiredtiger_' + name + '.a', builtin_names)), include_dirs = inc_paths, library_dirs = lib_paths, ) @@ -353,7 +293,8 @@ extensions = [ wt_ext ] env = { "CFLAGS" : ' '.join(cflags), "CPPFLAGS" : ' '.join(cppflags), "LDFLAGS" : ' '.join(ldflags), - "PATH" : build_path } + "CMAKE_LIBRARY_PATH" : os.getenv("CMAKE_LIBRARY_PATH", default=""), + "PATH" : os.getenv("PATH", default="") } class BinaryDistribution(Distribution): def is_pure(self): @@ -362,6 +303,10 @@ class BinaryDistribution(Distribution): class WTInstall(setuptools.command.install.install): def run(self): self.run_command("build_ext") + # Copy the SWIG generated python module to our build directory. This will + # then subsequently be installed by pip into the package directory. + shutil.move(os.path.join(conf_make_dir, python_rel_dir, 'wiredtiger.py'), + os.path.join(self.build_lib, 'wiredtiger','swig_wiredtiger.py')) return setuptools.command.install.install.run(self) class WTBuildExt(setuptools.command.build_ext.build_ext): @@ -376,12 +321,12 @@ class WTBuildExt(setuptools.command.build_ext.build_ext): except OSError: pass self.execute( - lambda: build_commands(configure_cmds, conf_make_dir, env), [], + lambda: build_commands(configure_cmds, wt_dir, env), [], 'wiredtiger configure') if not os.path.isfile(makefile): die('configure failed, file does not exist: ' + makefile) self.execute( - lambda: build_commands(make_cmds, conf_make_dir, env), [], + lambda: build_commands(make_cmds, wt_dir, env), [], 'wiredtiger make') open(built_sentinal, 'a').close() return setuptools.command.build_ext.build_ext.run(self) @@ -404,7 +349,7 @@ setup( package_dir = { 'wiredtiger' : '.' }, cmdclass = { 'install': WTInstall, 'build_ext': WTBuildExt }, package_data = { - 'wiredtiger' : [ wt_swig_lib_name, '*.py' ] + 'wiredtiger' : [ '*.py'] }, classifiers=[ 'Intended Audience :: Developers', @@ -417,6 +362,10 @@ setup( 'Operating System :: POSIX :: BSD', 'Operating System :: POSIX :: Linux', 'Operating System :: POSIX :: SunOS/Solaris', + ], + install_requires=[ + 'cmake', + 'ninja', ] ) diff --git a/src/third_party/wiredtiger/src/block/block_addr.c b/src/third_party/wiredtiger/src/block/block_addr.c index abaa0280330..1b06768f7de 100644 --- a/src/third_party/wiredtiger/src/block/block_addr.c +++ b/src/third_party/wiredtiger/src/block/block_addr.c @@ -8,24 +8,54 @@ #include "wt_internal.h" +#define WT_BLOCK_COOKIE_FILEID 0x01 /* The following bytes are an object ID. */ + /* - * __block_buffer_to_addr -- - * Convert a filesystem address cookie into its components, UPDATING the caller's buffer - * reference so it can be called repeatedly to load a buffer. + * __block_addr_unpack -- + * Unpack an address cookie into components, UPDATING the caller's buffer reference so this + * function can be called repeatedly to unpack a buffer containing multiple address cookies. */ static int -__block_buffer_to_addr(WT_BLOCK *block, const uint8_t **pp, uint32_t *objectidp, wt_off_t *offsetp, - uint32_t *sizep, uint32_t *checksump) +__block_addr_unpack(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t **pp, size_t addr_size, + uint32_t *objectidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump) { - uint64_t l, o, s, c; + uint64_t i, o, s, c; + uint8_t flags; + const uint8_t *begin; - if (block->has_objects) - WT_RET(__wt_vunpack_uint(pp, 0, &l)); - else - l = 0; + /* + * Address cookies are a file offset, size and checksum triplet, with optional object ID: unpack + * the trailing object ID if there are bytes following the triple. The checkpoint cookie is more + * complicated: it has four address blocks all having the same object ID. Rather than retrofit + * the object ID into each of those address blocks (which would mean somehow figuring out the + * length of each individual address block), the object ID is appended to the end of the + * checkpoint cookie and this function skips the object ID unpack when the passed in cookie size + * is 0. (This feature is ONLY used by the checkpoint code, all other callers assert the cookie + * size is not 0. We could alternatively have a "checkpoint cookie" boolean, or use a NULL + * object ID address when never returning a object ID, but a cookie size of 0 seems equivalent.) + */ + begin = *pp; WT_RET(__wt_vunpack_uint(pp, 0, &o)); WT_RET(__wt_vunpack_uint(pp, 0, &s)); WT_RET(__wt_vunpack_uint(pp, 0, &c)); + i = 0; + flags = 0; + if (addr_size != 0 && WT_PTRDIFF(*pp, begin) < addr_size) { + flags = **pp; + ++(*pp); + if (LF_ISSET(WT_BLOCK_COOKIE_FILEID)) + WT_RET(__wt_vunpack_uint(pp, 0, &i)); + } + + /* + * If there's an address cookie size and either there is no flag value or the flag value is the + * object ID flag by itself, assert the cookie was entirely consumed. Future extensions will use + * different cookie flag values (although the file-ID flag might still be set, our test is for + * equality). + */ + WT_ASSERT(session, + addr_size == 0 || (flags != 0 && flags != WT_BLOCK_COOKIE_FILEID) || + WT_PTRDIFF(*pp, begin) == addr_size); /* * To avoid storing large offsets, we minimize the value by subtracting a block for description @@ -37,60 +67,73 @@ __block_buffer_to_addr(WT_BLOCK *block, const uint8_t **pp, uint32_t *objectidp, * that's simpler than testing sizes of 0 all over the place. */ if (s == 0) { + *objectidp = 0; *offsetp = 0; - *objectidp = *sizep = *checksump = 0; + *sizep = *checksump = 0; } else { - if (block->has_objects && l == 0 && o != WT_BLOCK_INVALID_OFFSET) - WT_RET_MSG(NULL, EINVAL, "address cookie decoding for Btree with objects has object 0"); - *objectidp = (uint32_t)l; + *objectidp = (uint32_t)i; *offsetp = (wt_off_t)(o + 1) * block->allocsize; *sizep = (uint32_t)s * block->allocsize; *checksump = (uint32_t)c; } + return (0); } /* - * __wt_block_addr_to_buffer -- - * Convert the filesystem components into its address cookie. + * __wt_block_addr_pack -- + * Pack components into an address cookie, UPDATING the caller's buffer reference. */ int -__wt_block_addr_to_buffer(WT_BLOCK *block, uint8_t **pp, uint32_t objectid, wt_off_t offset, +__wt_block_addr_pack(WT_BLOCK *block, uint8_t **pp, uint32_t objectid, wt_off_t offset, uint32_t size, uint32_t checksum) { - uint64_t l, o, s, c; + uint64_t i, o, s, c; - /* See the comment above: this is the reverse operation. */ + /* See the comment above about storing large offsets: this is the reverse operation. */ if (size == 0) { + i = 0; o = WT_BLOCK_INVALID_OFFSET; - l = s = c = 0; + s = c = 0; } else { - l = objectid; + i = objectid; o = (uint64_t)offset / block->allocsize - 1; s = size / block->allocsize; c = checksum; } - if (block->has_objects) { - if (l == 0 && o != WT_BLOCK_INVALID_OFFSET) - WT_RET_MSG(NULL, EINVAL, "address cookie encoding for Btree with objects has object 0"); - WT_RET(__wt_vpack_uint(pp, 0, l)); - } WT_RET(__wt_vpack_uint(pp, 0, o)); WT_RET(__wt_vpack_uint(pp, 0, s)); WT_RET(__wt_vpack_uint(pp, 0, c)); + + /* + * Don't store object IDs of zero, the function that cracks the cookie defaults IDs to 0. + * + * TODO: testing has-objects is not quite right. Ideally, we don't store a object ID if there's + * only a single object. We want to be able to convert existing object to a stack, which means + * starting with a single object with no object IDs, where all future objects in the stack know + * a missing object ID is a reference to the base object. + */ + if (i != 0 && block->has_objects) { + **pp = WT_BLOCK_COOKIE_FILEID; + ++(*pp); + WT_RET(__wt_vpack_uint(pp, 0, i)); + } return (0); } /* - * __wt_block_buffer_to_addr -- - * Convert a filesystem address cookie into its components NOT UPDATING the caller's buffer - * reference. + * __wt_block_addr_unpack -- + * Unpack an address cookie into components, NOT UPDATING the caller's buffer reference. */ int -__wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, uint32_t *objectidp, wt_off_t *offsetp, - uint32_t *sizep, uint32_t *checksump) +__wt_block_addr_unpack(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *p, + size_t addr_size, uint32_t *objectidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump) { - return (__block_buffer_to_addr(block, &p, objectidp, offsetp, sizep, checksump)); + /* Checkpoint passes zero as the cookie size, nobody else should. */ + WT_ASSERT(session, addr_size != 0); + + return ( + __block_addr_unpack(session, block, &p, addr_size, objectidp, offsetp, sizep, checksump)); } /* @@ -104,12 +147,11 @@ __wt_block_addr_invalid( wt_off_t offset; uint32_t checksum, objectid, size; - WT_UNUSED(session); - WT_UNUSED(addr_size); WT_UNUSED(live); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); + WT_RET(__wt_block_addr_unpack( + session, block, addr, addr_size, &objectid, &offset, &size, &checksum)); #ifdef HAVE_DIAGNOSTIC /* @@ -135,10 +177,9 @@ __wt_block_addr_string( wt_off_t offset; uint32_t checksum, objectid, size; - WT_UNUSED(addr_size); - /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); + WT_RET(__wt_block_addr_unpack( + session, block, addr, addr_size, &objectid, &offset, &size, &checksum)); /* Printable representation. */ WT_RET(__wt_buf_fmt(session, buf, @@ -149,46 +190,75 @@ __wt_block_addr_string( } /* - * __block_buffer_to_ckpt -- + * __block_ckpt_unpack -- * Convert a checkpoint cookie into its components. */ static int -__block_buffer_to_ckpt( - WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *p, WT_BLOCK_CKPT *ci) +__block_ckpt_unpack(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *ckpt, + size_t ckpt_size, WT_BLOCK_CKPT *ci) { uint64_t a; - const uint8_t **pp; + uint32_t objectid; + uint8_t flags; + const uint8_t *begin; - ci->version = *p++; + begin = ckpt; + + ci->version = ckpt[0]; if (ci->version != WT_BM_CHECKPOINT_VERSION) WT_RET_MSG(session, WT_ERROR, "unsupported checkpoint version"); - pp = &p; - WT_RET(__block_buffer_to_addr( - block, pp, &ci->root_objectid, &ci->root_offset, &ci->root_size, &ci->root_checksum)); - WT_RET(__block_buffer_to_addr( - block, pp, &ci->alloc.objectid, &ci->alloc.offset, &ci->alloc.size, &ci->alloc.checksum)); - WT_RET(__block_buffer_to_addr( - block, pp, &ci->avail.objectid, &ci->avail.offset, &ci->avail.size, &ci->avail.checksum)); - WT_RET(__block_buffer_to_addr(block, pp, &ci->discard.objectid, &ci->discard.offset, + /* + * See the comment above about address cookies and sizes for an explanation. + * + * Passing an address cookie size of 0 so the unpack function doesn't read an object ID. + */ + ++ckpt; + WT_RET(__block_addr_unpack(session, block, &ckpt, 0, &ci->root_objectid, &ci->root_offset, + &ci->root_size, &ci->root_checksum)); + WT_RET(__block_addr_unpack(session, block, &ckpt, 0, &ci->alloc.objectid, &ci->alloc.offset, + &ci->alloc.size, &ci->alloc.checksum)); + WT_RET(__block_addr_unpack(session, block, &ckpt, 0, &ci->avail.objectid, &ci->avail.offset, + &ci->avail.size, &ci->avail.checksum)); + WT_RET(__block_addr_unpack(session, block, &ckpt, 0, &ci->discard.objectid, &ci->discard.offset, &ci->discard.size, &ci->discard.checksum)); - WT_RET(__wt_vunpack_uint(pp, 0, &a)); + WT_RET(__wt_vunpack_uint(&ckpt, 0, &a)); ci->file_size = (wt_off_t)a; - WT_RET(__wt_vunpack_uint(pp, 0, &a)); + WT_RET(__wt_vunpack_uint(&ckpt, 0, &a)); ci->ckpt_size = a; + /* The first part of the checkpoint cookie is optionally followed by an object ID. */ + objectid = 0; + flags = 0; + if (WT_PTRDIFF(ckpt, begin) != ckpt_size) { + flags = *ckpt++; + if (LF_ISSET(WT_BLOCK_COOKIE_FILEID)) { + WT_RET(__wt_vunpack_uint(&ckpt, 0, &a)); + objectid = (uint32_t)a; + } + } + ci->root_objectid = ci->alloc.objectid = ci->avail.objectid = ci->discard.objectid = objectid; + + /* + * If there is no flag value or the flag value is the object ID flag by itself, assert the + * cookie was entirely consumed. Future extensions will use different cookie flag values + * (although the file-ID flag might still be set, our test is for equality). + */ + WT_ASSERT(session, + (flags != 0 && flags != WT_BLOCK_COOKIE_FILEID) || WT_PTRDIFF(ckpt, begin) == ckpt_size); + return (0); } /* - * __wt_block_buffer_to_ckpt -- + * __wt_block_ckpt_unpack -- * Convert a checkpoint cookie into its components, block manager version. */ int -__wt_block_buffer_to_ckpt( - WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *p, WT_BLOCK_CKPT *ci) +__wt_block_ckpt_unpack(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *ckpt, + size_t ckpt_size, WT_BLOCK_CKPT *ci) { - return (__block_buffer_to_ckpt(session, block, p, ci)); + return (__block_ckpt_unpack(session, block, ckpt, ckpt_size, ci)); } /* @@ -196,27 +266,24 @@ __wt_block_buffer_to_ckpt( * Convert a checkpoint cookie into its components, external utility version. */ int -__wt_block_ckpt_decode(WT_SESSION *wt_session, WT_BLOCK *block, const uint8_t *p, WT_BLOCK_CKPT *ci) - WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) +__wt_block_ckpt_decode(WT_SESSION *wt_session, WT_BLOCK *block, const uint8_t *ckpt, + size_t ckpt_size, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_SESSION_IMPL *session; session = (WT_SESSION_IMPL *)wt_session; - return (__block_buffer_to_ckpt(session, block, p, ci)); + return (__block_ckpt_unpack(session, block, ckpt, ckpt_size, ci)); } /* - * __wt_block_ckpt_to_buffer -- + * __wt_block_ckpt_pack -- * Convert the components into its checkpoint cookie. */ int -__wt_block_ckpt_to_buffer( +__wt_block_ckpt_pack( WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci, bool skip_avail) { uint64_t a; - uint32_t objectid; - - objectid = block->objectid; if (ci->version != WT_BM_CHECKPOINT_VERSION) WT_RET_MSG(session, WT_ERROR, "unsupported checkpoint version"); @@ -224,21 +291,32 @@ __wt_block_ckpt_to_buffer( (*pp)[0] = ci->version; (*pp)++; - WT_RET(__wt_block_addr_to_buffer( - block, pp, objectid, ci->root_offset, ci->root_size, ci->root_checksum)); - WT_RET(__wt_block_addr_to_buffer( - block, pp, objectid, ci->alloc.offset, ci->alloc.size, ci->alloc.checksum)); + /* + * See the comment above about address cookies and sizes for an explanation. + * + * Passing an object ID of 0 so the pack function doesn't store an object ID. + */ + WT_RET(__wt_block_addr_pack(block, pp, 0, ci->root_offset, ci->root_size, ci->root_checksum)); + WT_RET( + __wt_block_addr_pack(block, pp, 0, ci->alloc.offset, ci->alloc.size, ci->alloc.checksum)); if (skip_avail) - WT_RET(__wt_block_addr_to_buffer(block, pp, 0, 0, 0, 0)); + WT_RET(__wt_block_addr_pack(block, pp, 0, 0, 0, 0)); else - WT_RET(__wt_block_addr_to_buffer( - block, pp, objectid, ci->avail.offset, ci->avail.size, ci->avail.checksum)); - WT_RET(__wt_block_addr_to_buffer( - block, pp, objectid, ci->discard.offset, ci->discard.size, ci->discard.checksum)); + WT_RET( + __wt_block_addr_pack(block, pp, 0, ci->avail.offset, ci->avail.size, ci->avail.checksum)); + WT_RET(__wt_block_addr_pack( + block, pp, 0, ci->discard.offset, ci->discard.size, ci->discard.checksum)); a = (uint64_t)ci->file_size; WT_RET(__wt_vpack_uint(pp, 0, a)); a = ci->ckpt_size; WT_RET(__wt_vpack_uint(pp, 0, a)); + /* Don't store object IDs of zero, the function that cracks the cookie defaults IDs to 0. */ + if (block->objectid != 0) { + **pp = WT_BLOCK_COOKIE_FILEID; + ++(*pp); + a = block->objectid; + WT_RET(__wt_vpack_uint(pp, 0, a)); + } return (0); } @@ -249,7 +327,7 @@ __wt_block_ckpt_to_buffer( */ void __wt_ckpt_verbose(WT_SESSION_IMPL *session, WT_BLOCK *block, const char *tag, const char *ckpt_name, - const uint8_t *ckpt_string) + const uint8_t *ckpt_string, size_t ckpt_size) { WT_BLOCK_CKPT *ci, _ci; WT_DECL_ITEM(tmp); @@ -264,10 +342,11 @@ __wt_ckpt_verbose(WT_SESSION_IMPL *session, WT_BLOCK *block, const char *tag, co /* Initialize the checkpoint, crack the cookie. */ ci = &_ci; WT_ERR(__wt_block_ckpt_init(session, ci, "string")); - WT_ERR(__wt_block_buffer_to_ckpt(session, block, ckpt_string, ci)); + WT_ERR(__wt_block_ckpt_unpack(session, block, ckpt_string, ckpt_size, ci)); WT_ERR(__wt_scr_alloc(session, 0, &tmp)); WT_ERR(__wt_buf_fmt(session, tmp, "version=%" PRIu8, ci->version)); + WT_ERR(__wt_buf_catfmt(session, tmp, ", object ID=%" PRIu32, ci->root_objectid)); if (ci->root_offset == WT_BLOCK_INVALID_OFFSET) WT_ERR(__wt_buf_catfmt(session, tmp, ", root=[Empty]")); else diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index 54a1f98942e..addcd9b447f 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -52,7 +52,7 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint ci = NULL; if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) - __wt_ckpt_verbose(session, block, "load", NULL, addr); + __wt_ckpt_verbose(session, block, "load", NULL, addr, addr_size); /* * There's a single checkpoint in the file that can be written, all of the others are read-only. @@ -85,7 +85,7 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint ci->file_size = block->allocsize; else { /* Crack the checkpoint cookie. */ - WT_ERR(__wt_block_buffer_to_ckpt(session, block, addr, ci)); + WT_ERR(__wt_block_ckpt_unpack(session, block, addr, addr_size, ci)); /* Verify sets up next. */ if (block->verify) @@ -94,7 +94,7 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint /* Read any root page. */ if (ci->root_offset != WT_BLOCK_INVALID_OFFSET) { endp = root_addr; - WT_ERR(__wt_block_addr_to_buffer( + WT_ERR(__wt_block_addr_pack( block, &endp, ci->root_objectid, ci->root_offset, ci->root_size, ci->root_checksum)); *root_addr_sizep = WT_PTRDIFF(endp, root_addr); } @@ -283,7 +283,7 @@ __ckpt_extlist_read(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckpt) ci = ckpt->bpriv; WT_RET(__wt_block_ckpt_init(session, ci, ckpt->name)); - WT_RET(__wt_block_buffer_to_ckpt(session, block, ckpt->raw.data, ci)); + WT_RET(__wt_block_ckpt_unpack(session, block, ckpt->raw.data, ckpt->raw.size, ci)); WT_RET(__wt_block_extlist_read(session, block, &ci->alloc, ci->file_size)); WT_RET(__wt_block_extlist_read(session, block, &ci->discard, ci->file_size)); @@ -614,7 +614,7 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase) continue; if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) - __wt_ckpt_verbose(session, block, "delete", ckpt->name, ckpt->raw.data); + __wt_ckpt_verbose(session, block, "delete", ckpt->name, ckpt->raw.data, ckpt->raw.size); /* * Find the checkpoint into which we'll roll this checkpoint's blocks: it's the next real @@ -814,7 +814,7 @@ __ckpt_update( */ WT_RET(__wt_buf_init(session, &ckpt->raw, WT_BLOCK_CHECKPOINT_BUFFER)); endp = ckpt->raw.mem; - WT_RET(__wt_block_ckpt_to_buffer(session, block, &endp, ci, true)); + WT_RET(__wt_block_ckpt_pack(session, block, &endp, ci, true)); ckpt->raw.size = WT_PTRDIFF(endp, ckpt->raw.mem); /* @@ -883,11 +883,11 @@ __ckpt_update( /* Copy the COMPLETE checkpoint information into the checkpoint. */ WT_RET(__wt_buf_init(session, &ckpt->raw, WT_BLOCK_CHECKPOINT_BUFFER)); endp = ckpt->raw.mem; - WT_RET(__wt_block_ckpt_to_buffer(session, block, &endp, ci, false)); + WT_RET(__wt_block_ckpt_pack(session, block, &endp, ci, false)); ckpt->raw.size = WT_PTRDIFF(endp, ckpt->raw.mem); if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) - __wt_ckpt_verbose(session, block, "create", ckpt->name, ckpt->raw.data); + __wt_ckpt_verbose(session, block, "create", ckpt->name, ckpt->raw.data, ckpt->raw.size); return (0); } diff --git a/src/third_party/wiredtiger/src/block/block_ckpt_scan.c b/src/third_party/wiredtiger/src/block/block_ckpt_scan.c index b419a42ed73..74b2257b7a0 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt_scan.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt_scan.c @@ -176,24 +176,26 @@ __block_checkpoint_update(WT_SESSION_IMPL *session, WT_BLOCK *block, struct save checkpoint = info->checkpoint; if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) - __wt_ckpt_verbose(session, block, "import original", NULL, checkpoint->mem); + __wt_ckpt_verbose( + session, block, "import original", NULL, checkpoint->mem, checkpoint->size); /* * Convert the final checkpoint data blob to a WT_BLOCK_CKPT structure, update it with the avail * list information, and convert it back to a data blob. */ - WT_RET(__wt_block_buffer_to_ckpt(session, block, checkpoint->data, &ci)); + WT_RET(__wt_block_ckpt_unpack(session, block, checkpoint->data, checkpoint->size, &ci)); ci.avail.offset = info->offset; ci.avail.size = info->size; ci.avail.checksum = info->checksum; ci.file_size = (wt_off_t)info->file_size; WT_RET(__wt_buf_extend(session, checkpoint, WT_BLOCK_CHECKPOINT_BUFFER)); endp = checkpoint->mem; - WT_RET(__wt_block_ckpt_to_buffer(session, block, &endp, &ci, false)); + WT_RET(__wt_block_ckpt_pack(session, block, &endp, &ci, false)); checkpoint->size = WT_PTRDIFF(endp, checkpoint->mem); if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) - __wt_ckpt_verbose(session, block, "import replace", NULL, checkpoint->mem); + __wt_ckpt_verbose( + session, block, "import replace", NULL, checkpoint->mem, checkpoint->size); return (0); } diff --git a/src/third_party/wiredtiger/src/block/block_compact.c b/src/third_party/wiredtiger/src/block/block_compact.c index 735b4d5c4f4..6d7e87edbee 100644 --- a/src/third_party/wiredtiger/src/block/block_compact.c +++ b/src/third_party/wiredtiger/src/block/block_compact.c @@ -138,11 +138,11 @@ __wt_block_compact_page_skip( wt_off_t limit, offset; uint32_t checksum, objectid, size; - WT_UNUSED(addr_size); *skipp = true; /* Return a default skip. */ /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); + WT_RET(__wt_block_addr_unpack( + session, block, addr, addr_size, &objectid, &offset, &size, &checksum)); /* * If this block is in the chosen percentage of the file and there's a block on the available diff --git a/src/third_party/wiredtiger/src/block/block_ext.c b/src/third_party/wiredtiger/src/block/block_ext.c index 70903985d57..37916e482de 100644 --- a/src/third_party/wiredtiger/src/block/block_ext.c +++ b/src/third_party/wiredtiger/src/block/block_ext.c @@ -565,11 +565,15 @@ __wt_block_free(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, wt_off_t offset; uint32_t checksum, objectid, size; - WT_UNUSED(addr_size); WT_STAT_DATA_INCR(session, block_free); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); + WT_RET(__wt_block_addr_unpack( + session, block, addr, addr_size, &objectid, &offset, &size, &checksum)); + + /* We can't reuse free space in an object. */ + if (objectid != block->objectid) + return (0); __wt_verbose(session, WT_VERB_BLOCK, "free %" PRIu32 ": %" PRIdMAX "/%" PRIdMAX, objectid, (intmax_t)offset, (intmax_t)size); @@ -578,12 +582,11 @@ __wt_block_free(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, WT_RET(__wt_block_misplaced( session, block, "free", offset, size, true, __PRETTY_FUNCTION__, __LINE__)); #endif - if (objectid == block->objectid) { - WT_RET(__wt_block_ext_prealloc(session, 5)); - __wt_spin_lock(session, &block->live_lock); - ret = __wt_block_off_free(session, block, objectid, offset, (wt_off_t)size); - __wt_spin_unlock(session, &block->live_lock); - } + + WT_RET(__wt_block_ext_prealloc(session, 5)); + __wt_spin_lock(session, &block->live_lock); + ret = __wt_block_off_free(session, block, objectid, offset, (wt_off_t)size); + __wt_spin_unlock(session, &block->live_lock); return (ret); } diff --git a/src/third_party/wiredtiger/src/block/block_read.c b/src/third_party/wiredtiger/src/block/block_read.c index 796d071b305..3fec75acfe8 100644 --- a/src/third_party/wiredtiger/src/block/block_read.c +++ b/src/third_party/wiredtiger/src/block/block_read.c @@ -29,7 +29,8 @@ __wt_bm_preload(WT_BM *bm, WT_SESSION_IMPL *session, const uint8_t *addr, size_t WT_STAT_CONN_INCR(session, block_preload); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); + WT_RET(__wt_block_addr_unpack( + session, block, addr, addr_size, &objectid, &offset, &size, &checksum)); WT_RET(__wt_block_fh(session, block, objectid, &fh)); handle = fh->handle; @@ -67,11 +68,11 @@ __wt_bm_read( uint32_t checksum, objectid, size; bool mapped; - WT_UNUSED(addr_size); block = bm->block; /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); + WT_RET(__wt_block_addr_unpack( + session, block, addr, addr_size, &objectid, &offset, &size, &checksum)); /* * Map the block if it's possible. @@ -165,7 +166,8 @@ __wt_bm_corrupt(WT_BM *bm, WT_SESSION_IMPL *session, const uint8_t *addr, size_t WT_ERR(__wt_bm_read(bm, session, tmp, addr, addr_size)); /* Crack the cookie, dump the block. */ - WT_ERR(__wt_block_buffer_to_addr(bm->block, addr, &objectid, &offset, &size, &checksum)); + WT_ERR(__wt_block_addr_unpack( + session, bm->block, addr, addr_size, &objectid, &offset, &size, &checksum)); WT_ERR(__wt_bm_corrupt_dump(session, tmp, objectid, offset, size, checksum)); err: @@ -207,21 +209,6 @@ err: #endif /* - * __wt_block_fh -- - * Get a block file handle. - */ -int -__wt_block_fh(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t objectid, WT_FH **fhp) -{ - if (!block->has_objects) - *fhp = block->fh; - else - WT_RET(__wt_block_tiered_fh(session, block, objectid, fhp)); - - return (0); -} - -/* * __wt_block_read_off -- * Read an addr/size pair referenced block into a buffer. */ diff --git a/src/third_party/wiredtiger/src/block/block_slvg.c b/src/third_party/wiredtiger/src/block/block_slvg.c index 46a87e76f10..32c8f8fc8ed 100644 --- a/src/third_party/wiredtiger/src/block/block_slvg.c +++ b/src/third_party/wiredtiger/src/block/block_slvg.c @@ -152,7 +152,7 @@ __wt_block_salvage_next( /* Re-create the address cookie that should reference this block. */ endp = addr; - WT_ERR(__wt_block_addr_to_buffer(block, &endp, objectid, offset, size, checksum)); + WT_ERR(__wt_block_addr_pack(block, &endp, objectid, offset, size, checksum)); *addr_sizep = WT_PTRDIFF(endp, addr); done: @@ -172,13 +172,12 @@ __wt_block_salvage_valid( wt_off_t offset; uint32_t size, objectid, checksum; - WT_UNUSED(addr_size); - /* * Crack the cookie. If the upper layer took the block, move past it; if the upper layer * rejected the block, move past an allocation size chunk and free it. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); + WT_RET(__wt_block_addr_unpack( + session, block, addr, addr_size, &objectid, &offset, &size, &checksum)); if (valid) block->slvg_off = offset + size; else { diff --git a/src/third_party/wiredtiger/src/block/block_tiered.c b/src/third_party/wiredtiger/src/block/block_tiered.c index 2733e9a1386..d912b058d30 100644 --- a/src/third_party/wiredtiger/src/block/block_tiered.c +++ b/src/third_party/wiredtiger/src/block/block_tiered.c @@ -15,7 +15,6 @@ static int __block_switch_writeable(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t object_id) { - WT_DECL_RET; WT_FH *new_fh, *old_fh; /* @@ -25,27 +24,29 @@ __block_switch_writeable(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t obj * requests in flight. */ old_fh = block->fh; - WT_ERR(block->opener->open( + WT_RET(block->opener->open( block->opener, session, object_id, WT_FS_OPEN_FILE_TYPE_DATA, block->file_flags, &new_fh)); block->fh = new_fh; block->objectid = object_id; - WT_ERR(__wt_close(session, &old_fh)); - -err: - return (ret); + return (__wt_close(session, &old_fh)); } /* - * __wt_block_tiered_fh -- - * Open an object from the shared tier. + * __wt_block_fh -- + * Get a block file handle. */ int -__wt_block_tiered_fh(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t object_id, WT_FH **fhp) +__wt_block_fh(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t object_id, WT_FH **fhp) { - WT_DECL_ITEM(tmp); WT_DECL_RET; + /* It's the local object if there's no object ID or the object ID matches our own. */ + if (object_id == 0 || object_id == block->objectid) { + *fhp = block->fh; + return (0); + } + /* * FIXME-WT-7470: take a read lock to get a handle, and a write lock to open a handle or extend * the array. @@ -63,15 +64,24 @@ __wt_block_tiered_fh(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t object_ if ((*fhp = block->ofh[object_id]) != NULL) return (0); - WT_RET(__wt_scr_alloc(session, 0, &tmp)); - WT_ERR(block->opener->open(block->opener, session, object_id, WT_FS_OPEN_FILE_TYPE_DATA, - WT_FS_OPEN_READONLY | block->file_flags, &block->ofh[object_id])); - *fhp = block->ofh[object_id]; - WT_ASSERT(session, *fhp != NULL); + /* + * Fail gracefully if we don't have an opener, or if the opener fails: a release that can't read + * tiered storage blocks might have been pointed at a file that it can read, but that references + * files it doesn't know about, or there may have been some other mismatch. Regardless, we want + * to log a specific error message, we're missing a file. + */ + ret = block->opener->open == NULL ? + WT_NOTFOUND : + block->opener->open(block->opener, session, object_id, WT_FS_OPEN_FILE_TYPE_DATA, + WT_FS_OPEN_READONLY | block->file_flags, &block->ofh[object_id]); + if (ret == 0) { + *fhp = block->ofh[object_id]; + return (0); + } -err: - __wt_scr_free(session, &tmp); - return (ret); + WT_RET_MSG(session, ret, + "object %s with ID %" PRIu32 " referenced unknown object with ID %" PRIu32, block->name, + block->objectid, object_id); } /* diff --git a/src/third_party/wiredtiger/src/block/block_vrfy.c b/src/third_party/wiredtiger/src/block/block_vrfy.c index 28c95415254..01ec16f59ab 100644 --- a/src/third_party/wiredtiger/src/block/block_vrfy.c +++ b/src/third_party/wiredtiger/src/block/block_vrfy.c @@ -119,7 +119,7 @@ __verify_last_avail(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckpt) ci = &_ci; WT_RET(__wt_block_ckpt_init(session, ci, ckpt->name)); - WT_ERR(__wt_block_buffer_to_ckpt(session, block, ckpt->raw.data, ci)); + WT_ERR(__wt_block_ckpt_unpack(session, block, ckpt->raw.data, ckpt->raw.size, ci)); el = &ci->avail; if (el->offset != WT_BLOCK_INVALID_OFFSET) { @@ -148,7 +148,7 @@ __verify_set_file_size(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckpt) ci = &_ci; WT_RET(__wt_block_ckpt_init(session, ci, ckpt->name)); - WT_ERR(__wt_block_buffer_to_ckpt(session, block, ckpt->raw.data, ci)); + WT_ERR(__wt_block_ckpt_unpack(session, block, ckpt->raw.data, ckpt->raw.size, ci)); if (block->verify_layout) { WT_ERR(__wt_scr_alloc(session, 0, &tmp)); @@ -318,10 +318,9 @@ __wt_block_verify_addr( wt_off_t offset; uint32_t checksum, objectid, size; - WT_UNUSED(addr_size); - /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); + WT_RET(__wt_block_addr_unpack( + session, block, addr, addr_size, &objectid, &offset, &size, &checksum)); /* Add to the per-file list. */ WT_RET(__verify_filefrag_add(session, block, NULL, offset, size, false)); diff --git a/src/third_party/wiredtiger/src/block/block_write.c b/src/third_party/wiredtiger/src/block/block_write.c index 72381c5492d..442d69927f8 100644 --- a/src/third_party/wiredtiger/src/block/block_write.c +++ b/src/third_party/wiredtiger/src/block/block_write.c @@ -196,7 +196,7 @@ __wt_block_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint8_ data_checksum, checkpoint_io, false)); endp = addr; - WT_RET(__wt_block_addr_to_buffer(block, &endp, objectid, offset, size, checksum)); + WT_RET(__wt_block_addr_pack(block, &endp, objectid, offset, size, checksum)); *addr_sizep = WT_PTRDIFF(endp, addr); return (0); diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index 96ad1432c87..32a6172cde6 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -116,7 +116,7 @@ new_page: __wt_upd_value_clear(cbt->upd_value); if (cbt->ins != NULL) restart_read: - WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, cbt->ins->upd, NULL)); + WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, cbt->ins->upd)); if (cbt->upd_value->type == WT_UPDATE_INVALID) { cbt->v = __bit_getv_recno(cbt->ref, cbt->recno, btree->bitcnt); cbt->iface.value.data = &cbt->v; @@ -211,6 +211,7 @@ __cursor_var_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skip return (WT_NOTFOUND); __cursor_set_recno(cbt, cbt->ref->ref_recno); cbt->cip_saved = NULL; + F_CLR(cbt, WT_CBT_CACHEABLE_RLE_CELL); goto new_page; } @@ -245,56 +246,84 @@ restart_read: } /* - * If we're at the same slot as the last reference and there's no matching insert list item, - * re-use the return information (so encoded items with large repeat counts aren't - * repeatedly decoded). Otherwise, unpack the cell and build the return information. + * There's no matching insert list item. If we're on the same slot as the last reference, + * and the cell is cacheable (it might not be if it's not globally visible), reuse the + * previous return data to avoid repeatedly decoding items. */ - if (cbt->cip_saved != cip) { - cell = WT_COL_PTR(page, cip); - __wt_cell_unpack_kv(session, page->dsk, cell, &unpack); - if (unpack.type == WT_CELL_DEL) { - if ((rle = __wt_cell_rle(&unpack)) == 1) { - ++*skippedp; - continue; - } - - /* - * There can be huge gaps in the variable-length column-store name space appearing - * as deleted records. If more than one deleted record, do the work of finding the - * next record to return instead of looping through the records. - * - * First, find the smallest record in the update list that's larger than the current - * record. - */ - ins = __col_insert_search_gt(cbt->ins_head, cbt->recno); - - /* - * Second, for records with RLEs greater than 1, the above call to __col_var_search - * located this record in the page's list of repeating records, and returned the - * starting record. The starting record plus the RLE is the record to which we could - * skip, if there was no smaller record in the update list. - */ - cbt->recno = rle_start + rle; - if (ins != NULL && WT_INSERT_RECNO(ins) < cbt->recno) - cbt->recno = WT_INSERT_RECNO(ins); + if (cbt->cip_saved == cip && F_ISSET(cbt, WT_CBT_CACHEABLE_RLE_CELL)) { + F_CLR(&cbt->iface, WT_CURSTD_VALUE_EXT); + F_SET(&cbt->iface, WT_CURSTD_VALUE_INT); + cbt->iface.value.data = cbt->tmp->data; + cbt->iface.value.size = cbt->tmp->size; + return (0); + } - /* Adjust for the outer loop increment. */ - --cbt->recno; + /* Unpack the cell and build the return information. */ + cell = WT_COL_PTR(page, cip); + __wt_cell_unpack_kv(session, page->dsk, cell, &unpack); + rle = __wt_cell_rle(&unpack); + if (unpack.type == WT_CELL_DEL) { + if (rle == 1) { ++*skippedp; continue; } - WT_RET(__wt_bt_col_var_cursor_walk_txn_read(session, cbt, page, &unpack, cip)); - if (cbt->upd_value->type == WT_UPDATE_INVALID || - cbt->upd_value->type == WT_UPDATE_TOMBSTONE) { - ++*skippedp; - continue; - } + /* + * There can be huge gaps in the variable-length column-store name space appearing as + * deleted records. If more than one deleted record, do the work of finding the next + * record to return instead of looping through the records. + * + * First, find the smallest record in the update list that's larger than the current + * record. + */ + ins = __col_insert_search_gt(cbt->ins_head, cbt->recno); - return (0); + /* + * Second, for records with RLEs greater than 1, the above call to __col_var_search + * located this record in the page's list of repeating records, and returned the + * starting record. The starting record plus the RLE is the record to which we could + * skip, if there was no smaller record in the update list. + */ + cbt->recno = rle_start + rle; + if (ins != NULL && WT_INSERT_RECNO(ins) < cbt->recno) + cbt->recno = WT_INSERT_RECNO(ins); + + /* Adjust for the outer loop increment. */ + --cbt->recno; + ++*skippedp; + continue; + } + + WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, cbt->ins ? cbt->ins->upd : NULL)); + if (cbt->upd_value->type == WT_UPDATE_INVALID || + cbt->upd_value->type == WT_UPDATE_TOMBSTONE) { + ++*skippedp; + continue; } - cbt->iface.value.data = cbt->tmp->data; - cbt->iface.value.size = cbt->tmp->size; + WT_RET(__wt_value_return(cbt, cbt->upd_value)); + + /* + * It is only safe to cache the value for other keys in the same RLE cell if it is globally + * visible. Otherwise, there might be some older timestamp where the value isn't uniform + * across the cell. Always set cip_saved so it's easy to tell when we change cells. + */ + cbt->cip_saved = cip; + if (rle > 1 && + __wt_txn_visible_all(session, unpack.tw.start_txn, unpack.tw.durable_start_ts)) { + /* + * Copy the value into cbt->tmp to cache it. This is perhaps unfortunate, because + * copying isn't free, but it's currently necessary. The memory we're copying might be + * on the disk page (which is safe because the page is pinned as long as the cursor is + * sitting on it) but if not it belongs to cbt->upd_value, and that (though it belongs + * to the cursor and won't disappear arbitrarily) might be invalidated or changed by + * other paths through this function on a subsequent call but before we are done with + * this RLE cell. In principle those paths could clear WT_CBT_CACHEABLE_RLE_CELL, but + * the code is currently structured in a way that makes that difficult. + */ + WT_RET(__wt_buf_set(session, cbt->tmp, cbt->iface.value.data, cbt->iface.value.size)); + F_SET(cbt, WT_CBT_CACHEABLE_RLE_CELL); + } else + F_CLR(cbt, WT_CBT_CACHEABLE_RLE_CELL); return (0); } /* NOTREACHED */ @@ -413,8 +442,8 @@ restart_read_page: WT_STAT_CONN_DATA_INCR(session, cursor_search_near_prefix_fast_paths); return (WT_NOTFOUND); } - WT_RET(__wt_txn_read( - session, cbt, &cbt->iface.key, WT_RECNO_OOB, WT_ROW_UPDATE(page, rip), NULL)); + WT_RET( + __wt_txn_read(session, cbt, &cbt->iface.key, WT_RECNO_OOB, WT_ROW_UPDATE(page, rip))); if (cbt->upd_value->type == WT_UPDATE_INVALID) { ++*skippedp; continue; @@ -590,6 +619,7 @@ __wt_btcur_iterate_setup(WT_CURSOR_BTREE *cbt) /* Clear saved iteration cursor position information. */ cbt->cip_saved = NULL; cbt->rip_saved = NULL; + F_CLR(cbt, WT_CBT_CACHEABLE_RLE_CELL); /* * If we don't have a search page, then we're done, we're starting at the beginning or end of diff --git a/src/third_party/wiredtiger/src/btree/bt_curprev.c b/src/third_party/wiredtiger/src/btree/bt_curprev.c index 5eb2dbdbc25..bcf9f7e6d5a 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curprev.c +++ b/src/third_party/wiredtiger/src/btree/bt_curprev.c @@ -256,7 +256,7 @@ new_page: __wt_upd_value_clear(cbt->upd_value); if (cbt->ins != NULL) restart_read: - WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, cbt->ins->upd, NULL)); + WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, cbt->ins->upd)); if (cbt->upd_value->type == WT_UPDATE_INVALID) { cbt->v = __bit_getv_recno(cbt->ref, cbt->recno, btree->bitcnt); cbt->iface.value.data = &cbt->v; @@ -327,7 +327,7 @@ __cursor_var_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skip WT_INSERT *ins; WT_PAGE *page; WT_SESSION_IMPL *session; - uint64_t rle_start; + uint64_t rle, rle_start; session = CUR2S(cbt); page = cbt->ref->page; @@ -350,6 +350,7 @@ __cursor_var_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skip return (WT_NOTFOUND); __cursor_set_recno(cbt, cbt->last_standard_recno); cbt->cip_saved = NULL; + F_CLR(cbt, WT_CBT_CACHEABLE_RLE_CELL); goto new_page; } @@ -385,55 +386,84 @@ restart_read: } /* - * If we're at the same slot as the last reference and there's no matching insert list item, - * re-use the return information (so encoded items with large repeat counts aren't - * repeatedly decoded). Otherwise, unpack the cell and build the return information. + * There's no matching insert list item. If we're on the same slot as the last reference, + * and the cell is cacheable (it might not be if it's not globally visible), reuse the + * previous return data to avoid repeatedly decoding items. */ - if (cbt->cip_saved != cip) { - cell = WT_COL_PTR(page, cip); - __wt_cell_unpack_kv(session, page->dsk, cell, &unpack); - if (unpack.type == WT_CELL_DEL) { - if (__wt_cell_rle(&unpack) == 1) { - ++*skippedp; - continue; - } - - /* - * There can be huge gaps in the variable-length column-store name space appearing - * as deleted records. If more than one deleted record, do the work of finding the - * next record to return instead of looping through the records. - * - * First, find the largest record in the update list that's smaller than the current - * record. - */ - ins = __col_insert_search_lt(cbt->ins_head, cbt->recno); - - /* - * Second, for records with RLEs greater than 1, the above call to __col_var_search - * located this record in the page's list of repeating records, and returned the - * starting record. The starting record - 1 is the record to which we could skip, if - * there was no larger record in the update list. - */ - cbt->recno = rle_start - 1; - if (ins != NULL && WT_INSERT_RECNO(ins) > cbt->recno) - cbt->recno = WT_INSERT_RECNO(ins); + if (cbt->cip_saved == cip && F_ISSET(cbt, WT_CBT_CACHEABLE_RLE_CELL)) { + F_CLR(&cbt->iface, WT_CURSTD_VALUE_EXT); + F_SET(&cbt->iface, WT_CURSTD_VALUE_INT); + cbt->iface.value.data = cbt->tmp->data; + cbt->iface.value.size = cbt->tmp->size; + return (0); + } - /* Adjust for the outer loop decrement. */ - ++cbt->recno; + /* Unpack the cell and build the return information. */ + cell = WT_COL_PTR(page, cip); + __wt_cell_unpack_kv(session, page->dsk, cell, &unpack); + rle = __wt_cell_rle(&unpack); + if (unpack.type == WT_CELL_DEL) { + if (rle == 1) { ++*skippedp; continue; } - WT_RET(__wt_bt_col_var_cursor_walk_txn_read(session, cbt, page, &unpack, cip)); - if (cbt->upd_value->type == WT_UPDATE_INVALID || - cbt->upd_value->type == WT_UPDATE_TOMBSTONE) { - ++*skippedp; - continue; - } - return (0); + /* + * There can be huge gaps in the variable-length column-store name space appearing as + * deleted records. If more than one deleted record, do the work of finding the next + * record to return instead of looping through the records. + * + * First, find the largest record in the update list that's smaller than the current + * record. + */ + ins = __col_insert_search_lt(cbt->ins_head, cbt->recno); + + /* + * Second, for records with RLEs greater than 1, the above call to __col_var_search + * located this record in the page's list of repeating records, and returned the + * starting record. The starting record - 1 is the record to which we could skip, if + * there was no larger record in the update list. + */ + cbt->recno = rle_start - 1; + if (ins != NULL && WT_INSERT_RECNO(ins) > cbt->recno) + cbt->recno = WT_INSERT_RECNO(ins); + + /* Adjust for the outer loop decrement. */ + ++cbt->recno; + ++*skippedp; + continue; + } + + WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, cbt->ins ? cbt->ins->upd : NULL)); + if (cbt->upd_value->type == WT_UPDATE_INVALID || + cbt->upd_value->type == WT_UPDATE_TOMBSTONE) { + ++*skippedp; + continue; } - cbt->iface.value.data = cbt->tmp->data; - cbt->iface.value.size = cbt->tmp->size; + WT_RET(__wt_value_return(cbt, cbt->upd_value)); + + /* + * It is only safe to cache the value for other keys in the same RLE cell if it is globally + * visible. Otherwise, there might be some older timestamp where the value isn't uniform + * across the cell. Always set cip_saved so it's easy to tell when we change cells. + */ + cbt->cip_saved = cip; + if (rle > 1 && + __wt_txn_visible_all(session, unpack.tw.start_txn, unpack.tw.durable_start_ts)) { + /* + * Copy the value into cbt->tmp to cache it. This is perhaps unfortunate, because + * copying isn't free, but it's currently necessary. The memory we're copying might be + * on the disk page (which is safe because the page is pinned as long as the cursor is + * sitting on it) but if not it belongs to cbt->upd_value, and that (though it belongs + * to the cursor and won't disappear arbitrarily) might be invalidated or changed by + * other paths through this function on a subsequent call but before we are done with + * this RLE cell. In principle those paths could clear WT_CBT_CACHEABLE_RLE_CELL, but + * the code is currently structured in a way that makes that difficult. + */ + WT_RET(__wt_buf_set(session, cbt->tmp, cbt->iface.value.data, cbt->iface.value.size)); + F_SET(cbt, WT_CBT_CACHEABLE_RLE_CELL); + } else + F_CLR(cbt, WT_CBT_CACHEABLE_RLE_CELL); return (0); } /* NOTREACHED */ @@ -561,8 +591,8 @@ restart_read_page: WT_STAT_CONN_DATA_INCR(session, cursor_search_near_prefix_fast_paths); return (WT_NOTFOUND); } - WT_RET(__wt_txn_read( - session, cbt, &cbt->iface.key, WT_RECNO_OOB, WT_ROW_UPDATE(page, rip), NULL)); + WT_RET( + __wt_txn_read(session, cbt, &cbt->iface.key, WT_RECNO_OOB, WT_ROW_UPDATE(page, rip))); if (cbt->upd_value->type == WT_UPDATE_INVALID) { ++*skippedp; continue; diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 46da608e096..ef8e42f97e5 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -297,7 +297,7 @@ __wt_cursor_valid(WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint64_t recno, bool *vali * Check for an update ondisk or in the history store. For column store, an insert object * can have the same key as an on-page or history store object. */ - WT_RET(__wt_txn_read(session, cbt, key, recno, NULL, NULL)); + WT_RET(__wt_txn_read(session, cbt, key, recno, cbt->ins ? cbt->ins->upd : NULL)); if (cbt->upd_value->type != WT_UPDATE_INVALID) { if (cbt->upd_value->type == WT_UPDATE_TOMBSTONE) return (0); @@ -325,8 +325,7 @@ __wt_cursor_valid(WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint64_t recno, bool *vali WT_RET(__wt_txn_read(session, cbt, key, WT_RECNO_OOB, (page->modify != NULL && page->modify->mod_row_update != NULL) ? page->modify->mod_row_update[cbt->slot] : - NULL, - NULL)); + NULL)); if (cbt->upd_value->type != WT_UPDATE_INVALID) { if (cbt->upd_value->type == WT_UPDATE_TOMBSTONE) return (0); diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index 0e8daa06e34..81f419184f5 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -383,17 +383,19 @@ __wt_debug_offset( /* * This routine depends on the default block manager's view of files, where an address consists - * of a file offset, length, and checksum. This is for debugging only: other block managers - * might not see a file or address the same way, that's why there's no block manager method. + * of a file ID, file offset, length, and checksum. This is only for debugging, other block + * managers might not describe underlying objects the same way, that's why there's no block + * manager method. * * Convert the triplet into an address structure. */ block = S2BT(session)->bm->block; endp = addr; - WT_RET(__wt_block_addr_to_buffer(block, &endp, block->objectid, offset, size, checksum)); + WT_RET(__wt_block_addr_pack(block, &endp, block->objectid, offset, size, checksum)); /* - * Read the address through the btree I/O functions (so the block is decompressed as necessary). + * Read the address through the btree I/O functions (so the block is decompressed and/or + * unencrypted as necessary). */ WT_RET(__wt_scr_alloc(session, 0, &buf)); WT_ERR(__wt_bt_read(session, buf, addr, WT_PTRDIFF(endp, addr))); diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index fdbb192774c..d5251268944 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -655,7 +655,7 @@ __wt_btree_tree_open(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_ * the disk image on return, the in-memory object steals it. */ WT_ERR(__wt_page_inmem(session, NULL, dsk.data, - WT_DATA_IN_ITEM(&dsk) ? WT_PAGE_DISK_ALLOC : WT_PAGE_DISK_MAPPED, &page)); + WT_DATA_IN_ITEM(&dsk) ? WT_PAGE_DISK_ALLOC : WT_PAGE_DISK_MAPPED, &page, NULL)); dsk.mem = NULL; /* Finish initializing the root, root reference links. */ diff --git a/src/third_party/wiredtiger/src/btree/bt_page.c b/src/third_party/wiredtiger/src/btree/bt_page.c index 22766b682bf..d37b3cdb204 100644 --- a/src/third_party/wiredtiger/src/btree/bt_page.c +++ b/src/third_party/wiredtiger/src/btree/bt_page.c @@ -10,9 +10,9 @@ static void __inmem_col_fix(WT_SESSION_IMPL *, WT_PAGE *); static void __inmem_col_int(WT_SESSION_IMPL *, WT_PAGE *); -static int __inmem_col_var(WT_SESSION_IMPL *, WT_PAGE *, uint64_t, size_t *); +static int __inmem_col_var(WT_SESSION_IMPL *, WT_PAGE *, uint64_t, bool *, size_t *); static int __inmem_row_int(WT_SESSION_IMPL *, WT_PAGE *, size_t *); -static int __inmem_row_leaf(WT_SESSION_IMPL *, WT_PAGE *); +static int __inmem_row_leaf(WT_SESSION_IMPL *, WT_PAGE *, bool *); static int __inmem_row_leaf_entries(WT_SESSION_IMPL *, const WT_PAGE_HEADER *, uint32_t *); /* @@ -121,12 +121,184 @@ err: } /* + * __page_inmem_prepare_update -- + * Create the actual update for a prepared value. + */ +static int +__page_inmem_prepare_update(WT_SESSION_IMPL *session, WT_ITEM *value, WT_CELL_UNPACK_KV *unpack, + WT_UPDATE **updp, size_t *sizep) +{ + WT_DECL_RET; + WT_UPDATE *upd, *tombstone; + size_t size, total_size; + + *sizep = 0; + + tombstone = upd = NULL; + total_size = 0; + + WT_RET(__wt_upd_alloc(session, value, WT_UPDATE_STANDARD, &upd, &size)); + total_size += size; + upd->durable_ts = unpack->tw.durable_start_ts; + upd->start_ts = unpack->tw.start_ts; + upd->txnid = unpack->tw.start_txn; + F_SET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS); + + /* + * Instantiate both update and tombstone if the prepared update is a tombstone. This is required + * to ensure that written prepared delete operation must be removed from the data store, when + * the prepared transaction gets rollback. + */ + if (WT_TIME_WINDOW_HAS_STOP(&unpack->tw)) { + WT_ERR(__wt_upd_alloc_tombstone(session, &tombstone, &size)); + total_size += size; + tombstone->durable_ts = WT_TS_NONE; + tombstone->start_ts = unpack->tw.stop_ts; + tombstone->txnid = unpack->tw.stop_txn; + tombstone->prepare_state = WT_PREPARE_INPROGRESS; + F_SET(tombstone, WT_UPDATE_PREPARE_RESTORED_FROM_DS); + + /* + * Mark the update also as in-progress if the update and tombstone are from same transaction + * by comparing both the transaction and timestamps as the transaction information gets lost + * after restart. + */ + if (unpack->tw.start_ts == unpack->tw.stop_ts && + unpack->tw.durable_start_ts == unpack->tw.durable_stop_ts && + unpack->tw.start_txn == unpack->tw.stop_txn) { + upd->durable_ts = WT_TS_NONE; + upd->prepare_state = WT_PREPARE_INPROGRESS; + } + + tombstone->next = upd; + *updp = tombstone; + } else { + upd->durable_ts = WT_TS_NONE; + upd->prepare_state = WT_PREPARE_INPROGRESS; + *updp = upd; + } + + *sizep = total_size; + return (0); + +err: + __wt_free(session, upd); + __wt_free(session, tombstone); + + return (ret); +} + +/* + * __wt_page_inmem_prepare -- + * Instantiate prepared updates. + */ +int +__wt_page_inmem_prepare(WT_SESSION_IMPL *session, WT_REF *ref) +{ + WT_CELL *cell; + WT_CELL_UNPACK_KV unpack; + WT_COL *cip; + WT_CURSOR_BTREE cbt; + WT_DECL_ITEM(key); + WT_DECL_ITEM(value); + WT_DECL_RET; + WT_PAGE *page; + WT_ROW *rip; + WT_UPDATE *upd; + size_t size, total_size; + uint64_t recno, rle; + uint32_t i; + + page = ref->page; + upd = NULL; + total_size = 0; + + /* + * We don't handle in-memory prepare resolution here, and prepare only applies to row-store and + * variable-length column store leaf pages. + */ + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_IN_MEMORY)); + WT_ASSERT(session, page->type == WT_PAGE_COL_VAR || page->type == WT_PAGE_ROW_LEAF); + + __wt_btcur_init(session, &cbt); + __wt_btcur_open(&cbt); + + WT_ERR(__wt_scr_alloc(session, 0, &value)); + if (page->type == WT_PAGE_COL_VAR) { + recno = ref->ref_recno; + WT_COL_FOREACH (page, cip, i) { + /* Search for prepare records. */ + cell = WT_COL_PTR(page, cip); + __wt_cell_unpack_kv(session, page->dsk, cell, &unpack); + rle = __wt_cell_rle(&unpack); + if (!unpack.tw.prepare) { + recno += rle; + continue; + } + + /* Get the value. */ + WT_ERR(__wt_page_cell_data_ref(session, page, &unpack, value)); + + /* For each record, create an update to resolve the prepare. */ + for (; rle > 0; --rle, ++recno) { + /* Create an update to resolve the prepare. */ + WT_ERR(__page_inmem_prepare_update(session, value, &unpack, &upd, &size)); + total_size += size; + + /* Search the page and apply the modification. */ + WT_ERR(__wt_col_search(&cbt, recno, ref, true, NULL)); +#ifdef HAVE_DIAGNOSTIC + WT_ERR(__wt_col_modify(&cbt, recno, NULL, upd, WT_UPDATE_INVALID, true, true)); +#else + WT_ERR(__wt_col_modify(&cbt, recno, NULL, upd, WT_UPDATE_INVALID, true)); +#endif + upd = NULL; + } + } + } else { + WT_ERR(__wt_scr_alloc(session, 0, &key)); + WT_ROW_FOREACH (page, rip, i) { + /* Search for prepare records. */ + __wt_row_leaf_value_cell(session, page, rip, &unpack); + if (!unpack.tw.prepare) + continue; + + /* Get the key/value pair and create an update to resolve the prepare. */ + WT_ERR(__wt_row_leaf_key(session, page, rip, key, false)); + WT_ERR(__wt_page_cell_data_ref(session, page, &unpack, value)); + WT_ERR(__page_inmem_prepare_update(session, value, &unpack, &upd, &size)); + total_size += size; + + /* Search the page and apply the modification. */ + WT_ERR(__wt_row_search(&cbt, key, true, ref, true, NULL)); +#ifdef HAVE_DIAGNOSTIC + WT_ERR(__wt_row_modify(&cbt, key, NULL, upd, WT_UPDATE_INVALID, true, true)); +#else + WT_ERR(__wt_row_modify(&cbt, key, NULL, upd, WT_UPDATE_INVALID, true)); +#endif + upd = NULL; + } + } + + __wt_cache_page_inmem_incr(session, page, total_size); + + if (0) { +err: + __wt_free_update_list(session, &upd); + } + WT_TRET(__wt_btcur_close(&cbt, true)); + __wt_scr_free(session, &key); + __wt_scr_free(session, &value); + return (ret); +} + +/* * __wt_page_inmem -- * Build in-memory page information. */ int -__wt_page_inmem( - WT_SESSION_IMPL *session, WT_REF *ref, const void *image, uint32_t flags, WT_PAGE **pagep) +__wt_page_inmem(WT_SESSION_IMPL *session, WT_REF *ref, const void *image, uint32_t flags, + WT_PAGE **pagep, bool *preparedp) { WT_DECL_RET; WT_PAGE *page; @@ -136,6 +308,9 @@ __wt_page_inmem( *pagep = NULL; + if (preparedp != NULL) + *preparedp = false; + dsk = image; alloc_entries = 0; @@ -206,13 +381,13 @@ __wt_page_inmem( __inmem_col_int(session, page); break; case WT_PAGE_COL_VAR: - WT_ERR(__inmem_col_var(session, page, dsk->recno, &size)); + WT_ERR(__inmem_col_var(session, page, dsk->recno, preparedp, &size)); break; case WT_PAGE_ROW_INT: WT_ERR(__inmem_row_int(session, page, &size)); break; case WT_PAGE_ROW_LEAF: - WT_ERR(__inmem_row_leaf(session, page)); + WT_ERR(__inmem_row_leaf(session, page, preparedp)); break; default: WT_ERR(__wt_illegal_value(session, page->type)); @@ -314,7 +489,8 @@ __inmem_col_var_repeats(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t *np) * Build in-memory index for variable-length, data-only leaf pages in column-store trees. */ static int -__inmem_col_var(WT_SESSION_IMPL *session, WT_PAGE *page, uint64_t recno, size_t *sizep) +__inmem_col_var( + WT_SESSION_IMPL *session, WT_PAGE *page, uint64_t recno, bool *preparedp, size_t *sizep) { WT_CELL_UNPACK_KV unpack; WT_COL *cip; @@ -322,10 +498,12 @@ __inmem_col_var(WT_SESSION_IMPL *session, WT_PAGE *page, uint64_t recno, size_t size_t size; uint64_t rle; uint32_t indx, n, repeat_off; + bool prepare; void *p; repeats = NULL; repeat_off = 0; + prepare = false; /* * Walk the page, building references: the page contains unsorted value items. The value items @@ -359,11 +537,19 @@ __inmem_col_var(WT_SESSION_IMPL *session, WT_PAGE *page, uint64_t recno, size_t repeats[repeat_off].recno = recno; repeats[repeat_off++].rle = rle; } + + /* If we find a prepare, we'll have to instantiate it in the update chain later. */ + if (unpack.tw.prepare) + prepare = true; + indx++; recno += rle; } WT_CELL_FOREACH_END; + if (preparedp != NULL && prepare) + *preparedp = true; + return (0); } @@ -529,24 +715,20 @@ __inmem_row_leaf_entries(WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk, ui * Build in-memory index for row-store leaf pages. */ static int -__inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) +__inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page, bool *preparedp) { - enum { PREPARE_INSTANTIATE, PREPARE_INITIALIZED, PREPARE_IGNORE } prepare; WT_BTREE *btree; WT_CELL_UNPACK_KV unpack; - WT_DECL_ITEM(value); WT_DECL_RET; WT_ROW *rip; - WT_UPDATE *tombstone, *upd; - size_t size, total_size; uint32_t best_prefix_count, best_prefix_start, best_prefix_stop; uint32_t last_slot, prefix_count, prefix_start, prefix_stop, slot; uint8_t smallest_prefix; + bool prepare; btree = S2BT(session); - tombstone = upd = NULL; last_slot = 0; - size = total_size = 0; + prepare = false; /* The code depends on the prefix count variables, other initialization shouldn't matter. */ best_prefix_count = prefix_count = 0; @@ -554,15 +736,6 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) prefix_start = prefix_stop = 0; /* [-Wconditional-uninitialized] */ best_prefix_start = best_prefix_stop = 0; /* [-Wconditional-uninitialized] */ - /* - * Optionally instantiate prepared updates. In-memory databases restore non-obsolete updates on - * the page as part of the __split_multi_inmem function. - */ - prepare = F_ISSET(session, WT_SESSION_INSTANTIATE_PREPARE) && - !F_ISSET(S2C(session), WT_CONN_IN_MEMORY) ? - PREPARE_INSTANTIATE : - PREPARE_IGNORE; - /* Walk the page, building indices. */ rip = page->pg_row; WT_CELL_FOREACH_KV (session, page->dsk, unpack) { @@ -657,72 +830,9 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) WT_ERR(__wt_illegal_value(session, unpack.type)); } - if (!unpack.tw.prepare || prepare == PREPARE_IGNORE) - continue; - - /* First prepared transaction setup. */ - if (prepare == PREPARE_INSTANTIATE) { - WT_ERR(__wt_page_modify_init(session, page)); - if (!F_ISSET(btree, WT_BTREE_READONLY)) - __wt_page_modify_set(session, page); - - /* Allocate the per-page update array. */ - WT_ERR(__wt_calloc_def(session, page->entries, &page->modify->mod_row_update)); - total_size += page->entries * sizeof(*page->modify->mod_row_update); - - WT_ERR(__wt_scr_alloc(session, 0, &value)); - - prepare = PREPARE_INITIALIZED; - } - - /* Make sure that there is no in-memory update for this key. */ - WT_ASSERT(session, page->modify->mod_row_update[WT_ROW_SLOT(page, rip - 1)] == NULL); - - /* Take the value from the page cell. */ - WT_ERR(__wt_page_cell_data_ref(session, page, &unpack, value)); - - WT_ERR(__wt_upd_alloc(session, value, WT_UPDATE_STANDARD, &upd, &size)); - total_size += size; - upd->durable_ts = unpack.tw.durable_start_ts; - upd->start_ts = unpack.tw.start_ts; - upd->txnid = unpack.tw.start_txn; - F_SET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS); - - /* - * Instantiate both update and tombstone if the prepared update is a tombstone. This is - * required to ensure that written prepared delete operation must be removed from the data - * store, when the prepared transaction gets rollback. - */ - if (WT_TIME_WINDOW_HAS_STOP(&unpack.tw)) { - WT_ERR(__wt_upd_alloc_tombstone(session, &tombstone, &size)); - total_size += size; - tombstone->durable_ts = WT_TS_NONE; - tombstone->start_ts = unpack.tw.stop_ts; - tombstone->txnid = unpack.tw.stop_txn; - tombstone->prepare_state = WT_PREPARE_INPROGRESS; - F_SET(tombstone, WT_UPDATE_PREPARE_RESTORED_FROM_DS); - - /* - * Mark the update also as in-progress if the update and tombstone are from same - * transaction by comparing both the transaction and timestamps as the transaction - * information gets lost after restart. - */ - if (unpack.tw.start_ts == unpack.tw.stop_ts && - unpack.tw.durable_start_ts == unpack.tw.durable_stop_ts && - unpack.tw.start_txn == unpack.tw.stop_txn) { - upd->durable_ts = WT_TS_NONE; - upd->prepare_state = WT_PREPARE_INPROGRESS; - } - - tombstone->next = upd; - } else { - upd->durable_ts = WT_TS_NONE; - upd->prepare_state = WT_PREPARE_INPROGRESS; - tombstone = upd; - } - - page->modify->mod_row_update[WT_ROW_SLOT(page, rip - 1)] = tombstone; - tombstone = upd = NULL; + /* If we find a prepare, we'll have to instantiate it in the update chain later. */ + if (unpack.tw.prepare) + prepare = true; } WT_CELL_FOREACH_END; @@ -743,12 +853,9 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) if (best_prefix_count <= 10) F_SET_ATOMIC(page, WT_PAGE_BUILD_KEYS); - __wt_cache_page_inmem_incr(session, page, total_size); + if (preparedp != NULL && prepare) + *preparedp = true; err: - __wt_free(session, tombstone); - __wt_free(session, upd); - __wt_scr_free(session, &value); - return (ret); } diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index dc070d5c700..687c21ec086 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -94,7 +94,7 @@ __page_read(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) uint64_t time_diff, time_start, time_stop; uint32_t page_flags; uint8_t previous_state; - bool timer; + bool prepare, timer; time_start = time_stop = 0; @@ -159,11 +159,10 @@ __page_read(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) page_flags = WT_DATA_IN_ITEM(&tmp) ? WT_PAGE_DISK_ALLOC : WT_PAGE_DISK_MAPPED; if (LF_ISSET(WT_READ_IGNORE_CACHE_SIZE)) FLD_SET(page_flags, WT_PAGE_EVICT_NO_PROGRESS); - F_SET(session, WT_SESSION_INSTANTIATE_PREPARE); - ret = __wt_page_inmem(session, ref, tmp.data, page_flags, ¬used); - F_CLR(session, WT_SESSION_INSTANTIATE_PREPARE); - WT_ERR(ret); + WT_ERR(__wt_page_inmem(session, ref, tmp.data, page_flags, ¬used, &prepare)); tmp.mem = NULL; + if (prepare) + WT_ERR(__wt_page_inmem_prepare(session, ref)); skip_read: switch (previous_state) { diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index 7db2b39d486..8cb00cbde81 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -639,7 +639,7 @@ __slvg_trk_leaf(WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk, uint8_t *ad * Page flags are 0 because we aren't releasing the memory used to read the page into memory * and we don't want page discard to free it. */ - WT_ERR(__wt_page_inmem(session, NULL, dsk, 0, &page)); + WT_ERR(__wt_page_inmem(session, NULL, dsk, 0, &page, NULL)); WT_ERR(__wt_row_leaf_key_copy(session, page, &page->pg_row[0], &trk->row_start)); WT_ERR( __wt_row_leaf_key_copy(session, page, &page->pg_row[page->entries - 1], &trk->row_stop)); @@ -1691,7 +1691,7 @@ __slvg_row_trk_update_start(WT_SESSION_IMPL *session, WT_ITEM *stop, uint32_t sl */ WT_RET(__wt_scr_alloc(session, trk->trk_size, &dsk)); WT_ERR(__wt_bt_read(session, dsk, trk->trk_addr, trk->trk_addr_size)); - WT_ERR(__wt_page_inmem(session, NULL, dsk->data, 0, &page)); + WT_ERR(__wt_page_inmem(session, NULL, dsk->data, 0, &page, NULL)); /* * Walk the page, looking for a key sorting greater than the specified stop key -- that's our diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index c3af3cbb2e4..f7ca3b9f080 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -520,8 +520,6 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) * * Note: as the root page cannot currently be evicted, the root split generation isn't ever * used. That said, it future proofs eviction and isn't expensive enough to special-case. - * - * Getting a new split generation implies a full barrier, no additional barrier is needed. */ WT_FULL_BARRIER(); split_gen = __wt_gen(session, WT_GEN_SPLIT); @@ -772,8 +770,6 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, uint32_t * Get a generation for this split, mark the page. This must be after the new index is swapped * into place in order to know that no readers with the new generation will look at the old * index. - * - * Getting a new split generation implies a full barrier, no additional barrier is needed. */ WT_FULL_BARRIER(); split_gen = __wt_gen(session, WT_GEN_SPLIT); @@ -1044,8 +1040,6 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) * Get a generation for this split, mark the parent page. This must be after the new index is * swapped into place in order to know that no readers with the new generation will look at the * old index. - * - * Getting a new split generation implies a full barrier, no additional barrier is needed. */ WT_FULL_BARRIER(); split_gen = __wt_gen(session, WT_GEN_SPLIT); @@ -1366,6 +1360,7 @@ __split_multi_inmem(WT_SESSION_IMPL *session, WT_PAGE *orig, WT_MULTI *multi, WT WT_UPDATE *prev_onpage, *upd; uint64_t recno; uint32_t i, slot; + bool prepare; /* * In 04/2016, we removed column-store record numbers from the WT_PAGE structure, leading to @@ -1387,13 +1382,17 @@ __split_multi_inmem(WT_SESSION_IMPL *session, WT_PAGE *orig, WT_MULTI *multi, WT * our caller will not discard the disk image when discarding the original page, and our caller * will discard the allocated page on error, when discarding the allocated WT_REF. */ - F_SET(session, WT_SESSION_INSTANTIATE_PREPARE); - ret = __wt_page_inmem(session, ref, multi->disk_image, WT_PAGE_DISK_ALLOC, &page); - F_CLR(session, WT_SESSION_INSTANTIATE_PREPARE); - WT_RET(ret); + WT_RET(__wt_page_inmem(session, ref, multi->disk_image, WT_PAGE_DISK_ALLOC, &page, &prepare)); multi->disk_image = NULL; /* + * In-memory databases restore non-obsolete updates directly in this function, don't call the + * underlying page functions to do it. + */ + if (prepare && !F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) + WT_RET(__wt_page_inmem_prepare(session, ref)); + + /* * Put the re-instantiated page in the same LRU queue location as the original page, unless this * was a forced eviction, in which case we leave the new page with the read generation unset. * Eviction will set the read generation next time it visits this page. diff --git a/src/third_party/wiredtiger/src/btree/bt_sync.c b/src/third_party/wiredtiger/src/btree/bt_sync.c index d6a02ca1a30..9a157903053 100644 --- a/src/third_party/wiredtiger/src/btree/bt_sync.c +++ b/src/third_party/wiredtiger/src/btree/bt_sync.c @@ -385,6 +385,12 @@ __sync_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, void *context, bool *ski if (ref->state != WT_REF_DISK) return (0); + /* Don't read any pages when the cache is operating in aggressive mode. */ + if (__wt_cache_aggressive(session)) { + *skipp = true; + return (0); + } + /* Don't read pages into cache during startup or shutdown phase. */ if (F_ISSET(S2C(session), WT_CONN_RECOVERING | WT_CONN_CLOSING_TIMESTAMP)) { *skipp = true; diff --git a/src/third_party/wiredtiger/src/btree/col_modify.c b/src/third_party/wiredtiger/src/btree/col_modify.c index ec7b66b1012..632526f83df 100644 --- a/src/third_party/wiredtiger/src/btree/col_modify.c +++ b/src/third_party/wiredtiger/src/btree/col_modify.c @@ -31,7 +31,7 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U WT_PAGE *page; WT_PAGE_MODIFY *mod; WT_SESSION_IMPL *session; - WT_UPDATE *old_upd, *upd; + WT_UPDATE *last_upd, *old_upd, *upd; wt_timestamp_t prev_upd_ts; size_t ins_size, upd_size; u_int i, skipdepth; @@ -141,19 +141,33 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U #endif WT_ERR(__wt_txn_modify(session, upd)); logged = true; + + /* Avoid a data copy in WT_CURSOR.update. */ + __wt_upd_value_assign(cbt->modify_update, upd); } else { - upd = upd_arg; - upd_size = WT_UPDATE_MEMSIZE(upd); - } + upd_size = __wt_update_list_memsize(upd); - /* Avoid a data copy in WT_CURSOR.update. */ - __wt_upd_value_assign(cbt->modify_update, upd); + /* If there are existing updates, append them after the new updates. */ + for (last_upd = upd; last_upd->next != NULL; last_upd = last_upd->next) + ; + last_upd->next = old_upd; - /* - * If we restore an update chain in update restore eviction, there should be no update on - * the existing update chain. - */ - WT_ASSERT(session, !restore || old_upd == NULL); + /* + * If we restore an update chain in update restore eviction, there should be no update + * on the existing update chain. + */ + WT_ASSERT(session, !restore || old_upd == NULL); + + /* + * We can either put multiple new updates or a single update on the update chain. + * + * Set the "old" entry to the second update in the list so that the serialization + * function succeeds in swapping the first update into place. + */ + if (upd->next != NULL) + cbt->ins->upd = upd->next; + old_upd = cbt->ins->upd; + } /* * Point the new WT_UPDATE item to the next element in the list. If we get it right, the diff --git a/src/third_party/wiredtiger/src/btree/row_modify.c b/src/third_party/wiredtiger/src/btree/row_modify.c index 7c13e88e2be..c96b963cb1f 100644 --- a/src/third_party/wiredtiger/src/btree/row_modify.c +++ b/src/third_party/wiredtiger/src/btree/row_modify.c @@ -206,7 +206,7 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, WT_ERR(__wt_txn_modify(session, upd)); logged = true; - /* Avoid WT_CURSOR.update data copy. */ + /* Avoid a data copy in WT_CURSOR.update. */ __wt_upd_value_assign(cbt->modify_update, upd); } else { /* @@ -307,18 +307,21 @@ __wt_row_insert_alloc(WT_SESSION_IMPL *session, const WT_ITEM *key, u_int skipde /* * __wt_update_obsolete_check -- - * Check for obsolete updates. + * Check for obsolete updates and force evict the page if the update list is too long. */ WT_UPDATE * __wt_update_obsolete_check( - WT_SESSION_IMPL *session, WT_PAGE *page, WT_UPDATE *upd, bool update_accounting) + WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd, bool update_accounting) { + WT_PAGE *page; WT_TXN_GLOBAL *txn_global; WT_UPDATE *first, *next; size_t size; uint64_t oldest, stable; u_int count, upd_seen, upd_unstable; + next = NULL; + page = cbt->ref->page; txn_global = &S2C(session)->txn_global; upd_seen = upd_unstable = 0; @@ -372,10 +375,23 @@ __wt_update_obsolete_check( if (size != 0) __wt_cache_page_inmem_decr(session, page, size); } - return (next); } /* + * Force evict a page when there are more than WT_THOUSAND updates to a single item. Increasing + * the minSnapshotHistoryWindowInSeconds to 300 introduced a performance regression in which the + * average number of updates on a single item was approximately 1000 in write-heavy workloads. + * This is why we use WT_THOUSAND here. + */ + if (count > WT_THOUSAND) { + WT_STAT_CONN_INCR(session, cache_eviction_force_long_update_list); + __wt_page_evict_soon(session, cbt->ref); + } + + if (next != NULL) + return (next); + + /* * If the list is long, don't retry checks on this page until the transaction state has moved * forwards. This function is used to trim update lists independently of the page state, ensure * there is a modify structure. diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 75f2db226f2..693bf9aa554 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -227,8 +227,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_commit_transaction[] = { {"commit_timestamp", "string", NULL, NULL, NULL, 0}, {"durable_timestamp", "string", NULL, NULL, NULL, 0}, {"operation_timeout_ms", "int", NULL, "min=1", NULL, 0}, - {"sync", "string", NULL, "choices=[\"background\",\"off\",\"on\"]", NULL, 0}, - {NULL, NULL, NULL, NULL, NULL, 0}}; + {"sync", "string", NULL, "choices=[\"off\",\"on\"]", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_WT_SESSION_compact[] = { {"timeout", "int", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -329,7 +328,8 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_drop[] = { static const WT_CONFIG_CHECK confchk_WT_SESSION_flush_tier[] = { {"flush_timestamp", "string", NULL, NULL, NULL, 0}, {"force", "boolean", NULL, NULL, NULL, 0}, {"lock_wait", "boolean", NULL, NULL, NULL, 0}, - {"sync", "string", NULL, "choices=[\"off\",\"on\"]", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + {"sync", "string", NULL, "choices=[\"off\",\"on\"]", NULL, 0}, + {"timeout", "int", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_WT_SESSION_join[] = { {"bloom_bit_count", "int", NULL, "min=2,max=1000", NULL, 0}, @@ -342,8 +342,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_join[] = { {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_WT_SESSION_log_flush[] = { - {"sync", "string", NULL, "choices=[\"background\",\"off\",\"on\"]", NULL, 0}, - {NULL, NULL, NULL, NULL, NULL, 0}}; + {"sync", "string", NULL, "choices=[\"off\",\"on\"]", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_WT_SESSION_open_cursor_debug_subconfigs[] = { {"release_evict", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -404,9 +403,6 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_timestamp_transaction[] = { {"prepare_timestamp", "string", NULL, NULL, NULL, 0}, {"read_timestamp", "string", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; -static const WT_CONFIG_CHECK confchk_WT_SESSION_transaction_sync[] = { - {"timeout_ms", "int", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; - static const WT_CONFIG_CHECK confchk_WT_SESSION_verify[] = { {"dump_address", "boolean", NULL, NULL, NULL, 0}, {"dump_blocks", "boolean", NULL, NULL, NULL, 0}, {"dump_layout", "boolean", NULL, NULL, NULL, 0}, {"dump_offsets", "list", NULL, NULL, NULL, 0}, @@ -1211,8 +1207,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "checkpoint_wait=true,force=false,lock_wait=true," "remove_files=true", confchk_WT_SESSION_drop, 4}, - {"WT_SESSION.flush_tier", "flush_timestamp=,force=false,lock_wait=true,sync=on", - confchk_WT_SESSION_flush_tier, 4}, + {"WT_SESSION.flush_tier", "flush_timestamp=,force=false,lock_wait=true,sync=on,timeout=0", + confchk_WT_SESSION_flush_tier, 5}, {"WT_SESSION.join", "bloom_bit_count=16,bloom_false_positives=false," "bloom_hash_count=8,compare=\"eq\",count=,operation=\"and\"," @@ -1243,7 +1239,6 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "commit_timestamp=,durable_timestamp=,prepare_timestamp=," "read_timestamp=", confchk_WT_SESSION_timestamp_transaction, 4}, - {"WT_SESSION.transaction_sync", "timeout_ms=1200000", confchk_WT_SESSION_transaction_sync, 1}, {"WT_SESSION.truncate", "", NULL, 0}, {"WT_SESSION.upgrade", "", NULL, 0}, {"WT_SESSION.verify", "dump_address=false,dump_blocks=false,dump_layout=false," diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index 8012c5d84fc..e4b3c5e981d 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -555,9 +555,8 @@ __log_file_server(void *arg) WT_DECL_RET; WT_FH *close_fh; WT_LOG *log; - WT_LSN close_end_lsn, min_lsn; + WT_LSN close_end_lsn; WT_SESSION_IMPL *session; - uint64_t yield_count; uint32_t filenum; bool locked; @@ -565,7 +564,6 @@ __log_file_server(void *arg) conn = S2C(session); log = conn->log; locked = false; - yield_count = 0; while (FLD_ISSET(conn->server_flags, WT_CONN_SERVER_LOG)) { /* * If there is a log file to close, make sure any outstanding write operations have @@ -617,54 +615,6 @@ __log_file_server(void *arg) __wt_spin_unlock(session, &log->log_sync_lock); } } - /* - * If a later thread asked for a background sync, do it now. - */ - if (__wt_log_cmp(&log->bg_sync_lsn, &log->sync_lsn) > 0) { - /* - * Save the latest write LSN which is the minimum we will have written to disk. - */ - WT_ASSIGN_LSN(&min_lsn, &log->write_lsn); - /* - * We have to wait until the LSN we asked for is written. If it isn't signal the wrlsn - * thread to get it written. - * - * We also have to wait for the written LSN and the sync LSN to be in the same file so - * that we know we have synchronized all earlier log files. - */ - if (__wt_log_cmp(&log->bg_sync_lsn, &min_lsn) <= 0) { - /* - * If the sync file is behind either the one wanted for a background sync or the - * write LSN has moved to another file continue to let this worker thread process - * that older file immediately. - */ - if ((log->sync_lsn.l.file < log->bg_sync_lsn.l.file) || - (log->sync_lsn.l.file < min_lsn.l.file)) - continue; - WT_ERR(__wt_fsync(session, log->log_fh, true)); - __wt_spin_lock(session, &log->log_sync_lock); - WT_NOT_READ(locked, true); - /* - * The sync LSN could have advanced while we were writing to disk. - */ - if (__wt_log_cmp(&log->sync_lsn, &min_lsn) <= 0) { - WT_ASSERT(session, min_lsn.l.file == log->sync_lsn.l.file); - WT_ASSIGN_LSN(&log->sync_lsn, &min_lsn); - __wt_cond_signal(session, log->log_sync_cond); - } - locked = false; - __wt_spin_unlock(session, &log->log_sync_lock); - } else { - __wt_cond_signal(session, conn->log_wrlsn_cond); - /* - * We do not want to wait potentially a second to process this. Yield to give the - * wrlsn thread a chance to run and try again in this case. - */ - yield_count++; - __wt_yield(); - continue; - } - } /* Wait until the next event. */ __wt_cond_wait(session, conn->log_file_cond, 100000, NULL); @@ -674,7 +624,6 @@ __log_file_server(void *arg) err: WT_IGNORE_RET(__wt_panic(session, ret, "log close server error")); } - WT_STAT_CONN_INCRV(session, log_server_sync_blocked, yield_count); if (locked) __wt_spin_unlock(session, &log->log_sync_lock); return (WT_THREAD_RET_VALUE); diff --git a/src/third_party/wiredtiger/src/conn/conn_tiered.c b/src/third_party/wiredtiger/src/conn/conn_tiered.c index 39524e0758b..58b2026c5b8 100644 --- a/src/third_party/wiredtiger/src/conn/conn_tiered.c +++ b/src/third_party/wiredtiger/src/conn/conn_tiered.c @@ -23,20 +23,27 @@ * __flush_tier_wait -- * Wait for all previous work units queued to be processed. */ -static void -__flush_tier_wait(WT_SESSION_IMPL *session) +static int +__flush_tier_wait(WT_SESSION_IMPL *session, const char **cfg) { + WT_CONFIG_ITEM cval; WT_CONNECTION_IMPL *conn; + uint64_t now, start, timeout; int yield_count; conn = S2C(session); yield_count = 0; + now = start = 0; /* * The internal thread needs the schema lock to perform its operations and flush tier also * acquires the schema lock. We cannot be waiting in this function while holding that lock or no * work will get done. */ WT_ASSERT(session, !FLD_ISSET(session->lock_flags, WT_SESSION_LOCKED_SCHEMA)); + WT_RET(__wt_config_gets(session, cfg, "timeout", &cval)); + timeout = (uint64_t)cval.val; + if (timeout != 0) + __wt_seconds(session, &start); /* * It may be worthwhile looking at the add and decrement values and make choices of whether to @@ -44,11 +51,17 @@ __flush_tier_wait(WT_SESSION_IMPL *session) * take a long time so yielding may not be effective. */ while (!WT_FLUSH_STATE_DONE(conn->flush_state)) { + if (start != 0) { + __wt_seconds(session, &now); + if (now - start > timeout) + return (EBUSY); + } if (++yield_count < WT_THOUSAND) __wt_yield(); else __wt_cond_wait(session, conn->flush_cond, 200, NULL); } + return (0); } /* @@ -337,7 +350,7 @@ __wt_flush_tier(WT_SESSION_IMPL *session, const char *config) WT_DECL_RET; uint32_t flags; const char *cfg[3]; - bool wait; + bool locked, wait; conn = S2C(session); WT_STAT_CONN_INCR(session, flush_tier); @@ -372,6 +385,7 @@ __wt_flush_tier(WT_SESSION_IMPL *session, const char *config) __wt_spin_lock(session, &conn->flush_tier_lock); else WT_RET(__wt_spin_trylock(session, &conn->flush_tier_lock)); + locked = true; /* * We cannot perform another flush tier until any earlier ones are done. Often threads will wait @@ -379,7 +393,7 @@ __wt_flush_tier(WT_SESSION_IMPL *session, const char *config) * turned off then any following call must wait and will do so here. We have to wait while not * holding the schema lock. */ - __flush_tier_wait(session); + WT_ERR(__flush_tier_wait(session, cfg)); if (wait) WT_WITH_CHECKPOINT_LOCK( session, WT_WITH_SCHEMA_LOCK(session, ret = __flush_tier_once(session, flags))); @@ -387,9 +401,14 @@ __wt_flush_tier(WT_SESSION_IMPL *session, const char *config) WT_WITH_CHECKPOINT_LOCK_NOWAIT(session, ret, WT_WITH_SCHEMA_LOCK_NOWAIT(session, ret, ret = __flush_tier_once(session, flags))); __wt_spin_unlock(session, &conn->flush_tier_lock); + locked = false; if (ret == 0 && LF_ISSET(WT_FLUSH_TIER_ON)) - __flush_tier_wait(session); + WT_ERR(__flush_tier_wait(session, cfg)); + +err: + if (locked) + __wt_spin_unlock(session, &conn->flush_tier_lock); return (ret); } @@ -520,6 +539,7 @@ __tiered_mgr_server(void *arg) WT_ITEM path, tmp; WT_SESSION_IMPL *session; WT_TIERED_MANAGER *mgr; + const char *cfg[2]; session = arg; conn = S2C(session); @@ -527,6 +547,8 @@ __tiered_mgr_server(void *arg) WT_CLEAR(path); WT_CLEAR(tmp); + cfg[0] = "timeout=0"; + cfg[1] = NULL; for (;;) { /* Wait until the next event. */ @@ -542,7 +564,7 @@ __tiered_mgr_server(void *arg) WT_WITH_SCHEMA_LOCK(session, ret = __flush_tier_once(session, 0)); WT_ERR(ret); if (ret == 0) - __flush_tier_wait(session); + WT_ERR(__flush_tier_wait(session, cfg)); WT_ERR(__tier_storage_remove(session, false)); } diff --git a/src/third_party/wiredtiger/src/docs/Doxyfile b/src/third_party/wiredtiger/src/docs/Doxyfile index 8bd05ca0bfa..cfef83d0445 100644 --- a/src/third_party/wiredtiger/src/docs/Doxyfile +++ b/src/third_party/wiredtiger/src/docs/Doxyfile @@ -297,6 +297,7 @@ ALIASES = "arch_page_table{2}=<div class= arch_head><table><tr><t "row{7}=<tr><td>\1</td><td>\2</td><td>\3</td><td>\4</td><td>\5</td><td>\6</td><td>\7</td></tr>" \ "row{8}=<tr><td>\1</td><td>\2</td><td>\3</td><td>\4</td><td>\5</td><td>\6</td><td>\7</td><td>\8</td></tr>" \ "row{9}=<tr><td>\1</td><td>\2</td><td>\3</td><td>\4</td><td>\5</td><td>\6</td><td>\7</td><td>\8</td><td>\9</td></tr>" \ + "sic=" \ "subpage_single=@subpage" # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources diff --git a/src/third_party/wiredtiger/src/docs/arch-backup.dox b/src/third_party/wiredtiger/src/docs/arch-backup.dox new file mode 100644 index 00000000000..7969e11190b --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-backup.dox @@ -0,0 +1,69 @@ +/*! @arch_page arch-backup Backup + +# Overview # + +Backup in WiredTiger can be performed using the \c "backup:" cursor. WiredTiger +supports four types of backups. + +1. Full database +2. Block-based incremental backup +3. Log-based incremental backup +4. Target backup + +The following page describes how backup works inside the WiredTiger. Please refer to +@ref backup for details on how to use each of these types of backups. + +# Full database # + +When the application opens the backup cursor, internally WiredTiger generates a list of all +files in the database that are necessary for the backup. WiredTiger takes both the checkpoint +and schema locks to block database file modifications while generating the list files. There +is a contract that all files that exist at the time the backup starts exist for the entire time +the backup cursor is open. This contract applies to all files in the database, not just files +that are included in the generated list. + +WiredTiger log files are also part of the generated file list for backup. There is a contract +that the log files do not contain or make visible operations that are added to the log after +the backup started that could have adverse affects during recovery and restore. Therefore +WiredTiger forces a log file switch when opening the backup cursor. To fulfill the earlier +contract that all files must exist during backup, WiredTiger does not use any pre-allocated +log files nor does it archive or remove old log files during the time the backup cursor is open. + +Once the backup is opened successfully, any checkpoints that exist on the database before +backup starts must be retained until the backup cursor is closed. All the newer checkpoints +that are created during the backup in progress are cleaned whenever a newer checkpoint is +created. + +# Block-based incremental backup # + +Block-based incremental backup is performed by tracking the modified blocks in the checkpoint. +Whenever a new checkpoint occurs on a file, any new or modified blocks in this checkpoint are +recorded as a bit string. This bit string information is updated with every new checkpoint. + +Whenever the incremental backup cursor is opened to perform the backup, WiredTiger returns all +the file offsets and sizes that are modified from the previous source backup identifier. +WiredTiger returns the full file information to be copied for all the new files that are created, +renamed or imported into the database after the incremental backup cursor is opened. Refer to +@ref backup_incremental-block for more information on how to use the block-based incremental backup. + +# Log-based incremental backup # + +When the backup cursor is opened with the \c target configuration string as \c "target=(\"log:\\")" +the log-based incremental backup is performed by adding all the existing log files in the database +to the list of files that needs to be copied. Applications wanting to use log files for incremental +backup must first disable automatic log file removal using the \c log=(archive=false) configuration +to ::wiredtiger_open. By default, WiredTiger automatically removes log files no longer required +for recovery. Refer to @ref backup_incremental for more information on how to use the log-based +incremental backup. + +# Target backup # + +When the backup cursor is opened with the \c target configuration string as \c "target=", +WiredTiger internally generates a list of targeted files instead of all files like the full backup. +If the targeted object is a table, all the indexes and column groups of the table are also backed up. + +Note: There can only be one backup cursor open at a time unless using duplicate backup +cursors. The duplicate backup cursors are used for catching up with the log files or with +block-based incremental backup. Please refer to @ref backup_duplicate for more details. + +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-column.dox b/src/third_party/wiredtiger/src/docs/arch-column.dox deleted file mode 100644 index 4b3b399349a..00000000000 --- a/src/third_party/wiredtiger/src/docs/arch-column.dox +++ /dev/null @@ -1,23 +0,0 @@ -/*! @arch_page arch-column Column Store - -Column Stores are Btrees stored in WiredTiger that have as their -key a record id, that is, a 64 bit unsigned integer. Thus, they implement -a specialized version of a Btree, where the key is a predictable length. - -A particular kind of column store is the fixed length column store. -As its name implies, the value is fixed length, and furthermore the -value is restricted to 1 and 8 bits in length. The bit length is specified -when the column store is created. The fixed length column store -has specialized use cases like bitmaps. - -The more general case is the variable length column store which allows -for values that have any length, and may have arbitrary types, including -aggregates of various types. - -Internally, row stores and both kinds of column stores all use a common -\c WT_BTREE structure. Column stores are distinguished in that -<code>WT_BTREE->type == BTREE_COL_VAR</code> for variable length column stores -and <code>WT_BTREE->type == BTREE_COL_FIX</code> for fixed length column stores. -Internal functions that navigate, access and manipulate Btrees have -code sprinkled throughout that is conditional on <code>WT_BTREE->type</code>. -*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-data-file.dox b/src/third_party/wiredtiger/src/docs/arch-data-file.dox index 834329c981f..580b355d838 100644 --- a/src/third_party/wiredtiger/src/docs/arch-data-file.dox +++ b/src/third_party/wiredtiger/src/docs/arch-data-file.dox @@ -1,17 +1,108 @@ /*! @arch_page arch-data-file Data File Format -The format of the WiredTiger data file is given by structures -in \c block.h , defining the overall structure of the file and -its blocks. The \c WT_BLOCK_DESC starts the file. Following that, -individual pages appear, each with a \c WT_PAGE_HEADER defined in \c btmem.h -and a \c WT_BLOCK_HEADER defined in \c block.h . Individual cells then -sequentially appear as defined in \c cell.h . Each cell encodes -a key or value. Ordering is important, values are all associated with -the key that precedes them. Multiple values may be present, which -can represent multiple versions. Extremely large values may be represented -as a reference to another page. - -The exact encoding is rather complex, and beyond what can be described here. -The encoding strikes a balance between data that can be compacted efficiently -in time and space, extensibility, and compatibility with previous versions. +@section data_file_database_files Database Files + +WiredTiger stores databases in a durable format through writing data to files +it has created in the database directory. These include data files representing +tables in a MongoDB database, a history store file used to track previous +versions of the database tables, as well as other metadata files for the +entire system. + +Data files are denoted by the .wt suffix. The history store file +(WiredTigerHS.wt) and the metadata file (WiredTiger.wt) are special WiredTiger +files, but have the same underlying structure as normal data files. + +@section data_file_format Data File Format and Layout + +In memory, a database table is represented by a B-Tree data structure, which is +made up of nodes that are page structures. The root and internal pages will only +store keys and references to other pages, while leaf pages store keys, values and +sometimes references to overflow pages. When these pages are written onto the +disk, they are written out as units of data called blocks. On the disk, a +WiredTiger data file is just a collection of blocks which logically represent +the pages of the B-Tree. + +The layout of a .wt file consists of a file description \c WT_BLOCK_DESC which +always occupies the first block, followed by a set of on-disk pages. +The file description contains metadata about the file such as the WiredTiger +major and minor version, a magic number, and a checksum of the block +contents. This information is used to verify that the file is a legitimate +WiredTiger data file with a compatible WiredTiger version, and that its +contents are not corrupted. + +@section data_file_on_disk_page_format On-Disk Page Format and Layout + +Pages consist of a header (\c WT_PAGE_HEADER and \c WT_BLOCK_HEADER) followed +by a variable number of cells, which encode keys, values or addresses (see +\c cell.h). The page header \c WT_PAGE_HEADER consists of information such +as the column-store record number, write generation (required for ordering +pages in time), in-memory size, cell count (the number of cells on the page), +data length (the overflow record length), and the page type. This is +immediately followed by the block header \c WT_BLOCK_HEADER which contains +block-manager specific information such as flags and version. + +After the page header, pages with variable-length keys or values will have +a set of cells. There are four main types of cells - keys, data, deleted +cells, and off-page references. Cells are also usually followed by some +additional data depending on its type. For example, values may have a time +validity window, and off-page references may have a validity window and an +address cookie. Ordering is important, values are all associated with the +key that precedes them. Multiple values may be present, which can represent +multiple versions. Extremely large values may be represented as a reference +to another page (see overflow pages). + +The exact encoding of cells is rather complex, and beyond what can be +described here. The encoding strikes a balance between data that can be +compacted efficiently in time and space, extensibility, and compatibility +with previous versions. + +@section data_file_page_types Page Types + +Different types of pages are made up of different types of cells. + +(1) Internal Pages - as the B-Tree grows in size and layer/s are added, +there will be pages between the root page and leaf page. These internal +pages contain keys that point to other pages. +- The row-store internal page \c WT_PAGE_ROW_INT is made up of a key cell +(\c WT_CELL_KEY or \c WT_CELL_KEY_OVFL) followed by an off-page reference +(\c WT_CELL_ADDR_XXX). +- The column-store internal page \c WT_PAGE_COL_INT simply contains an +off-page reference. + +(2) Leaf Pages - leaf pages consist of a page header, followed by keys, +values or addresses (these reference overflow pages). +- The row-store leaf page \c WT_PAGE_ROW_LEAF is made up of a key cell +(\c WT_CELL_KEY or \c WT_CELL_KEY_OVFL) followed by a value +(\c WT_CELL_VALUE/VALUE_COPY/VALUE_OVFL). +- The column-store leaf page (for variable-length data) \c WT_PAGE_COL_VAR +is made up of data cells (\c WT_CELL_VALUE/VALUE_COPY/VALUE_OVFL) or +deleted cells (\c WT_CELL_DEL). The page header has a starting record +number, so the associated keys for all column store values is deduced by +their position in the page. + +(3) Overflow Pages - overflow pages are needed when keys/values that are +too large must be stored separately in the file, apart from where the item +is logically placed. Page sizes and configuration values such as +internal_key_max, leaf_key_max and leaf_value_max are used to determine +overflow items. + +While the above gives a general overview of how different page types are +structured, it is a simplified representation. Due to the large amount of +data being stored, WiredTiger may compress data to preserve memory and disk +space. Keys may not be stored in their entirety when prefix compression +is used (the identical key prefix is stored only once per page to reduce +size). In addition, each block written may be compressed and/or encrypted. +The header portion of each block remains uncompressed and unencrypted. + +@section data_file_functions Data File Functions + +When data needs to be read from a data file, it is unpacked into in-memory +structures, and vice versa, we pack the contents when we want to write a +page image to a data file. Low level functions that pack or unpack the +contents of individual cells are in \c cell_inline.h. Functions +like \c __wt_cell_unpack_kv and \c __wt_cell_unpack_addr to unpack keys, +values and addresses are found here, and uses of such functions appear in +the B-Tree code. Similarly, there are \c __wt_cell_pack_* functions to +pack particular kinds of keys, values and addresses; these are used in +the reconciliation code. */ diff --git a/src/third_party/wiredtiger/src/docs/arch-glossary.dox b/src/third_party/wiredtiger/src/docs/arch-glossary.dox index ea71fe5893b..f12468aaaaf 100644 --- a/src/third_party/wiredtiger/src/docs/arch-glossary.dox +++ b/src/third_party/wiredtiger/src/docs/arch-glossary.dox @@ -4,31 +4,98 @@ WiredTiger has a lot of domain specific nomenclature - this page attempts to decode it. This is intended for those navigating the WiredTiger source tree - it describes terms internal to the storage engine. -@section arch-glossary-general General Terms Used in WiredTiger +The definitions list how the terms are used in WiredTiger, which may not match how +they are used in other settings. -<table> -<caption id="general_terms">General Term Table</caption> -<tr><th>Term <th>Definition -<tr><td>Hello<td>The typical next word is World -</table> - -@section arch-glossary-checkpoint Terms Related to Checkpoints +@section arch-glossary-transactions Glossary (click headings to sort) -<table> -<caption id="checkpoint_terms">Checkpoint Term Table</caption> -<tr><th>Term <th>Definition -<tr><td>Hello<td>The typical next word is World -</table> -@section arch-glossary-transactions Terms Related to Transactions +<!-- Please keep glossary formatted a single entry to a line, and sorted alphabetically (case insensitive). --> -<table> -<caption id="transaction_terms">Transaction Term Table</caption> -<tr><th>Term <th>Definition -<tr><td>transaction<td>A unit of work performed in WiredTiger that is atomic and consistent with the specified isolation levels, and durability levels. -<tr><td>session<td>The container that manages transactions and performs the transactional operations on behalf of the users in a single thread. -<tr><td>atomicity<td>Atomicity is a guarantee provided within the context of a transaction that all operations made within that transaction either succeed or fail. -<tr><td>isolation<td>Isolation determines which versions of data are visible to a running transaction, and whether it can see changes made by concurrently running transactions. -<tr><td>durability<td>Durability is the property that guarantees that transactions that have been committed will survive permanently. +<table class="doxtable sortable"> +<tr><th>Term<th>Category<th>Definition +<tr><td>address cookie<td>block manager<td>an opaque set of bytes returned by the block manager to reference a block in a Btree file, it includes an offset, size, checksum, and object id. +<tr><td>atomicity<td>transactions<td>a guarantee provided within the context of a transaction that all operations made within that transaction either succeed or fail. +<tr><td>barrier<td>general<td>a call that forces the CPU and compiler to enforce ordering constraints on memory operations, used in multithreaded code. +<tr><td>block<td>block manager<td>the smallest granularity of data that is read from or written to a Btree file. A btree page, when it appears on disk, may consist of one or more blocks. +<tr><td>block compression<td>btree<td>after pages of btrees are encoded to be written, they may be compressed according to a configured algorithm. Compressors may be added to WiredTiger via the extension mechanism. +<tr><td>block manager<td>block manager<td>an internal API that abstracts writing, reading and allocating blocks to and from files. A block manager is associated with a Btree. +<tr><td>Bloom filter<td>lsm<td>a data structure that identifies that a key cannot be present. Used by LSM. +<tr><td>btree<td>btree<td>a data structure on disk or in memory that stores keys, it with its associated value. In WiredTiger, all btrees are in fact B+trees, meaning that adjacent keys and values are grouped into pages. +<tr><td>bulk insert<td>btree<td>the capability to rapidly insert a set of ordered key value pairs into a Btree. In WiredTiger, a Btree's dhandle must be obtained exclusively, guaranteeing that the inserts can be done single threaded. +<tr><td>cell<td>file format<td>a key or value packed as it appears on disk for a Btree. +<tr><td>checkpoint<td>checkpoint<td>a stable point that bounds how much work recovery must do. Data committed before a checkpoint is guaranteed to reside in data files. Thus, recovery must replay log files starting at the last completed checkpoint. +<tr><td>chunk<td>lsm<td>a single file within an LSM tree. +<tr><td>collator<td>general<td>a comparison function that determines how keys will be ordered in a Btree. The default collator is byte oriented. A custom collator may be packaged as an extension and attached to a Btree. +<tr><td>column<td>schema<td>a single field in a record, which must be unpacked from a raw key or value. +<tr><td>column group<td>schema<td>a set of column store Btrees, whose record ids logically correspond. Together, a column group represents a table of data, with its columns being the union of columns in the individual Btrees. +<tr><td>column store<td>btree<td>a Btrees that has as its key a record id +<tr><td>commit timestamp<td>transactions<td>values in this transaction become visible at this timestamp. +<tr><td>compare and swap (CAS)<td>general<td>a CPU instruction that can be used to coordinate the activity of multiple threads. Specifically, the instruction atomically changes a memory location to a new value only if the existing value matches a second value. +<tr><td>compression<td>general<td>one of several techniques to reduce the size of data on disk and memory. See block compression, run length encoding, Huffman encoding, key prefix compression. +<tr><td>connection<td>general<td>an encapsulation of information stored for an application's use of a WiredTiger instance attached to a specific home directory. This encapsulation is kept in a handle that is used in the API. +<tr><td>cursor<td>general<td>a handle in the WiredTiger API that encapsulates a position in a Btree, and may hold the key and value associated with that position. +<tr><td>data handle<td>general<td>an abstraction of a named file, table or other data source in WiredTiger +<tr><td>data source<td>general<td>an abstraction that extends the uri name space. A custom data source defines APIs that are invoked when objects are created in, or cursors are opened on, the new name space. +<tr><td>dhandle<td>general<td>see data handle +<tr><td>diagnostic build<td>general<td>a build of WiredTiger that includes extra code to check invariants, and abort if they fail. +<tr><td>durability<td>transactions<td>the property that guarantees that transactions that have been committed will survive permanently. +<tr><td>encryption<td>btree<td>in WiredTiger, encryption is applied to data files, log files and metadata. Encryption algorithms may be added to WiredTiger via the extension mechanism. +<tr><td>encryptor<td>btree<td>an encryption algorithm packaged as an extension. +<tr><td>eviction<td>general<td>the process of freeing memory in the cache, which often includes reconciling and writing dirty pages. This term generally includes the process of deciding which pages should be evicted. +<tr><td>extension<td>general<td>a module that uses the WiredTiger extension interface, allowing it to be added in an encapsulated way, without altering the core of the WiredTiger library. Extensions are typically compiled into shared libraries. +<tr><td>extent list<td>block manager<td>a list of contiguous sets of blocks in a file. Extent lists are used to track available blocks and/or used blocks. +<tr><td>fast truncate<td>btree<td>a truncate that spans a key range over more than one leaf page may be performed quickly by deleting entire data pages at a time. +<tr><td>frag<td>verify<td>in verify, a part of a file that is the minimum allocation size and aligned the same. The smallest blocks may be the same as a frag, other blocks may be composed of multiple contiguous frags. +<tr><td>hazard pointer<td>general<td>a data structure that assists in lock free algorithms, used by WiredTiger Btrees +<tr><td>history store<td>general<td>storage in WiredTiger (currently implemented as a Btree) that keeps previous values associated with a key. +<tr><td>home directory<td>general<td>the directory containing all the data for a WiredTiger connection. The connection is attached to this directory via a call to ::wiredtiger_open. +<tr><td>Huffman encoding<td>btree<td>data in a btree may optionally be encoded using a Huffman encoding, based on either English letter frequencies or a custom encoding. See @subpage_single huffman for details. +<tr><td>index<td>schema<td>a Btree with records having values that are keys to another Btree. An index (or set of indices) become associated with a primary Btree via the table construct. +<tr><td>insert<td>btree<td>an in-memory structure holding a key and a set of updates for a Btree. +<tr><td>internal page<td>btree<td>either in-memory or on disk, a page in a Btree that has a set of keys that reference pages beneath it. +<tr><td>isolation<td>transactions<td>determines which versions of data are visible to a running transaction, and whether it can see changes made by concurrently running transactions. +<tr><td>key prefix compression<td>btree<td>a technique to store identical key prefixes only once per page. See @ref file_formats_compression +<tr><td>kv or k/v<td>btree<td>a key/value pair in a Btree. +<tr><td>leaf page<td>btree<td>either in-memory or on disk, a page in a Btree that has a pairs of keys and values, and no pages beneath it. +<tr><td>log structured merge tree (or LSM tree)<td>lsm<td>a data structure that stores keys and values (as a Btree), but composed of potentially many trees (Btrees in WiredTiger). +<tr><td>LSM<td>lsm<td>see log structured merge tree +<tr><td>merge<td>btree<td>the process of making a single page out of two adjacent smaller pages in a Btree. This can occur with leaf pages or internal pages. +<tr><td>metadata<td>general<td>a set of data that is used to help manage files, tables, indices and column groups in WiredTiger. +<tr><td>metadata tracking<td>general<td>a technique to track a set of changes to the metadata, so they can be "rolled back" when an error occurs. +<tr><td>mutex<td>general<td>a locking algorithm that waits, allowing other threads to execute, when the lock cannot be immediately acquired, and is woken to retry when the lock is available. Although more heavyweight than a spin lock, it consumes fewer resources while waiting. Appropriate for resources that may be in contention and may be held for longer periods. +<tr><td>object id<td>tiered storage<td>an integer that indicates an individual file that is a part of a Btree. +<tr><td>oldest timestamp<td>transactions<td>a connection-wide value, specifies the oldest timestamp that must be tracked by the system. Values with stop timestamps older than this value do not have to be retained. +<tr><td>on disk<td>general<td>a shorthand meaning "in the file system", and may or may not be stored on a physical disk. +<tr><td>overflow page<td>btree<td>either in-memory or on disk, a page in a Btree that has a key or value that is too large to appear on a leaf or internal page. +<tr><td>pack<td>general<td>convert in-memory representations of keys or values to a compact format for file storage +<tr><td>page<td>btree<td>one logical node of a Btree, in memory, or on disk, that may store keys and/or values. See internal page, leaf page, overflow page. +<tr><td>page reference<td>btree<td>an indirect reference to a key on a page in a Btree. The reference struct (WT_REF) acts as the indirection and may include a pointer to an in memory object or an address cookie to find the object on disk. "ref" for short. +<tr><td>page split<td>btree<td>the process of breaking up a large page in a Btree into multiple smaller pages. This may occur with either leaf pages or internal pages. +<tr><td>pthread<td>general<td>a thread implementation used on POSIX systems +<tr><td>raw<td>schema<td>data as it appears on-disk, typically in a packed format. +<tr><td>read timestamp<td>transactions<td>a timestamp that specifies what data should be visible. +<tr><td>recno<td>general<td>a 64 bit integer. When used as a key, the Btree is said to be a column store, and the keys are assigned incrementally by WiredTiger when new records are inserted. +<tr><td>reconcile<td>general<td>convert the in-memory version of a page to a more compact form for file storage. +<tr><td>recovery<td>general<td>a procedure used on opening WiredTiger whereby, starting at the last checkpoint written, log records are played back to bring the database up to date. +<tr><td>ref<td>btree<td>see page reference +<tr><td>RLE<td>general<td>see run length encoding +<tr><td>run length encoding<td>general<td>also known as "RLE". A technique to save space whereby repeated items can be represented by a count followed by the item. For example the string of letters \sic "AAAAAAABBBCDDDD" might be encoded as \sic "7A3BC4D" using "out of band" digits. +<tr><td>salvage<td>general<td>functionality to repair Btree disk files. +<tr><td>schema<td>schema<td>the algorithms that use metadata to map the raw keys and values in Btrees to higher level abstractions like tables with typed columns. +<tr><td>session<td>general<td>an encapsulation of information stored for single thread of control in the application. This encapsulation is kept in a handle that is used in the API. Sessions manage transactions and performs the transactional operations. +<tr><td>skip list<td>general<td>a variation of a sorted linked list that allows for faster (O(log(N))) searching. It is used in WiredTiger as a lock-free data structure for keys inserted on a page. +<tr><td>spin lock<td>general<td>a simple locking algorithm that continually attempts to acquire a lock, appropriate for resources that are likely to be available and may be held for a short time. This lock is CPU intensive while it is waiting. Contrast with mutex +<tr><td>split<td>btree<td>see page split +<tr><td>stable timestamp<td>transactions<td>a connection-wide value, checkpoints will not include commits newer than this value. +<tr><td>table<td>schema<td>an abstraction that allows one or more btrees to be used in a coordinated way, which is specified by the schema. The table coordinates indices and column groups. +<tr><td>thread<td>general<td>a thread of control. WiredTiger uses pthreads for POSIX systems. +<tr><td>thread group<td>general<td>a group of threads that perform a common task. Eviction is performed by a thread group. +<tr><td>timestamp<td>transactions<td>refers to one of several types of 64 bit values used to control transactional visibility of data in WiredTiger. See oldest timestamp, stable timestamp, commit timestamp, read timestamp. +<tr><td>transaction<td>transactions<td>a unit of work performed in WiredTiger that is atomic and consistent with the specified isolation level and durability level. +<tr><td>truncate<td>btree<td>an operation to remove a range of key/values from a Btree. See also fast truncate +<tr><td>update<td>btree<td>a value associated with a key in a Btree. A key may have many updates, these correspond to values written to the key previously. +<tr><td>update chain<td>btree<td>a list of updates. +<tr><td>verify<td>general<td>functionality to check integrity of Btree disk files. </table> */ diff --git a/src/third_party/wiredtiger/src/docs/arch-hs.dox b/src/third_party/wiredtiger/src/docs/arch-hs.dox index 6ed50c5fbf6..e1d0a88c677 100644 --- a/src/third_party/wiredtiger/src/docs/arch-hs.dox +++ b/src/third_party/wiredtiger/src/docs/arch-hs.dox @@ -1,8 +1,272 @@ /*! @arch_page arch-hs History Store -The History Store in WiredTiger tracks old (all but the latest committed) -versions of records. By having these records in storage separate from -the current version, they can be used to service long running transactions, +The history store in WiredTiger tracks historical versions of records required to +service older readers. By having these records in storage separate from +the current version, they can be used to service long running transactions and be evicted as necessary, without interfering with activity that uses -the most recent committed versions. +the most recent committed versions. With the introduction of history store, only the +newest update to a key is written to the user table while the older updates +for the key are written to the history store. All user tables in the database are +backed by a single history store table. The history store has no outward +visibility to the application code and only WiredTiger's internal modules can +perform operations on the history store table using a predefined cursor API. + +@section arch_hs_table History store table structure +WiredTiger writes the history store on the disk as a standard row-store table. The key for +the history store table is a combination of: +- btree ID of user table this update belongs to +- record key (byte-string for row-store, record number for column-store) +- start timestamp for the update +- counter value + +This key format allows us to search efficiently for a given record key and read +timestamp combination. The last part of the key is a monotonically increasing counter +to keep the key unique in scenarios where a key has multiple updates with the same commit timestamp. +As the key for the history store table is different for row- and column-store, +we store both key types in a byte string otherwise we'd need two history store files with +different key formats. + +The corresponding value for each key is stored as a combination of: +- stop timestamp +- durable timestamp +- update type +- value + +@subsection arch_hs_table_tombstone Tombstones in the history store table +The stop timestamp refers to the point in time after which the update is no longer visible. This can +either be a result of deletion or of a new update being committed. An update with a valid stop time +is also called a \c tombstone. Every update in the history store has a valid stop timestamp and +transaction id associated with it, except for cases where the update preceded a prepared update. +The stop timestamp of the update immediately before the prepared update is set to the maximum +possible timestamp (\c WT_TS_MAX). The stop timestamp of the latest update in the history store is +updated once the prepared update is resolved. A tombstone becomes globally visible if the stop +timestamp is less than the oldest timestamp and the stop transaction id is visible to all concurrent +transactions in the system. A checkpoint can delete history store pages that only contain globally +visible tombstones. The garbage collection process is discussed in @ref arch-checkpoint. + +@section arch_hs_initialize History store initialization +WiredTiger checks for an existing history store table at startup and creates a new table if it is +not able to find one. The history store table is usually created when WiredTiger creates a +new database, with one exception where WiredTiger is opening an existing database created by an +older version of WiredTiger that doesn't support the history store. WiredTiger uses +\c WiredTigerHS.wt as the filename for the history store table. + +@section arch_hs_cursor History store cursor interface +WiredTiger uses a cursor interface for history store table to make it easier for different modules +to perform data operations on the history store table. A new cursor can be opened by calling +\c __wt_curhs_open(). History store cursor implementation supports most of the standard \c WT_CURSOR +methods except for \c WT_CURSOR::search() method. Recall that the key contains a +counter and timestamp value and it is expected that the caller can not possibly know the exact +values for both fields beforehand. Therefore, \c WT_CURSOR::search_near() is used for all search +operations. Two helper functions, \c __wt_curhs_search_near_before() and +\c __wt_curhs_search_near_after(), +have been provided to facilitate searching for the required record in the history store table. + +@section arch_hs_cursor_visibility History store cursor interface and visibility rules +When using the history store cursor interface, the user can configure the type of visibility checks +that are performed on the records. The behavior is controlled by a set of cursor flags: + +- By default, snapshot based visibility checks are performed on all records returned to the API +user. + +- When the flag \c WT_CURSTD_HS_READ_ALL is set on the cursor, no visibility checks are performed +on the records returned to the API user. This means that cursor interface can even return a record +with a globally visible tombstone. When this flag is set, it suppresses the effect of other +visibility flags. + +- When the flag \c WT_CURSTD_HS_READ_COMMITTED is set, the cursor interface can return any record +except for globally deleted records. This flag has no effect when the flag +\c WT_CURSTD_HS_READ_ALL is set. + +History store cursor users must use one of the flags if the caller thread is running in a lower +isolation level and doesn't hold a valid snapshot. + +@section arch_hs_reconciliation History store and reconciliation +When a dirty page is reconciled on a user file btree, the update chain is examined and the latest +committed update is chosen as the on-disk value. All older updates are added to the history store +table assuming they are not yet obsolete. Additionally any out of order timestamps will +be corrected. + +Consider the following update chain for a user table with btree id 1000 and data store key "AAA": + +@plantuml_start{hs_update_chain.png} +@startuml{hs_update_chain.png} + +together { +rectangle " U1 \n @ts 70" as U1 +rectangle " U2 \n @ts 80" as U2 +rectangle " U3 \n @ts 90" as U3 +rectangle " U4 \n @ts 100" as U4 + +U2-right->U1 +U3-right->U2 +U4-right->U3 + +} +@enduml +@plantuml_end + +Assuming all updates are committed, updates \c U1, \c U2 and \c U3 will be added to the history +store table as shown in the table below. + +<table style="height: 195px; width: 811px; border-color: Black;" border="4"> + <tr style="height: 32px;"> + <td style="width: 353.391px; height: 32px; text-align: center;" colspan="4"> + <strong>KEY</strong> + </td> + <td style="width: 456.609px; height: 32px; text-align: center;" colspan="4"> + <strong>VALUE</strong> + </td> + </tr> + <tr style="height: 32px;"> + <td style="width: 72.5156px; height: 32px; text-align: center;"> + <strong>Btree ID</strong> + </td> + <td style="width: 89.875px; height: 32px; text-align: center;"> + <strong>User Key</strong> + </td> + <td style="width: 97.0312px; height: 32px; text-align: center;"> + <strong>Start ts</strong> + </td> + <td style="width: 93.9688px; height: 32px; text-align: center;"> + <strong>Counter</strong> + </td> + <td style="width: 92.4375px; height: 32px; text-align: center;"> + <strong>Stop ts</strong> + </td> + <td style="width: 105.719px; height: 32px; text-align: center;"> + <strong>Durable ts</strong> + </td> + <td style="width: 110.312px; height: 32px; text-align: center;"> + <strong>Type</strong> + </td> + <td style="width: 148.141px; height: 32px; text-align: center;"> + <strong>Value</strong> + </td> + </tr> + <tr style="height: 32.5625px;"> + <td style="width: 72.5156px; height: 32.5625px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 89.875px; height: 32.5625px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 97.0312px; height: 32.5625px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 93.9688px; height: 32.5625px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 92.4375px; height: 32.5625px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 105.719px; height: 32.5625px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 110.312px; height: 32.5625px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 148.141px; height: 32.5625px; text-align: center;"> + <strong>...</strong> + </td> + </tr> + <tr style="height: 32px;"> + <td style="width: 72.5156px; height: 32px; text-align: center;">1000</td> + <td style="width: 89.875px; height: 32px; text-align: center;">"AAA"</td> + <td style="width: 97.0312px; height: 32px; text-align: center;">70</td> + <td style="width: 93.9688px; height: 32px; text-align: center;">0</td> + <td style="width: 92.4375px; height: 32px; text-align: center;">80</td> + <td style="width: 105.719px; height: 32px; text-align: center;">70</td> + <td style="width: 110.312px; height: 32px; text-align: center;">STANDARD</td> + <td style="width: 148.141px; height: 32px; text-align: center;">Value from U1</td> + </tr> + <tr style="height: 32px;"> + <td style="width: 72.5156px; height: 32px; text-align: center;">1000</td> + <td style="width: 89.875px; height: 32px; text-align: center;">"AAA"</td> + <td style="width: 97.0312px; height: 32px; text-align: center;">80</td> + <td style="width: 93.9688px; height: 32px; text-align: center;">0</td> + <td style="width: 92.4375px; height: 32px; text-align: center;">90</td> + <td style="width: 105.719px; height: 32px; text-align: center;">80</td> + <td style="width: 110.312px; height: 32px; text-align: center;">STANDARD</td> + <td style="width: 148.141px; height: 32px; text-align: center;">Value from U2</td> + </tr> + <tr style="height: 32px;"> + <td style="width: 72.5156px; height: 32px; text-align: center;">1000</td> + <td style="width: 89.875px; height: 32px; text-align: center;">"AAA"</td> + <td style="width: 97.0312px; height: 32px; text-align: center;">90</td> + <td style="width: 93.9688px; height: 32px; text-align: center;">0</td> + <td style="width: 92.4375px; height: 32px; text-align: center;">100</td> + <td style="width: 105.719px; height: 32px; text-align: center;">90</td> + <td style="width: 110.312px; height: 32px; text-align: center;">STANDARD</td> + <td style="width: 148.141px; height: 32px; text-align: center;">Value from U3</td> + </tr> + <tr style="height: 32px;"> + <td style="width: 72.5156px; height: 32px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 89.875px; height: 32px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 97.0312px; height: 32px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 93.9688px; height: 32px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 92.4375px; height: 32px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 105.719px; height: 32px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 110.312px; height: 32px; text-align: center;"> + <strong>...</strong> + </td> + <td style="width: 148.141px; height: 32px; text-align: center;"> + <strong>...</strong> + </td> + </tr> +</table> + +For the history store table entry corresponding to the update \c U1, the start timestamp in the key +is the start timestamp for the update \c U1 and the stop timestamp in the value is the start +timestamp for the update \c U2. + +When a modified history store table page is processed by reconciliation, a new page image is +created without records which have globally visible tombstones. This ensures that WiredTiger is +only keeping the relevant historical data required to serve the oldest reader in the system or +as dictated by the oldest timestamp set by the application. Once all records on a page are obsolete, +the page itself can be removed to reduce the size of the history store table +(see @ref arch-transaction for more details about oldest timestamp). + +@section arch_hs_read Searching for older versions of a key in History Store +When looking for an update visible to the current transaction, WiredTiger first searches the update +chain for any visible updates. If there is no visible update in the chain, WiredTiger then +inspects the on-disk version of the key. If that version is not visible to the transaction, +WiredTiger searches the history store table for a visible update for the key. When +searching for an update that satisfies the read timestamp constraint, WiredTiger starts +with the newest update of the key in the history store table and then iterates through the older +updates until there are no updates left to process. Note that although there can be multiple updates +in the history store for a key and read timestamp combination, each update would have different +visibility based on transaction and timestamp based visibility rules. In case there are no records +visible in the history store table, \c WT_NOTFOUND error is returned to the reader. + +@section arch_hs_rts History store and rollback-to-stable +Rollback-to-stable searches the history store table for valid updates that are stable according to +the supplied stable timestamp and replaces the on-disk version with an update from the history store +table that fulfills the criteria. Rollback-to-stable also removes updates from the +history store table that are not stable according to the stable timestamp and transaction snapshot +being used for rollback-to-stable operation. More details about the rollback-to-stable operation +can be found in the @ref arch-rts page. + +@section arch_hs_prepared History store and prepared transactions +When there is a prepared update for a key and the page is evicted, the prepared update is written +to the on-disk page and any older updates are written to the history store table. There can be +no update from a different transaction that is newer than the prepared update for the key and +therefore the history store should never contain a prepared update. When a prepared update is +committed, the stop timestamp of the newest update in the history store table is updated from +\c WT_TS_MAX to the commit timestamp of the prepared update. When a prepared update is rolled back, +the newest update from the history store table is restored as the on-disk version and then removed +from the history store table. + */ diff --git a/src/third_party/wiredtiger/src/docs/arch-index.dox b/src/third_party/wiredtiger/src/docs/arch-index.dox index 75d5a8abf5c..3d916e675aa 100644 --- a/src/third_party/wiredtiger/src/docs/arch-index.dox +++ b/src/third_party/wiredtiger/src/docs/arch-index.dox @@ -51,8 +51,7 @@ rectangle "**WiredTiger Engine** rectangle "[[arch-transaction.html Transactions]]" as txn rectangle "[[arch-metadata.html Metadata]]" as meta rectangle "[[arch-dhandle.html dhandle/\nBtree]]" as btree - rectangle "[[arch-row.html Row\nStorage]]" as row - rectangle "[[arch-column.html Column\nStorage]]" as column + rectangle "[[arch-row-column.html Row/Column\nStorage]]" as row rectangle "[[arch-hs.html History\nStore]]" as history rectangle "[[arch-snapshot.html Snapshots]]" as snapshot rectangle "[[arch-cache.html Cache]]" as cache @@ -123,6 +122,10 @@ wt_file -[hidden]right-> log_file We go into some detail for some of the internal components. +@subpage arch-backup + +Hot backup uses a type of cursor to backup the database. + @subpage arch-block The Block Manager manages the reading and writing of disk blocks. @@ -136,10 +139,6 @@ make up in-memory Btrees and subordinate data structures. A checkpoint is created by WiredTiger to serve as a point from which it can recover. -@subpage arch-column - -Column Stores are Btrees that have as their key a record id. - @subpage arch-cursor Cursors are used to get and modify data. @@ -184,9 +183,10 @@ Metadata is stored as <code>uri, config</code> K/V pairs in a designated table. WiredTiger has a Python API that is useful for scripting and experimentation. -@subpage arch-row +@subpage arch-row-column -Row Stores are Btrees that have a variable size key and data. +Row stores and column store are B-Trees. Row stores have a variable size key and data while column +stores have as their key a record id. @subpage arch-rts diff --git a/src/third_party/wiredtiger/src/docs/arch-metadata.dox b/src/third_party/wiredtiger/src/docs/arch-metadata.dox index d709dc78810..d5f89a43f3e 100644 --- a/src/third_party/wiredtiger/src/docs/arch-metadata.dox +++ b/src/third_party/wiredtiger/src/docs/arch-metadata.dox @@ -1,19 +1,76 @@ /*! @arch_page arch-metadata Metadata -Metadata in WiredTiger is stored as a table in the \c "WiredTiger.wt" file. -The table's key is a \c uri string, and the value is its complete -configuration string. The configuration itself is a list of key/value -pairs in string form. The configuration's keys are dependent on the -type of \c uri. Thus, a metadata entry with a \c uri key beginning with -\c "table:" will be a configuration string -having configuration entries like \c key_format and \c value_format to -describe the data encoding for the uri. A metadata key beginning with +Metadata in WiredTiger captures important information about the user's database. +Metadata includes tracking essential information such as the files and tables +present in the database, their associated configurations, as well as the latest checkpoints. +The checkpoint information tells WiredTiger where to the find +root page and all other parts of its tree when accessing the file. + +The main metadata in WiredTiger is stored in the \c WiredTiger.wt file. + +@section metadata_table Metadata Table + +The keys in the \c WiredTiger.wt table are \c uri strings. For example, keys can include \c "table:" & \c "file:" +prefixed URIs, representing typical row-store WiredTiger tables. A non-exhaustive list of other +example keys could also include \c "index:", \c "colgroups:" and \c "system:" prefixed URIs, representing +indexes, colgroups and special system values respectively. + +The value corresponding to a given \c uri key in the metadata table is its +associated configuration string. The configuration value itself is a list of key/value +pairs in string form. The key/value pairs in the configuration strings are dependent on the +type of \c uri. Thus, a metadata entry with a \c uri key beginning with \c "table:" will be +a configuration string having entries such as \c key_format and \c value_format to +describe the data encoding for the uri. A metadata key beginning with \c "file:" will have a different set of configuration entries associated -with it. +with it. See @ref config_strings for more details about WiredTiger configuration strings. + +@section metadata_checkpoint Storing Checkpoint Metadata + +The latest checkpoint information for each btree in the system is stored in its metadata entry. The checkpoint metadata +provides a known point in time from which WiredTiger can recover a btree's data after a restart. +The checkpoint metadata for a given btree is stored within the configuration string under its +associated \c uri key within the \c WiredTiger.wt table. Metadata configuration entries relevant to capturing +a given btree's checkpoint state include: +- \c checkpoint: The file's checkpoint entries, including their timestamp, transaction ID and write generation. Most importantly +this information also tracks an encoded address cookie that describes the blocks that make up the checkpoint's data. +- \c checkpoint_lsn: The LSN of the last checkpoint. +- \c checkpoint_backup_info: Incremental backup information. + +@section metadata_turtle Turtle file + +The metadata file \c WiredTiger.wt is a regular WiredTiger table itself. As such, the metadata table has a btree that also needs to be +checkpointed. Since a new metadata entry needs to be created for each btree at the time of checkpointing, the metadata table is the last +file to be checkpointed in the process. Checkpointing the metadata table last is important +for capturing a consistent state of the btrees in the database at the time of the last checkpoint. + +Due to this ordering, to successfully restore the checkpoint of the \c WiredTiger.wt file, WiredTiger captures the most +recent checkpointing information of the metadata table in a separate special turtle file called +\c "WiredTiger.turtle". The turtle file itself is metadata for the WiredTiger metadata. Using the +turtle file, WiredTiger can read the latest checkpoint information from the file when recovering the \c WiredTiger.wt table. +Recovering the checkpoint of the \c WiredTiger.wt table in turn restores a consistent view of the checkpointing information +for the other btrees in the system. + +@section metadata_reading Reading Metadata + +The contents of the metadata table is ASCII printable. The URIs and configuration values +are all printable strings. + +@subsection metadata_cursor "metadata:" cursor + +To read the metadata, a caller can open a cursor, via \c WT_SESSION::open_cursor, with a \c uri equal +to \c "metadata:". Changes to the metadata cannot be written through the metadata +cursor interface i.e. the metadata cursor is a read-only interface. Metadata is only changed/affected +by API calls. + +@subsection metadata_create_cursor "metadata:create" cursor + +Additionally, to only read strings relating to creation configurations of the various +\c uri keys present, a cursor can be opened with a \c uri equal to \c "metadata:create". This type of +cursor causes WiredTiger to strip out internal metadata when querying. -A caller of WiredTiger can use WT_SESSION::open_cursor with a \c uri equal to -\c "meta:" to read the metadata. Using this interface, metadata can only -be queried, not changed. Changes to the metadata are affected by API calls -such as WT_SESSION::create, WT_SESSION::drop, WT_SESSION::rename. +@subsection metadata_wt_util Printing metadata with the wt utility +Lastly the metadata in the \c WiredTiger.wt table can be dumped using the \c wt utility. A command option for +specifically printing the metadata is \c "wt list -v". This will display the various \c uri key strings with their associated +configuration strings. */ diff --git a/src/third_party/wiredtiger/src/docs/arch-row-column.dox b/src/third_party/wiredtiger/src/docs/arch-row-column.dox new file mode 100644 index 00000000000..d7d10f54795 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-row-column.dox @@ -0,0 +1,72 @@ +/*! @arch_page arch-row-column Row Store & Column Store + +@section row_column_definition Definition + +WiredTiger supports both row-stores and column-stores and each access method is represented +internally by a B-Tree. Keys and values in row-stores can be of variable length. Furthermore, keys +are explicitly stored. Column-stores use a 64-bit unsigned integer record id as keys and they are +implied given the row's position in the B-Tree. + +There are two types of column-stores in WiredTiger: +- The fixed length column-store, as its name implies, the value has a fixed length, and furthermore +the value is restricted to between 1 and 8 bits in length. The bit length is specified when the +column store is created. The fixed length column store has specialized use cases like bitmaps. +- The variable length is a more general case which allows for values to have any length similarly to +row-stores. + +Please refer to @ref arch-data-file to learn more about the on-disk format for row-stores and +column-stores. + +@section row_column_rs Row-stores + +Row-stores are probably seen as the traditional access method and they serve a general purpose. In +WiredTiger, row-store keys are explicitly stored and they can be duplicated within the same single +file which directly impacts the on-disk storage size. However, keys are only duplicated if they are +the first key on the page, which gets them copied/promoted to an internal page. They are not +necessarily duplicated, too, they can be prefix/suffix compressed to discard unnecessary bytes on +the internal pages. + +@section row_column_vlcs Variable length column-stores + +Keys in column-stores are not stored but derived from the row's position in the B-Tree which saves +disk space. In fact, there is a starting key on each page which is used to figure out the rest of +the keys present on that same page. Column-stores values also present an advantage to further save +more disk space as they can be written in more compact forms through encoding techniques. In +WiredTiger, the run-length encoding is used to replace any sequence of the same value with a count +and value indicator. However, this makes the access to a specific record more complicated, it makes +it impossible to jump to a record in a leaf page. + +@section row_column_flcs Fixed length column-stores + +Fixed length column stores are very different internally and usually serve specific purposes as it +is a <a href="https://en.wikipedia.org/wiki/Bitmap"><b>Bitmap</b></a>. It makes fixed length +column-stores efficient to retrieve a value at a given offset, the use of +<a href="https://en.wikipedia.org/wiki/Bloom_filter"><b>Bloom filter</b></a> is probably one of the +best examples. + +@section row_column_internal_use Internal usage + +Internally, row-stores and column-stores use a common \c WT_BTREE structure. The fundamental +difference is that the \c WT_BTREE->type field is set to \c BTREE_ROW for row-stores and +\c BTREE_COL for column-stores. Internal functions that navigate, access and manipulate B-Trees have +code sprinkled throughout that is conditional on \c WT_BTREE->type. + +<table> +@hrow{, Row Store, Column Store - Variable Length, Column Store - Fixed Length} +@row{Characteristics,,,,} +@row{Internal representation, B-Tree (\c WT_BTREE), B-Tree (\c WT_BTREE), B-Tree (\c WT_BTREE)} +@row{B-Tree type, Generalized (\c BTREE_ROW), Specialized (\c BTREE_COL_VAR), Specialized (\c BTREE_COL_FIX)} +@row{Leaf pages, Key count equal to half the number of physical entries (unless all empty values flag is set where key count is equal to the number of physical entries), Only the first key is stored, Only the first key is stored} +@row{Record key, Anything (byte-string) prefix compressed or overflow objects, Record id (64 bit unsigned integer), Record id (64 bit unsigned integer)} +@row{Record value, Variable byte string length, Variable byte string length, Fixed bit string length (up to 8 bits)} +@row{Features,,,,} +@row{Random cursor, Yes, No, No} +@row{Block compression, Yes, Yes, Yes} +@row{Dictionary compression, Yes, Yes, No} +@row{Fast truncate, Yes, No, No} +@row{Huffman encoding, Yes, Yes, No} +@row{Prefix and suffix compression, Yes, No, No} +@row{RLE compression, No, Yes, No} +</table> + +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-row.dox b/src/third_party/wiredtiger/src/docs/arch-row.dox deleted file mode 100644 index ae3ffe22589..00000000000 --- a/src/third_party/wiredtiger/src/docs/arch-row.dox +++ /dev/null @@ -1,12 +0,0 @@ -/*! @arch_page arch-row Row Store - -Row Stores are Btrees stored in WiredTiger that do not have a record id -as key. Thus, they implement a generalized version of a Btree, where -the key and data can be arbitrary length. - -Internally, a row store and column store both use a common \c WT_BTREE -structure. The fundamental difference is that -<code>WT_BTREE->type == BTREE_ROW</code> for row stores. -Internal functions that navigate, access and manipulate Btrees have -code sprinkled throughout that is conditional on <code>WT_BTREE->type</code>. -*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-rts.dox b/src/third_party/wiredtiger/src/docs/arch-rts.dox index 451732f5776..3e2cf6b70d0 100644 --- a/src/third_party/wiredtiger/src/docs/arch-rts.dox +++ b/src/third_party/wiredtiger/src/docs/arch-rts.dox @@ -16,8 +16,8 @@ updates are aborted and the on-disk version updates are replaced with an update from history store otherwise the data store version is removed. Rollback to stable is performed in three phases -1. WT startup -2. WT shutdown +1. WiredTiger startup +2. WiredTiger shutdown 3. Application initiated To improve the performance of rollback to stable operation, rollback to stable diff --git a/src/third_party/wiredtiger/src/docs/backup.dox b/src/third_party/wiredtiger/src/docs/backup.dox index 72188570706..01ecee137c7 100644 --- a/src/third_party/wiredtiger/src/docs/backup.dox +++ b/src/third_party/wiredtiger/src/docs/backup.dox @@ -77,6 +77,8 @@ the backup is available in the backup when that log file is included in the list of files. WiredTiger offers a mechanism to gather additional log files that may be created during the backup. +@section backup_duplicate Duplicate backup cursors + Since backups can take a long time, it may be desirable to catch up at the end of a backup with the log files so that operations that occurred during backup can be recovered. WiredTiger provides the ability to open a duplicate diff --git a/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/hs_update_chain.png b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/hs_update_chain.png Binary files differnew file mode 100644 index 00000000000..75b7261110d --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/hs_update_chain.png diff --git a/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.cmapx b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.cmapx index 2fdeef6a039..69ba9be858a 100644 --- a/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.cmapx +++ b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/wt_diagram.cmapx @@ -12,8 +12,7 @@ <area shape="rect" id="id11" href="arch-snapshot.html" title="arch-snapshot.html" alt="" coords="381,441,455,457"/> <area shape="rect" id="id12" href="arch-transaction.html" title="arch-transaction.html" alt="" coords="339,328,428,344"/> <area shape="rect" id="id13" href="arch-hs.html" title="arch-hs.html" alt="" coords="93,642,140,674"/> -<area shape="rect" id="id14" href="arch-row.html" title="arch-row.html" alt="" coords="163,433,217,465"/> -<area shape="rect" id="id15" href="arch-column.html" title="arch-column.html" alt="" coords="272,433,326,465"/> +<area shape="rect" id="id14" href="arch-row-column.html" title="arch-row-column.html" alt="" coords="163,433,217,465"/> <area shape="rect" id="id16" href="arch-block.html" title="arch-block.html" alt="" coords="196,642,256,674"/> <area shape="rect" id="id17" href="arch-dhandle.html" title="arch-dhandle.html" alt="" coords="181,320,242,352"/> <area shape="rect" id="id18" href="arch-data-file.html" title="arch-data-file.html" alt="" coords="179,865,245,897"/> diff --git a/src/third_party/wiredtiger/src/docs/js/sorttable.js b/src/third_party/wiredtiger/src/docs/js/sorttable.js new file mode 100644 index 00000000000..8b2d629c63b --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/js/sorttable.js @@ -0,0 +1,501 @@ +/* + SortTable + version 2 + 7th April 2007 + Stuart Langridge, http://www.kryogenix.org/code/browser/sorttable/ + + Instructions: + Download this file + Add <script src="sorttable.js"></script> to your HTML + Add class="sortable" to any table you'd like to make sortable + Click on the headers to sort + + Thanks to many, many people for contributions and suggestions. + Licenced as X11: http://www.kryogenix.org/code/browser/licence.html + This basically means: do what you want with it. +*/ +/* Modified for WiredTiger to sort alphabetic items with insensitive case. */ + + +var stIsIE = /*@cc_on!@*/false; + +sorttable = { + init: function() { + // quit if this function has already been called + if (arguments.callee.done) return; + // flag this function so we don't do the same thing twice + arguments.callee.done = true; + // kill the timer + if (_timer) clearInterval(_timer); + + if (!document.createElement || !document.getElementsByTagName) return; + + sorttable.DATE_RE = /^(\d\d?)[\/\.-](\d\d?)[\/\.-]((\d\d)?\d\d)$/; + + forEach(document.getElementsByTagName('table'), function(table) { + if (table.className.search(/\bsortable\b/) != -1) { + sorttable.makeSortable(table); + } + }); + + }, + + makeSortable: function(table) { + if (table.getElementsByTagName('thead').length == 0) { + // table doesn't have a tHead. Since it should have, create one and + // put the first table row in it. + the = document.createElement('thead'); + the.appendChild(table.rows[0]); + table.insertBefore(the,table.firstChild); + } + // Safari doesn't support table.tHead, sigh + if (table.tHead == null) table.tHead = table.getElementsByTagName('thead')[0]; + + if (table.tHead.rows.length != 1) return; // can't cope with two header rows + + // Sorttable v1 put rows with a class of "sortbottom" at the bottom (as + // "total" rows, for example). This is B&R, since what you're supposed + // to do is put them in a tfoot. So, if there are sortbottom rows, + // for backwards compatibility, move them to tfoot (creating it if needed). + sortbottomrows = []; + for (var i=0; i<table.rows.length; i++) { + if (table.rows[i].className.search(/\bsortbottom\b/) != -1) { + sortbottomrows[sortbottomrows.length] = table.rows[i]; + } + } + if (sortbottomrows) { + if (table.tFoot == null) { + // table doesn't have a tfoot. Create one. + tfo = document.createElement('tfoot'); + table.appendChild(tfo); + } + for (var i=0; i<sortbottomrows.length; i++) { + tfo.appendChild(sortbottomrows[i]); + } + delete sortbottomrows; + } + + // work through each column and calculate its type + headrow = table.tHead.rows[0].cells; + for (var i=0; i<headrow.length; i++) { + // manually override the type with a sorttable_type attribute + if (!headrow[i].className.match(/\bsorttable_nosort\b/)) { // skip this col + mtch = headrow[i].className.match(/\bsorttable_([a-z0-9]+)\b/); + if (mtch) { override = mtch[1]; } + if (mtch && typeof sorttable["sort_"+override] == 'function') { + headrow[i].sorttable_sortfunction = sorttable["sort_"+override]; + } else { + headrow[i].sorttable_sortfunction = sorttable.guessType(table,i); + } + // make it clickable to sort + headrow[i].sorttable_columnindex = i; + headrow[i].sorttable_tbody = table.tBodies[0]; + dean_addEvent(headrow[i],"click", sorttable.innerSortFunction = function(e) { + + if (this.className.search(/\bsorttable_sorted\b/) != -1) { + // if we're already sorted by this column, just + // reverse the table, which is quicker + sorttable.reverse(this.sorttable_tbody); + this.className = this.className.replace('sorttable_sorted', + 'sorttable_sorted_reverse'); + this.removeChild(document.getElementById('sorttable_sortfwdind')); + sortrevind = document.createElement('span'); + sortrevind.id = "sorttable_sortrevind"; + sortrevind.innerHTML = stIsIE ? ' <font face="webdings">5</font>' : ' ▴'; + this.appendChild(sortrevind); + return; + } + if (this.className.search(/\bsorttable_sorted_reverse\b/) != -1) { + // if we're already sorted by this column in reverse, just + // re-reverse the table, which is quicker + sorttable.reverse(this.sorttable_tbody); + this.className = this.className.replace('sorttable_sorted_reverse', + 'sorttable_sorted'); + this.removeChild(document.getElementById('sorttable_sortrevind')); + sortfwdind = document.createElement('span'); + sortfwdind.id = "sorttable_sortfwdind"; + sortfwdind.innerHTML = stIsIE ? ' <font face="webdings">6</font>' : ' ▾'; + this.appendChild(sortfwdind); + return; + } + + // remove sorttable_sorted classes + theadrow = this.parentNode; + forEach(theadrow.childNodes, function(cell) { + if (cell.nodeType == 1) { // an element + cell.className = cell.className.replace('sorttable_sorted_reverse',''); + cell.className = cell.className.replace('sorttable_sorted',''); + } + }); + sortfwdind = document.getElementById('sorttable_sortfwdind'); + if (sortfwdind) { sortfwdind.parentNode.removeChild(sortfwdind); } + sortrevind = document.getElementById('sorttable_sortrevind'); + if (sortrevind) { sortrevind.parentNode.removeChild(sortrevind); } + + this.className += ' sorttable_sorted'; + sortfwdind = document.createElement('span'); + sortfwdind.id = "sorttable_sortfwdind"; + sortfwdind.innerHTML = stIsIE ? ' <font face="webdings">6</font>' : ' ▾'; + this.appendChild(sortfwdind); + + // build an array to sort. This is a Schwartzian transform thing, + // i.e., we "decorate" each row with the actual sort key, + // sort based on the sort keys, and then put the rows back in order + // which is a lot faster because you only do getInnerText once per row + row_array = []; + col = this.sorttable_columnindex; + rows = this.sorttable_tbody.rows; + for (var j=0; j<rows.length; j++) { + row_array[row_array.length] = [sorttable.getInnerText(rows[j].cells[col]), rows[j]]; + } + /* If you want a stable sort, uncomment the following line */ + //sorttable.shaker_sort(row_array, this.sorttable_sortfunction); + /* and comment out this one */ + row_array.sort(this.sorttable_sortfunction); + + tb = this.sorttable_tbody; + for (var j=0; j<row_array.length; j++) { + tb.appendChild(row_array[j][1]); + } + + delete row_array; + }); + } + } + }, + + guessType: function(table, column) { + // guess the type of a column based on its first non-blank row + sortfn = sorttable.sort_alpha; + for (var i=0; i<table.tBodies[0].rows.length; i++) { + text = sorttable.getInnerText(table.tBodies[0].rows[i].cells[column]); + if (text != '') { + if (text.match(/^-?[£$¤]?[\d,.]+%?$/)) { + return sorttable.sort_numeric; + } + // check for a date: dd/mm/yyyy or dd/mm/yy + // can have / or . or - as separator + // can be mm/dd as well + possdate = text.match(sorttable.DATE_RE) + if (possdate) { + // looks like a date + first = parseInt(possdate[1]); + second = parseInt(possdate[2]); + if (first > 12) { + // definitely dd/mm + return sorttable.sort_ddmm; + } else if (second > 12) { + return sorttable.sort_mmdd; + } else { + // looks like a date, but we can't tell which, so assume + // that it's dd/mm (English imperialism!) and keep looking + sortfn = sorttable.sort_ddmm; + } + } + } + } + return sortfn; + }, + + getInnerText: function(node) { + // gets the text we want to use for sorting for a cell. + // strips leading and trailing whitespace. + // this is *not* a generic getInnerText function; it's special to sorttable. + // for example, you can override the cell text with a customkey attribute. + // it also gets .value for <input> fields. + + if (!node) return ""; + + hasInputs = (typeof node.getElementsByTagName == 'function') && + node.getElementsByTagName('input').length; + + if (node.getAttribute("sorttable_customkey") != null) { + return node.getAttribute("sorttable_customkey"); + } + else if (typeof node.textContent != 'undefined' && !hasInputs) { + return node.textContent.replace(/^\s+|\s+$/g, ''); + } + else if (typeof node.innerText != 'undefined' && !hasInputs) { + return node.innerText.replace(/^\s+|\s+$/g, ''); + } + else if (typeof node.text != 'undefined' && !hasInputs) { + return node.text.replace(/^\s+|\s+$/g, ''); + } + else { + switch (node.nodeType) { + case 3: + if (node.nodeName.toLowerCase() == 'input') { + return node.value.replace(/^\s+|\s+$/g, ''); + } + case 4: + return node.nodeValue.replace(/^\s+|\s+$/g, ''); + break; + case 1: + case 11: + var innerText = ''; + for (var i = 0; i < node.childNodes.length; i++) { + innerText += sorttable.getInnerText(node.childNodes[i]); + } + return innerText.replace(/^\s+|\s+$/g, ''); + break; + default: + return ''; + } + } + }, + + reverse: function(tbody) { + // reverse the rows in a tbody + newrows = []; + for (var i=0; i<tbody.rows.length; i++) { + newrows[newrows.length] = tbody.rows[i]; + } + for (var i=newrows.length-1; i>=0; i--) { + tbody.appendChild(newrows[i]); + } + delete newrows; + }, + + /* sort functions + each sort function takes two parameters, a and b + you are comparing a[0] and b[0] */ + sort_numeric: function(a,b) { + aa = parseFloat(a[0].replace(/[^0-9.-]/g,'')); + if (isNaN(aa)) aa = 0; + bb = parseFloat(b[0].replace(/[^0-9.-]/g,'')); + if (isNaN(bb)) bb = 0; + return aa-bb; + }, + sort_alpha: function(a,b) { + /* + * Now we sort case insensitive. This code was: + + if (a[0]==b[0]) return 0; + if (a[0]<b[0]) return -1; + */ + if (a[0].toLowerCase()==b[0].toLowerCase()) return 0; + if (a[0].toLowerCase()<b[0].toLowerCase()) return -1; + return 1; + }, + sort_ddmm: function(a,b) { + mtch = a[0].match(sorttable.DATE_RE); + y = mtch[3]; m = mtch[2]; d = mtch[1]; + if (m.length == 1) m = '0'+m; + if (d.length == 1) d = '0'+d; + dt1 = y+m+d; + mtch = b[0].match(sorttable.DATE_RE); + y = mtch[3]; m = mtch[2]; d = mtch[1]; + if (m.length == 1) m = '0'+m; + if (d.length == 1) d = '0'+d; + dt2 = y+m+d; + if (dt1==dt2) return 0; + if (dt1<dt2) return -1; + return 1; + }, + sort_mmdd: function(a,b) { + mtch = a[0].match(sorttable.DATE_RE); + y = mtch[3]; d = mtch[2]; m = mtch[1]; + if (m.length == 1) m = '0'+m; + if (d.length == 1) d = '0'+d; + dt1 = y+m+d; + mtch = b[0].match(sorttable.DATE_RE); + y = mtch[3]; d = mtch[2]; m = mtch[1]; + if (m.length == 1) m = '0'+m; + if (d.length == 1) d = '0'+d; + dt2 = y+m+d; + if (dt1==dt2) return 0; + if (dt1<dt2) return -1; + return 1; + }, + + shaker_sort: function(list, comp_func) { + // A stable sort function to allow multi-level sorting of data + // see: http://en.wikipedia.org/wiki/Cocktail_sort + // thanks to Joseph Nahmias + var b = 0; + var t = list.length - 1; + var swap = true; + + while(swap) { + swap = false; + for(var i = b; i < t; ++i) { + if ( comp_func(list[i], list[i+1]) > 0 ) { + var q = list[i]; list[i] = list[i+1]; list[i+1] = q; + swap = true; + } + } // for + t--; + + if (!swap) break; + + for(var i = t; i > b; --i) { + if ( comp_func(list[i], list[i-1]) < 0 ) { + var q = list[i]; list[i] = list[i-1]; list[i-1] = q; + swap = true; + } + } // for + b++; + + } // while(swap) + } +} + +/* ****************************************************************** + Supporting functions: bundled here to avoid depending on a library + ****************************************************************** */ + +// Dean Edwards/Matthias Miller/John Resig + +/* for Mozilla/Opera9 */ +if (document.addEventListener) { + document.addEventListener("DOMContentLoaded", sorttable.init, false); +} + +/* for Internet Explorer */ +/*@cc_on @*/ +/*@if (@_win32) + document.write("<script id=__ie_onload defer src=javascript:void(0)><\/script>"); + var script = document.getElementById("__ie_onload"); + script.onreadystatechange = function() { + if (this.readyState == "complete") { + sorttable.init(); // call the onload handler + } + }; +/*@end @*/ + +/* for Safari */ +if (/WebKit/i.test(navigator.userAgent)) { // sniff + var _timer = setInterval(function() { + if (/loaded|complete/.test(document.readyState)) { + sorttable.init(); // call the onload handler + } + }, 10); +} + +/* for other browsers */ +window.onload = sorttable.init; + +// written by Dean Edwards, 2005 +// with input from Tino Zijdel, Matthias Miller, Diego Perini + +// http://dean.edwards.name/weblog/2005/10/add-event/ + +function dean_addEvent(element, type, handler) { + if (element.addEventListener) { + element.addEventListener(type, handler, false); + } else { + // assign each event handler a unique ID + if (!handler.$$guid) handler.$$guid = dean_addEvent.guid++; + // create a hash table of event types for the element + if (!element.events) element.events = {}; + // create a hash table of event handlers for each element/event pair + var handlers = element.events[type]; + if (!handlers) { + handlers = element.events[type] = {}; + // store the existing event handler (if there is one) + if (element["on" + type]) { + handlers[0] = element["on" + type]; + } + } + // store the event handler in the hash table + handlers[handler.$$guid] = handler; + // assign a global event handler to do all the work + element["on" + type] = handleEvent; + } +}; +// a counter used to create unique IDs +dean_addEvent.guid = 1; + +function removeEvent(element, type, handler) { + if (element.removeEventListener) { + element.removeEventListener(type, handler, false); + } else { + // delete the event handler from the hash table + if (element.events && element.events[type]) { + delete element.events[type][handler.$$guid]; + } + } +}; + +function handleEvent(event) { + var returnValue = true; + // grab the event object (IE uses a global event object) + event = event || fixEvent(((this.ownerDocument || this.document || this).parentWindow || window).event); + // get a reference to the hash table of event handlers + var handlers = this.events[event.type]; + // execute each event handler + for (var i in handlers) { + this.$$handleEvent = handlers[i]; + if (this.$$handleEvent(event) === false) { + returnValue = false; + } + } + return returnValue; +}; + +function fixEvent(event) { + // add W3C standard event methods + event.preventDefault = fixEvent.preventDefault; + event.stopPropagation = fixEvent.stopPropagation; + return event; +}; +fixEvent.preventDefault = function() { + this.returnValue = false; +}; +fixEvent.stopPropagation = function() { + this.cancelBubble = true; +} + +// Dean's forEach: http://dean.edwards.name/base/forEach.js +/* + forEach, version 1.0 + Copyright 2006, Dean Edwards + License: http://www.opensource.org/licenses/mit-license.php +*/ + +// array-like enumeration +if (!Array.forEach) { // mozilla already supports this + Array.forEach = function(array, block, context) { + for (var i = 0; i < array.length; i++) { + block.call(context, array[i], i, array); + } + }; +} + +// generic enumeration +Function.prototype.forEach = function(object, block, context) { + for (var key in object) { + if (typeof this.prototype[key] == "undefined") { + block.call(context, object[key], key, object); + } + } +}; + +// character enumeration +String.forEach = function(string, block, context) { + Array.forEach(string.split(""), function(chr, index) { + block.call(context, chr, index, string); + }); +}; + +// globally resolve forEach enumeration +var forEach = function(object, block, context) { + if (object) { + var resolve = Object; // default + if (object instanceof Function) { + // functions have a "length" property + resolve = Function; + } else if (object.forEach instanceof Function) { + // the object implements a custom forEach method so use that + object.forEach(block, context); + return; + } else if (typeof object == "string") { + // the object is a string + resolve = String; + } else if (typeof object.length == "number") { + // the object is array-like + resolve = Array; + } + resolve.forEach(object, block, context); + } +}; diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index d4b22545abe..1d302409f2e 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -1,5 +1,6 @@ personal_ws-1.1 en 200 ACM +ADDR APIs ASAN ActiveState @@ -7,6 +8,7 @@ Adler's Atomicity BLOBs BLRrVv +CAS CFLAGS CMake COV @@ -107,9 +109,11 @@ PPC PRELOAD PROFDATA README +RLE RedHat RepMgr Riak +RLE RocksDB Roelofs RrVv @@ -149,6 +153,7 @@ aR abc abstime ack'ed +addr ajn alloc allocator @@ -179,6 +184,7 @@ br btmem btree btrees +btree's bufs builtin builtins @@ -193,6 +199,7 @@ cdb cds changelog checkpointed +checkpointing checksum checksums ckp @@ -201,6 +208,7 @@ clickable cmake colgroup colgroups +colspan combinatorial command's comparator @@ -224,6 +232,7 @@ cryptographically csuite ctest curfile +curhs cursortype curtable customerABC @@ -263,6 +272,7 @@ dl dll dlp dontlock +doxtable dp ds dsrc @@ -312,6 +322,7 @@ fillfactor firstname flamegraph flamegraphs +flcs fnv forw fprofile @@ -358,6 +369,7 @@ insn intl intpack inuse +invariants io ip je @@ -374,6 +386,7 @@ keyfmt keyid keyname keyvalue +kv kvs lang lastname @@ -510,6 +523,7 @@ putKey putKeyString putValue putValueString +px py qnx qqq @@ -560,6 +574,7 @@ sess sid skinparam skiplist +sortable spinlock spinlocks sql @@ -637,6 +652,7 @@ valuev vec versa viewable +vlcs vm vpmsum warmup diff --git a/src/third_party/wiredtiger/src/docs/style/header-web.html b/src/third_party/wiredtiger/src/docs/style/header-web.html index 63b22bb763e..ec15f09bba3 100644 --- a/src/third_party/wiredtiger/src/docs/style/header-web.html +++ b/src/third_party/wiredtiger/src/docs/style/header-web.html @@ -8,6 +8,7 @@ <link href="$relpath$tabs.css" rel="stylesheet" type="text/css"/> <script type="text/javascript" src="$relpath$jquery.js"></script> <script type="text/javascript" src="$relpath$dynsections.js"></script> +<script type="text/javascript" src="$relpath$sorttable.js"></script> $treeview $search $mathjax diff --git a/src/third_party/wiredtiger/src/docs/style/header.html b/src/third_party/wiredtiger/src/docs/style/header.html index 422268b7a47..559e3e690da 100644 --- a/src/third_party/wiredtiger/src/docs/style/header.html +++ b/src/third_party/wiredtiger/src/docs/style/header.html @@ -8,6 +8,7 @@ <link href="$relpath$tabs.css" rel="stylesheet" type="text/css"/> <script type="text/javascript" src="$relpath$jquery.js"></script> <script type="text/javascript" src="$relpath$dynsections.js"></script> +<script type="text/javascript" src="$relpath$sorttable.js"></script> $treeview $search $mathjax diff --git a/src/third_party/wiredtiger/src/docs/testing.dox b/src/third_party/wiredtiger/src/docs/testing.dox index b33ad279562..5b9835f6cd9 100644 --- a/src/third_party/wiredtiger/src/docs/testing.dox +++ b/src/third_party/wiredtiger/src/docs/testing.dox @@ -37,7 +37,7 @@ into the WiredTiger GitHub source tree. @subsection test_csuite CSuite @subsection test_cppsuite CPPSuite @subsection test_format Format -@subsection test_perf WT Perf +@subsection test_perf WiredTiger Perf @subsection test_workgen Workgen @section testing_compatibility Compatibility Testing diff --git a/src/third_party/wiredtiger/src/docs/tool-optrack.dox b/src/third_party/wiredtiger/src/docs/tool-optrack.dox index c13dd31ae84..e39a17e3a08 100644 --- a/src/third_party/wiredtiger/src/docs/tool-optrack.dox +++ b/src/third_party/wiredtiger/src/docs/tool-optrack.dox @@ -104,7 +104,7 @@ efficient than writing strings, and the map file is needed to decode them. Now the binary log files need to be converted into the text format. To do that use the script `wt_optrack_decode.py` in the WiredTiger tree. We will refer to -the path to your WiredTiger tree as WT. Suppose that the process ID that +the path to your WiredTiger tree as WiredTiger. Suppose that the process ID that generated the operation tracking files is 25660. Then you'd run the decode script like so: @@ -124,7 +124,7 @@ nanoseconds. In the end, you should see lots of output files whose names look like the optrack log files, but with the suffixes `-internal.txt` and `-external.txt`. -The "internal" files are the log files for WT internal sessions (such as +The "internal" files are the log files for WiredTiger internal sessions (such as eviction threads). The "external" files are for the sessions created by the client application. @@ -195,7 +195,7 @@ workload described earlier in this tutorial. Please open the URL, because we will now walk over what it shows. The main page shows a number of outlier histograms. Each histogram corresponds -to a function that was tracked by the operation tracking system in WT. The +to a function that was tracked by the operation tracking system in WiredTiger. The x-axis shows the execution timeline (in nanoseconds). The y-axis shows how many abnormally long executions of the function occurred during very period of the execution. diff --git a/src/third_party/wiredtiger/src/docs/tune-durability.dox b/src/third_party/wiredtiger/src/docs/tune-durability.dox index a16c2938d9c..5350c617d3b 100644 --- a/src/third_party/wiredtiger/src/docs/tune-durability.dox +++ b/src/third_party/wiredtiger/src/docs/tune-durability.dox @@ -79,12 +79,6 @@ by the \c transaction_sync=(method) setting before returning. If \c sync=off is configured then this commit operation will write its records into the in-memory buffer and return immediately. -If \c sync=background is configured then this commit operation will write -its record to an in-memory buffer, and will return. Prior to returning it -will signal an internal WiredTiger worker thread to synchronize this log -record. The caller may then check the status of that background -synchronization with the WT_SESSION::transaction_sync method. - The durability of the write-ahead log can be controlled independently as well via the WT_SESSION::log_flush method. The WT_SESSION::log_flush supports several durability modes with @@ -99,10 +93,6 @@ If \c sync=off is configured then this flush operation will force the logging subsystem to write any outstanding in-memory buffers to the file system before returning. -If \c sync=background is configured then this flush operation will force -the signaling of a background synchronization operation. -The caller may then check the status of that background -synchronization with the WT_SESSION::transaction_sync method. */ /*! @page tune_durability Commit-level durability diff --git a/src/third_party/wiredtiger/src/docs/upgrading.dox b/src/third_party/wiredtiger/src/docs/upgrading.dox index e92dde30464..c5f4b2d013e 100644 --- a/src/third_party/wiredtiger/src/docs/upgrading.dox +++ b/src/third_party/wiredtiger/src/docs/upgrading.dox @@ -1,6 +1,16 @@ /*! @page upgrading Upgrading WiredTiger applications </dl><hr> +@section version_1001 Upgrading to Version 10.0.1 +<dl> + +<dt>Background sync APIs</dt> +<dd> +The WT_SESSION::transaction_sync API and the sync=background configuration option to +WT_SESSION::log_flush and WT_SESSION::commit_transaction have all been removed from WiredTiger. +</dd> + +</dl><hr> @section version_1000 Upgrading to Version 10.0.0 <dl> diff --git a/src/third_party/wiredtiger/src/include/block.h b/src/third_party/wiredtiger/src/include/block.h index cd6cf3e662e..7b4fa732027 100644 --- a/src/third_party/wiredtiger/src/include/block.h +++ b/src/third_party/wiredtiger/src/include/block.h @@ -325,7 +325,7 @@ struct __wt_block_desc { */ struct __wt_block_file_opener { /* An id to be used with the open call to reference the current object. */ -#define WT_TIERED_CURRENT_ID 0xFFFFFFFFUL +#define WT_TIERED_CURRENT_ID UINT32_MAX int (*open)( WT_BLOCK_FILE_OPENER *, WT_SESSION_IMPL *, uint32_t, WT_FS_OPEN_FILE_TYPE, u_int, WT_FH **); uint32_t (*current_object_id)(WT_BLOCK_FILE_OPENER *); diff --git a/src/third_party/wiredtiger/src/include/btree_inline.h b/src/third_party/wiredtiger/src/include/btree_inline.h index 6e2add93605..adcf81b83bb 100644 --- a/src/third_party/wiredtiger/src/include/btree_inline.h +++ b/src/third_party/wiredtiger/src/include/btree_inline.h @@ -724,6 +724,13 @@ static inline void __wt_page_modify_set(WT_SESSION_IMPL *session, WT_PAGE *page) { /* + * Prepared records in the datastore require page updates, even for read-only handles, don't + * mark the tree or page dirty. + */ + if (F_ISSET(S2BT(session), WT_BTREE_READONLY)) + return; + + /* * Mark the tree dirty (even if the page is already marked dirty), newly created pages to * support "empty" files are dirty, but the file isn't marked dirty until there's a real change * needing to be written. @@ -1992,34 +1999,6 @@ __wt_page_swap_func(WT_SESSION_IMPL *session, WT_REF *held, WT_REF *want, uint32 } /* - * __wt_bt_col_var_cursor_walk_txn_read -- - * Set the value for variable-length column-store cursor walk. - */ -static inline int -__wt_bt_col_var_cursor_walk_txn_read(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_PAGE *page, - WT_CELL_UNPACK_KV *unpack, WT_COL *cip) -{ - WT_UPDATE *upd; - - upd = NULL; - - if (cbt->ins) - upd = cbt->ins->upd; - - cbt->slot = WT_COL_SLOT(page, cip); - WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, upd, unpack)); - if (cbt->upd_value->type == WT_UPDATE_INVALID || cbt->upd_value->type == WT_UPDATE_TOMBSTONE) - return (0); - - WT_RET(__wt_value_return(cbt, cbt->upd_value)); - - cbt->tmp->data = cbt->iface.value.data; - cbt->tmp->size = cbt->iface.value.size; - cbt->cip_saved = cip; - return (0); -} - -/* * __wt_btcur_skip_page -- * Return if the cursor is pointing to a page with deleted records and can be skipped for cursor * traversal. diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h index 3b092857ed3..29c576defd8 100644 --- a/src/third_party/wiredtiger/src/include/config.h +++ b/src/third_party/wiredtiger/src/include/config.h @@ -91,23 +91,22 @@ struct __wt_config_parser_impl { #define WT_CONFIG_ENTRY_WT_SESSION_salvage 37 #define WT_CONFIG_ENTRY_WT_SESSION_strerror 38 #define WT_CONFIG_ENTRY_WT_SESSION_timestamp_transaction 39 -#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 40 -#define WT_CONFIG_ENTRY_WT_SESSION_truncate 41 -#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 42 -#define WT_CONFIG_ENTRY_WT_SESSION_verify 43 -#define WT_CONFIG_ENTRY_colgroup_meta 44 -#define WT_CONFIG_ENTRY_file_config 45 -#define WT_CONFIG_ENTRY_file_meta 46 -#define WT_CONFIG_ENTRY_index_meta 47 -#define WT_CONFIG_ENTRY_lsm_meta 48 -#define WT_CONFIG_ENTRY_object_meta 49 -#define WT_CONFIG_ENTRY_table_meta 50 -#define WT_CONFIG_ENTRY_tier_meta 51 -#define WT_CONFIG_ENTRY_tiered_meta 52 -#define WT_CONFIG_ENTRY_wiredtiger_open 53 -#define WT_CONFIG_ENTRY_wiredtiger_open_all 54 -#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 55 -#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 56 +#define WT_CONFIG_ENTRY_WT_SESSION_truncate 40 +#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 41 +#define WT_CONFIG_ENTRY_WT_SESSION_verify 42 +#define WT_CONFIG_ENTRY_colgroup_meta 43 +#define WT_CONFIG_ENTRY_file_config 44 +#define WT_CONFIG_ENTRY_file_meta 45 +#define WT_CONFIG_ENTRY_index_meta 46 +#define WT_CONFIG_ENTRY_lsm_meta 47 +#define WT_CONFIG_ENTRY_object_meta 48 +#define WT_CONFIG_ENTRY_table_meta 49 +#define WT_CONFIG_ENTRY_tier_meta 50 +#define WT_CONFIG_ENTRY_tiered_meta 51 +#define WT_CONFIG_ENTRY_wiredtiger_open 52 +#define WT_CONFIG_ENTRY_wiredtiger_open_all 53 +#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 54 +#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 55 /* * configuration section: END * DO NOT EDIT: automatically built by dist/flags.py. diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index 9425292ad8a..569da6aabaa 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -154,7 +154,10 @@ struct __wt_cursor_btree { * Variable-length column-store values are run-length encoded and may be overflow values or * Huffman encoded. To avoid repeatedly reading overflow values or decompressing encoded values, * process it once and store the result in a temporary buffer. The cip_saved field is used to - * determine if we've switched columns since our last cursor call. + * determine if we've switched columns since our last cursor call. Note however that this result + * caching is not necessarily safe for all RLE cells. The flag WT_CBT_CACHEABLE_RLE_CELL + * indicates that the value is uniform across the whole cell. If it is not set (e.g. if the cell + * is not globally visible yet), the cached values should not be used. */ WT_COL *cip_saved; /* Last iteration reference */ @@ -214,16 +217,17 @@ struct __wt_cursor_btree { /* AUTOMATIC FLAG VALUE GENERATION START 0 */ #define WT_CBT_ACTIVE 0x001u /* Active in the tree */ -#define WT_CBT_ITERATE_APPEND 0x002u /* Col-store: iterating append list */ -#define WT_CBT_ITERATE_NEXT 0x004u /* Next iteration configuration */ -#define WT_CBT_ITERATE_PREV 0x008u /* Prev iteration configuration */ -#define WT_CBT_ITERATE_RETRY_NEXT 0x010u /* Prepare conflict by next. */ -#define WT_CBT_ITERATE_RETRY_PREV 0x020u /* Prepare conflict by prev. */ -#define WT_CBT_NO_TRACKING 0x040u /* Non tracking cursor. */ -#define WT_CBT_NO_TXN 0x080u /* Non-txn cursor (e.g. a checkpoint) */ -#define WT_CBT_READ_ONCE 0x100u /* Page in with WT_READ_WONT_NEED */ -#define WT_CBT_SEARCH_SMALLEST 0x200u /* Row-store: small-key insert list */ -#define WT_CBT_VAR_ONPAGE_MATCH 0x400u /* Var-store: on-page recno match */ +#define WT_CBT_CACHEABLE_RLE_CELL 0x002u /* Col-store: value in RLE cell valid for all its keys */ +#define WT_CBT_ITERATE_APPEND 0x004u /* Col-store: iterating append list */ +#define WT_CBT_ITERATE_NEXT 0x008u /* Next iteration configuration */ +#define WT_CBT_ITERATE_PREV 0x010u /* Prev iteration configuration */ +#define WT_CBT_ITERATE_RETRY_NEXT 0x020u /* Prepare conflict by next. */ +#define WT_CBT_ITERATE_RETRY_PREV 0x040u /* Prepare conflict by prev. */ +#define WT_CBT_NO_TRACKING 0x080u /* Non tracking cursor. */ +#define WT_CBT_NO_TXN 0x100u /* Non-txn cursor (e.g. a checkpoint) */ +#define WT_CBT_READ_ONCE 0x200u /* Page in with WT_READ_WONT_NEED */ +#define WT_CBT_SEARCH_SMALLEST 0x400u /* Row-store: small-key insert list */ +#define WT_CBT_VAR_ONPAGE_MATCH 0x800u /* Var-store: on-page recno match */ /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */ #define WT_CBT_POSITION_MASK /* Flags associated with position */ \ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 38d8f72a052..68b89d99b0e 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -4,7 +4,7 @@ extern WT_HAZARD *__wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref, WT_SESSION_IMPL **sessionp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern WT_THREAD_RET __wt_cache_pool_server(void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern WT_UPDATE *__wt_update_obsolete_check(WT_SESSION_IMPL *session, WT_PAGE *page, +extern WT_UPDATE *__wt_update_obsolete_check(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd, bool update_accounting) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_block_offset_invalid(WT_BLOCK *block, wt_off_t offset, uint32_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -87,18 +87,15 @@ extern int __wt_bad_object_type(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_addr_invalid(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool live) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_addr_pack(WT_BLOCK *block, uint8_t **pp, uint32_t objectid, wt_off_t offset, + uint32_t size, uint32_t checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_addr_string(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_addr_to_buffer(WT_BLOCK *block, uint8_t **pp, uint32_t objectid, - wt_off_t offset, uint32_t size, uint32_t checksum) +extern int __wt_block_addr_unpack(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *p, + size_t addr_size, uint32_t *objectidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_alloc(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t *offp, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, uint32_t *objectidp, - wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_buffer_to_ckpt(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *p, - WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_checkpoint(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, WT_CKPT *ckptbase, bool data_checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_checkpoint_final(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, @@ -114,13 +111,15 @@ extern int __wt_block_checkpoint_start(WT_SESSION_IMPL *session, WT_BLOCK *block WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_checkpoint_unload(WT_SESSION_IMPL *session, WT_BLOCK *block, bool checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_ckpt_decode(WT_SESSION *wt_session, WT_BLOCK *block, const uint8_t *p, - WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) +extern int __wt_block_ckpt_decode(WT_SESSION *wt_session, WT_BLOCK *block, const uint8_t *ckpt, + size_t ckpt_size, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_ckpt_init(WT_SESSION_IMPL *session, WT_BLOCK_CKPT *ci, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t **pp, +extern int __wt_block_ckpt_pack(WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci, bool skip_avail) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_ckpt_unpack(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *ckpt, + size_t ckpt_size, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_close(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_compact_end(WT_SESSION_IMPL *session, WT_BLOCK *block) @@ -156,7 +155,7 @@ extern int __wt_block_extlist_truncate(WT_SESSION_IMPL *session, WT_BLOCK *block WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_extlist_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, WT_EXTLIST *additional) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_fh(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t objectid, WT_FH **fhp) +extern int __wt_block_fh(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t object_id, WT_FH **fhp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_free(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -202,8 +201,6 @@ extern int __wt_block_size_alloc(WT_SESSION_IMPL *session, WT_SIZE **szp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_switch_object(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t object_id, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_tiered_fh(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t object_id, - WT_FH **fhp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_truncate(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_unmap(WT_SESSION_IMPL *session, WT_BLOCK *block, void *mapped_region, @@ -1178,7 +1175,9 @@ extern int __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t fla #endif ) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_inmem(WT_SESSION_IMPL *session, WT_REF *ref, const void *image, uint32_t flags, - WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); + WT_PAGE **pagep, bool *preparedp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_page_inmem_prepare(WT_SESSION_IMPL *session, WT_REF *ref) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_modify_alloc(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_release_evict(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) @@ -1694,7 +1693,7 @@ extern void __wt_checkpoint_progress(WT_SESSION_IMPL *session, bool closing); extern void __wt_checkpoint_signal(WT_SESSION_IMPL *session, wt_off_t logsize); extern void __wt_checkpoint_tree_reconcile_update(WT_SESSION_IMPL *session, WT_TIME_AGGREGATE *ta); extern void __wt_ckpt_verbose(WT_SESSION_IMPL *session, WT_BLOCK *block, const char *tag, - const char *ckpt_name, const uint8_t *ckpt_string); + const char *ckpt_name, const uint8_t *ckpt_string, size_t ckpt_size); extern void __wt_cond_auto_wait( WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool (*run_func)(WT_SESSION_IMPL *)); extern void __wt_cond_auto_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, @@ -1763,7 +1762,6 @@ extern void __wt_hs_close(WT_SESSION_IMPL *session); extern void __wt_hs_upd_time_window(WT_CURSOR *hs_cursor, WT_TIME_WINDOW **twp); extern void __wt_huffman_close(WT_SESSION_IMPL *session, void *huffman_arg); extern void __wt_json_close(WT_SESSION_IMPL *session, WT_CURSOR *cursor); -extern void __wt_log_background(WT_SESSION_IMPL *session, WT_LSN *lsn); extern void __wt_log_ckpt(WT_SESSION_IMPL *session, WT_LSN *ckpt_lsn); extern void __wt_log_slot_activate(WT_SESSION_IMPL *session, WT_LOGSLOT *slot); extern void __wt_log_slot_free(WT_SESSION_IMPL *session, WT_LOGSLOT *slot); @@ -1969,9 +1967,6 @@ static inline bool __wt_txn_visible_all(WT_SESSION_IMPL *session, uint64_t id, wt_timestamp_t timestamp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline double __wt_eviction_dirty_target(WT_CACHE *cache) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline int __wt_bt_col_var_cursor_walk_txn_read(WT_SESSION_IMPL *session, - WT_CURSOR_BTREE *cbt, WT_PAGE *page, WT_CELL_UNPACK_KV *unpack, WT_COL *cip) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_btree_block_free(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_buf_extend(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) @@ -2144,8 +2139,7 @@ static inline int __wt_txn_modify_page_delete(WT_SESSION_IMPL *session, WT_REF * static inline int __wt_txn_op_set_key(WT_SESSION_IMPL *session, const WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_read(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, - uint64_t recno, WT_UPDATE *upd, WT_CELL_UNPACK_KV *vpack) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); + uint64_t recno, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_read_upd_list(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_read_upd_list_internal(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h index 132a1512b37..641c91babc8 100644 --- a/src/third_party/wiredtiger/src/include/log.h +++ b/src/third_party/wiredtiger/src/include/log.h @@ -15,11 +15,10 @@ /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */ /* AUTOMATIC FLAG VALUE GENERATION START 0 */ -#define WT_LOG_BACKGROUND 0x01u -#define WT_LOG_DSYNC 0x02u -#define WT_LOG_FLUSH 0x04u -#define WT_LOG_FSYNC 0x08u -#define WT_LOG_SYNC_ENABLED 0x10u +#define WT_LOG_DSYNC 0x1u +#define WT_LOG_FLUSH 0x2u +#define WT_LOG_FSYNC 0x4u +#define WT_LOG_SYNC_ENABLED 0x8u /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */ #define WT_LOGOP_IGNORE 0x80000000 @@ -257,7 +256,6 @@ struct __wt_log { * System LSNs */ WT_LSN alloc_lsn; /* Next LSN for allocation */ - WT_LSN bg_sync_lsn; /* Latest background sync LSN */ WT_LSN ckpt_lsn; /* Last checkpoint LSN */ WT_LSN dirty_lsn; /* LSN of last non-synced write */ WT_LSN first_lsn; /* First LSN */ diff --git a/src/third_party/wiredtiger/src/include/serial_inline.h b/src/third_party/wiredtiger/src/include/serial_inline.h index 1fb7e31496e..8e6813f2866 100644 --- a/src/third_party/wiredtiger/src/include/serial_inline.h +++ b/src/third_party/wiredtiger/src/include/serial_inline.h @@ -229,6 +229,7 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_PAGE *page upd = *updp; *updp = NULL; prev_upd_ts = WT_TS_NONE; + #ifdef HAVE_DIAGNOSTIC prev_upd_ts = upd->prev_durable_ts; #endif @@ -296,7 +297,7 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_PAGE *page if (WT_PAGE_TRYLOCK(session, page) != 0) return (0); - obsolete = __wt_update_obsolete_check(session, page, upd->next, true); + obsolete = __wt_update_obsolete_check(session, cbt, upd->next, true); WT_PAGE_UNLOCK(session, page); diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 1bddc6a193e..465f641954b 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -138,9 +138,6 @@ struct __wt_session_impl { WT_TXN_ISOLATION isolation; WT_TXN *txn; /* Transaction state */ -#define WT_SESSION_BG_SYNC_MSEC 1200000 - WT_LSN bg_sync_lsn; /* Background sync operation LSN. */ - void *block_manager; /* Block-manager support */ int (*block_manager_cleanup)(WT_SESSION_IMPL *); @@ -200,17 +197,16 @@ struct __wt_session_impl { #define WT_SESSION_IGNORE_CACHE_SIZE 0x00020u #define WT_SESSION_IMPORT 0x00040u #define WT_SESSION_IMPORT_REPAIR 0x00080u -#define WT_SESSION_INSTANTIATE_PREPARE 0x00100u -#define WT_SESSION_INTERNAL 0x00200u -#define WT_SESSION_LOGGING_INMEM 0x00400u -#define WT_SESSION_NO_DATA_HANDLES 0x00800u -#define WT_SESSION_NO_LOGGING 0x01000u -#define WT_SESSION_NO_RECONCILE 0x02000u -#define WT_SESSION_QUIET_CORRUPT_FILE 0x04000u -#define WT_SESSION_READ_WONT_NEED 0x08000u -#define WT_SESSION_RESOLVING_TXN 0x10000u -#define WT_SESSION_ROLLBACK_TO_STABLE 0x20000u -#define WT_SESSION_SCHEMA_TXN 0x40000u +#define WT_SESSION_INTERNAL 0x00100u +#define WT_SESSION_LOGGING_INMEM 0x00200u +#define WT_SESSION_NO_DATA_HANDLES 0x00400u +#define WT_SESSION_NO_LOGGING 0x00800u +#define WT_SESSION_NO_RECONCILE 0x01000u +#define WT_SESSION_QUIET_CORRUPT_FILE 0x02000u +#define WT_SESSION_READ_WONT_NEED 0x04000u +#define WT_SESSION_RESOLVING_TXN 0x08000u +#define WT_SESSION_ROLLBACK_TO_STABLE 0x10000u +#define WT_SESSION_SCHEMA_TXN 0x20000u /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */ uint32_t flags; diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index b0b161b876d..893179db355 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -397,6 +397,7 @@ struct __wt_connection_stats { int64_t cache_eviction_force_clean_time; int64_t cache_eviction_force_dirty; int64_t cache_eviction_force_dirty_time; + int64_t cache_eviction_force_long_update_list; int64_t cache_eviction_force_delete; int64_t cache_eviction_force; int64_t cache_eviction_force_fail; @@ -716,7 +717,6 @@ struct __wt_connection_stats { int64_t conn_close_blocked_lsm; int64_t dhandle_lock_blocked; int64_t page_index_slot_ref_blocked; - int64_t log_server_sync_blocked; int64_t prepared_transition_blocked_page; int64_t page_busy_blocked; int64_t page_forcible_evict_blocked; @@ -791,7 +791,6 @@ struct __wt_connection_stats { int64_t txn_pinned_timestamp_oldest; int64_t txn_timestamp_oldest_active_read; int64_t txn_rollback_to_stable_running; - int64_t txn_sync; int64_t txn_walk_sessions; int64_t txn_commit; int64_t txn_rollback; diff --git a/src/third_party/wiredtiger/src/include/time_inline.h b/src/third_party/wiredtiger/src/include/time_inline.h index 5b7c28da5fb..611e38eab24 100644 --- a/src/third_party/wiredtiger/src/include/time_inline.h +++ b/src/third_party/wiredtiger/src/include/time_inline.h @@ -29,6 +29,13 @@ __wt_rdtsc(void) __asm__ volatile("rdtsc" : "=a"(a), "=d"(d)); return ((d << 32) | a); } +#elif defined(__aarch64__) + { + uint64_t t; + + __asm__ volatile("mrs %0, cntvct_el0" : "=r"(t)); + return (t); + } #else return (0); #endif diff --git a/src/third_party/wiredtiger/src/include/txn_inline.h b/src/third_party/wiredtiger/src/include/txn_inline.h index 97fea21a14c..df81b29fe23 100644 --- a/src/third_party/wiredtiger/src/include/txn_inline.h +++ b/src/third_party/wiredtiger/src/include/txn_inline.h @@ -973,8 +973,8 @@ __wt_txn_read_upd_list(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE * function will search the history store for a visible update. */ static inline int -__wt_txn_read(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint64_t recno, - WT_UPDATE *upd, WT_CELL_UNPACK_KV *vpack) +__wt_txn_read( + WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint64_t recno, WT_UPDATE *upd) { WT_TIME_WINDOW tw; WT_UPDATE *prepare_upd, *restored_upd; @@ -1013,14 +1013,7 @@ retry: have_stop_tw = WT_TIME_WINDOW_HAS_STOP(&cbt->upd_value->tw); /* Check the ondisk value. */ - if (vpack == NULL) { - WT_TIME_WINDOW_INIT(&tw); - WT_RET(__wt_value_return_buf(cbt, cbt->ref, &cbt->upd_value->buf, &tw)); - } else { - WT_TIME_WINDOW_COPY(&tw, &vpack->tw); - cbt->upd_value->buf.data = vpack->data; - cbt->upd_value->buf.size = vpack->size; - } + WT_RET(__wt_value_return_buf(cbt, cbt->ref, &cbt->upd_value->buf, &tw)); /* * If the stop time point is set, that means that there is a tombstone at that time. If it diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 028bcddbadb..2da0d297fb1 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -815,6 +815,9 @@ struct __wt_session { * internal thread. The \c on setting causes the caller to wait until all work queued for * this call to be completely processed before returning., a string\, chosen from the * following options: \c "off"\, \c "on"; default \c on.} + * @config{timeout, maximum amount of time to allow for waiting for previous flushing of + * objects\, in seconds. The actual amount of time spent waiting may exceed the configured + * value. A value of zero disables the timeout., an integer; default \c 0.} * @configend * @errors */ @@ -1428,11 +1431,9 @@ struct __wt_session { * @param session the session handle * @configstart{WT_SESSION.log_flush, see dist/api_data.py} * @config{sync, forcibly flush the log and wait for it to achieve the synchronization level - * specified. The \c background setting initiates a background synchronization intended to - * be used with a later call to WT_SESSION::transaction_sync. The \c off setting forces any - * buffered log records to be written to the file system. The \c on setting forces log - * records to be written to the storage device., a string\, chosen from the following - * options: \c "background"\, \c "off"\, \c "on"; default \c on.} + * specified. The \c off setting forces any buffered log records to be written to the file + * system. The \c on setting forces log records to be written to the storage device., a + * string\, chosen from the following options: \c "off"\, \c "on"; default \c on.} * @configend * @errors */ @@ -1716,12 +1717,10 @@ struct __wt_session { * operation may return a WT_ROLLBACK error. Default is to have no limit., an integer * greater than or equal to 1; default \c 0.} * @config{sync, override whether to sync log records when the transaction commits\, - * inherited from ::wiredtiger_open \c transaction_sync. The \c background setting - * initiates a background synchronization intended to be used with a later call to - * WT_SESSION::transaction_sync. The \c off setting does not wait for record to be written - * or synchronized. The \c on setting forces log records to be written to the storage - * device., a string\, chosen from the following options: \c "background"\, \c "off"\, \c - * "on"; default empty.} + * inherited from ::wiredtiger_open \c transaction_sync. The \c off setting does not wait + * for record to be written or synchronized. The \c on setting forces log records to be + * written to the storage device., a string\, chosen from the following options: \c "off"\, + * \c "on"; default empty.} * @configend * @errors */ @@ -1913,26 +1912,6 @@ struct __wt_session { */ int __F(transaction_pinned_range)(WT_SESSION* session, uint64_t *range); - /*! - * Wait for a transaction to become synchronized. This method is - * only useful when ::wiredtiger_open is configured with the - * \c transaction_sync setting disabled. - * - * @requires_notransaction - * - * @snippet ex_all.c Transaction sync - * - * @param session the session handle - * @configstart{WT_SESSION.transaction_sync, see dist/api_data.py} - * @config{timeout_ms, maximum amount of time to wait for background sync to complete in - * milliseconds. A value of zero disables the timeout and returns immediately., an integer; - * default \c 1200000.} - * @configend - * @errors - */ - int __F(transaction_sync)(WT_SESSION *session, const char *config); - /*! @} */ - #ifndef DOXYGEN /*! * Call into the library. @@ -5200,774 +5179,777 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); /*! cache: forced eviction - pages evicted that were dirty time (usecs) */ #define WT_STAT_CONN_CACHE_EVICTION_FORCE_DIRTY_TIME 1083 /*! + * cache: forced eviction - pages selected because of a large number of + * updates to a single item + */ +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_LONG_UPDATE_LIST 1084 +/*! * cache: forced eviction - pages selected because of too many deleted * items count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1084 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1085 /*! cache: forced eviction - pages selected count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1085 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1086 /*! cache: forced eviction - pages selected unable to be evicted count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1086 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1087 /*! cache: forced eviction - pages selected unable to be evicted time */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL_TIME 1087 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL_TIME 1088 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1088 +#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1089 /*! cache: hazard pointer check calls */ -#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1089 +#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1090 /*! cache: hazard pointer check entries walked */ -#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1090 +#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1091 /*! cache: hazard pointer maximum array length */ -#define WT_STAT_CONN_CACHE_HAZARD_MAX 1091 +#define WT_STAT_CONN_CACHE_HAZARD_MAX 1092 /*! cache: history store score */ -#define WT_STAT_CONN_CACHE_HS_SCORE 1092 +#define WT_STAT_CONN_CACHE_HS_SCORE 1093 /*! cache: history store table insert calls */ -#define WT_STAT_CONN_CACHE_HS_INSERT 1093 +#define WT_STAT_CONN_CACHE_HS_INSERT 1094 /*! cache: history store table insert calls that returned restart */ -#define WT_STAT_CONN_CACHE_HS_INSERT_RESTART 1094 +#define WT_STAT_CONN_CACHE_HS_INSERT_RESTART 1095 /*! cache: history store table max on-disk size */ -#define WT_STAT_CONN_CACHE_HS_ONDISK_MAX 1095 +#define WT_STAT_CONN_CACHE_HS_ONDISK_MAX 1096 /*! cache: history store table on-disk size */ -#define WT_STAT_CONN_CACHE_HS_ONDISK 1096 +#define WT_STAT_CONN_CACHE_HS_ONDISK 1097 /*! * cache: history store table out-of-order resolved updates that lose * their durable timestamp */ -#define WT_STAT_CONN_CACHE_HS_ORDER_LOSE_DURABLE_TIMESTAMP 1097 +#define WT_STAT_CONN_CACHE_HS_ORDER_LOSE_DURABLE_TIMESTAMP 1098 /*! * cache: history store table out-of-order updates that were fixed up by * reinserting with the fixed timestamp */ -#define WT_STAT_CONN_CACHE_HS_ORDER_REINSERT 1098 +#define WT_STAT_CONN_CACHE_HS_ORDER_REINSERT 1099 /*! cache: history store table reads */ -#define WT_STAT_CONN_CACHE_HS_READ 1099 +#define WT_STAT_CONN_CACHE_HS_READ 1100 /*! cache: history store table reads missed */ -#define WT_STAT_CONN_CACHE_HS_READ_MISS 1100 +#define WT_STAT_CONN_CACHE_HS_READ_MISS 1101 /*! cache: history store table reads requiring squashed modifies */ -#define WT_STAT_CONN_CACHE_HS_READ_SQUASH 1101 +#define WT_STAT_CONN_CACHE_HS_READ_SQUASH 1102 /*! * cache: history store table truncation by rollback to stable to remove * an unstable update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS_UNSTABLE 1102 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS_UNSTABLE 1103 /*! * cache: history store table truncation by rollback to stable to remove * an update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS 1103 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS 1104 /*! cache: history store table truncation to remove an update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE 1104 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE 1105 /*! * cache: history store table truncation to remove range of updates due * to key being removed from the data page during reconciliation */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 1105 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 1106 /*! * cache: history store table truncation to remove range of updates due * to out-of-order timestamp update on data page */ -#define WT_STAT_CONN_CACHE_HS_ORDER_REMOVE 1106 +#define WT_STAT_CONN_CACHE_HS_ORDER_REMOVE 1107 /*! cache: history store table writes requiring squashed modifies */ -#define WT_STAT_CONN_CACHE_HS_WRITE_SQUASH 1107 +#define WT_STAT_CONN_CACHE_HS_WRITE_SQUASH 1108 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1108 +#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1109 /*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1109 +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1110 /*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1110 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1111 /*! cache: internal pages queued for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_QUEUED 1111 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_QUEUED 1112 /*! cache: internal pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_SEEN 1112 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_SEEN 1113 /*! cache: internal pages seen by eviction walk that are already queued */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_ALREADY_QUEUED 1113 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_ALREADY_QUEUED 1114 /*! cache: internal pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1114 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1115 /*! cache: leaf pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1115 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1116 /*! cache: maximum bytes configured */ -#define WT_STAT_CONN_CACHE_BYTES_MAX 1116 +#define WT_STAT_CONN_CACHE_BYTES_MAX 1117 /*! cache: maximum page size at eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1117 +#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1118 /*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1118 +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1119 /*! cache: modified pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1119 +#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1120 /*! cache: operations timed out waiting for space in cache */ -#define WT_STAT_CONN_CACHE_TIMED_OUT_OPS 1120 +#define WT_STAT_CONN_CACHE_TIMED_OUT_OPS 1121 /*! cache: overflow pages read into cache */ -#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1121 +#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1122 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1122 +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1123 /*! cache: page written requiring history store records */ -#define WT_STAT_CONN_CACHE_WRITE_HS 1123 +#define WT_STAT_CONN_CACHE_WRITE_HS 1124 /*! cache: pages currently held in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_INUSE 1124 +#define WT_STAT_CONN_CACHE_PAGES_INUSE 1125 /*! cache: pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP 1125 +#define WT_STAT_CONN_CACHE_EVICTION_APP 1126 /*! cache: pages evicted in parallel with checkpoint */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_IN_PARALLEL_WITH_CHECKPOINT 1126 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_IN_PARALLEL_WITH_CHECKPOINT 1127 /*! cache: pages queued for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1127 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1128 /*! cache: pages queued for eviction post lru sorting */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_POST_LRU 1128 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_POST_LRU 1129 /*! cache: pages queued for urgent eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1129 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1130 /*! cache: pages queued for urgent eviction during walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1130 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1131 /*! * cache: pages queued for urgent eviction from history store due to high * dirty content */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT_HS_DIRTY 1131 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT_HS_DIRTY 1132 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1132 +#define WT_STAT_CONN_CACHE_READ 1133 /*! cache: pages read into cache after truncate */ -#define WT_STAT_CONN_CACHE_READ_DELETED 1133 +#define WT_STAT_CONN_CACHE_READ_DELETED 1134 /*! cache: pages read into cache after truncate in prepare state */ -#define WT_STAT_CONN_CACHE_READ_DELETED_PREPARED 1134 +#define WT_STAT_CONN_CACHE_READ_DELETED_PREPARED 1135 /*! cache: pages requested from the cache */ -#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1135 +#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1136 /*! cache: pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1136 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1137 /*! cache: pages seen by eviction walk that are already queued */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_ALREADY_QUEUED 1137 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_ALREADY_QUEUED 1138 /*! cache: pages selected for eviction unable to be evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1138 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1139 /*! * cache: pages selected for eviction unable to be evicted as the parent * page has overflow items */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_PARENT_HAS_OVERFLOW_ITEMS 1139 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_PARENT_HAS_OVERFLOW_ITEMS 1140 /*! * cache: pages selected for eviction unable to be evicted because of * active children on an internal page */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_ACTIVE_CHILDREN_ON_AN_INTERNAL_PAGE 1140 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_ACTIVE_CHILDREN_ON_AN_INTERNAL_PAGE 1141 /*! * cache: pages selected for eviction unable to be evicted because of * failure in reconciliation */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_IN_RECONCILIATION 1141 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_IN_RECONCILIATION 1142 /*! * cache: pages selected for eviction unable to be evicted because of * race between checkpoint and out of order timestamps handling */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_CHECKPOINT_OUT_OF_ORDER_TS 1142 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_CHECKPOINT_OUT_OF_ORDER_TS 1143 /*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1143 +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1144 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1144 +#define WT_STAT_CONN_CACHE_WRITE 1145 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1145 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1146 /*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1146 +#define WT_STAT_CONN_CACHE_OVERHEAD 1147 /*! cache: tracked bytes belonging to internal pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1147 +#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1148 /*! cache: tracked bytes belonging to leaf pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_LEAF 1148 +#define WT_STAT_CONN_CACHE_BYTES_LEAF 1149 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1149 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1150 /*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1150 +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1151 /*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1151 +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1152 /*! capacity: background fsync file handles considered */ -#define WT_STAT_CONN_FSYNC_ALL_FH_TOTAL 1152 +#define WT_STAT_CONN_FSYNC_ALL_FH_TOTAL 1153 /*! capacity: background fsync file handles synced */ -#define WT_STAT_CONN_FSYNC_ALL_FH 1153 +#define WT_STAT_CONN_FSYNC_ALL_FH 1154 /*! capacity: background fsync time (msecs) */ -#define WT_STAT_CONN_FSYNC_ALL_TIME 1154 +#define WT_STAT_CONN_FSYNC_ALL_TIME 1155 /*! capacity: bytes read */ -#define WT_STAT_CONN_CAPACITY_BYTES_READ 1155 +#define WT_STAT_CONN_CAPACITY_BYTES_READ 1156 /*! capacity: bytes written for checkpoint */ -#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1156 +#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1157 /*! capacity: bytes written for eviction */ -#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1157 +#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1158 /*! capacity: bytes written for log */ -#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1158 +#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1159 /*! capacity: bytes written total */ -#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1159 +#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1160 /*! capacity: threshold to call fsync */ -#define WT_STAT_CONN_CAPACITY_THRESHOLD 1160 +#define WT_STAT_CONN_CAPACITY_THRESHOLD 1161 /*! capacity: time waiting due to total capacity (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_TOTAL 1161 +#define WT_STAT_CONN_CAPACITY_TIME_TOTAL 1162 /*! capacity: time waiting during checkpoint (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_CKPT 1162 +#define WT_STAT_CONN_CAPACITY_TIME_CKPT 1163 /*! capacity: time waiting during eviction (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_EVICT 1163 +#define WT_STAT_CONN_CAPACITY_TIME_EVICT 1164 /*! capacity: time waiting during logging (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_LOG 1164 +#define WT_STAT_CONN_CAPACITY_TIME_LOG 1165 /*! capacity: time waiting during read (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_READ 1165 +#define WT_STAT_CONN_CAPACITY_TIME_READ 1166 /*! checkpoint-cleanup: pages added for eviction */ -#define WT_STAT_CONN_CC_PAGES_EVICT 1166 +#define WT_STAT_CONN_CC_PAGES_EVICT 1167 /*! checkpoint-cleanup: pages removed */ -#define WT_STAT_CONN_CC_PAGES_REMOVED 1167 +#define WT_STAT_CONN_CC_PAGES_REMOVED 1168 /*! checkpoint-cleanup: pages skipped during tree walk */ -#define WT_STAT_CONN_CC_PAGES_WALK_SKIPPED 1168 +#define WT_STAT_CONN_CC_PAGES_WALK_SKIPPED 1169 /*! checkpoint-cleanup: pages visited */ -#define WT_STAT_CONN_CC_PAGES_VISITED 1169 +#define WT_STAT_CONN_CC_PAGES_VISITED 1170 /*! connection: auto adjusting condition resets */ -#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1170 +#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1171 /*! connection: auto adjusting condition wait calls */ -#define WT_STAT_CONN_COND_AUTO_WAIT 1171 +#define WT_STAT_CONN_COND_AUTO_WAIT 1172 /*! * connection: auto adjusting condition wait raced to update timeout and * skipped updating */ -#define WT_STAT_CONN_COND_AUTO_WAIT_SKIPPED 1172 +#define WT_STAT_CONN_COND_AUTO_WAIT_SKIPPED 1173 /*! connection: detected system time went backwards */ -#define WT_STAT_CONN_TIME_TRAVEL 1173 +#define WT_STAT_CONN_TIME_TRAVEL 1174 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1174 +#define WT_STAT_CONN_FILE_OPEN 1175 /*! connection: hash bucket array size for data handles */ -#define WT_STAT_CONN_BUCKETS_DH 1175 +#define WT_STAT_CONN_BUCKETS_DH 1176 /*! connection: hash bucket array size general */ -#define WT_STAT_CONN_BUCKETS 1176 +#define WT_STAT_CONN_BUCKETS 1177 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1177 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1178 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1178 +#define WT_STAT_CONN_MEMORY_FREE 1179 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1179 +#define WT_STAT_CONN_MEMORY_GROW 1180 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1180 +#define WT_STAT_CONN_COND_WAIT 1181 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1181 +#define WT_STAT_CONN_RWLOCK_READ 1182 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1182 +#define WT_STAT_CONN_RWLOCK_WRITE 1183 /*! connection: total fsync I/Os */ -#define WT_STAT_CONN_FSYNC_IO 1183 +#define WT_STAT_CONN_FSYNC_IO 1184 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1184 +#define WT_STAT_CONN_READ_IO 1185 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1185 +#define WT_STAT_CONN_WRITE_IO 1186 /*! cursor: Total number of entries skipped by cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_TOTAL 1186 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_TOTAL 1187 /*! cursor: Total number of entries skipped by cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_TOTAL 1187 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_TOTAL 1188 /*! * cursor: Total number of entries skipped to position the history store * cursor */ -#define WT_STAT_CONN_CURSOR_SKIP_HS_CUR_POSITION 1188 +#define WT_STAT_CONN_CURSOR_SKIP_HS_CUR_POSITION 1189 /*! * cursor: Total number of pages skipped without reading by cursor next * calls */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_PAGE_COUNT 1189 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_PAGE_COUNT 1190 /*! * cursor: Total number of pages skipped without reading by cursor prev * calls */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_PAGE_COUNT 1190 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_PAGE_COUNT 1191 /*! * cursor: Total number of times a search near has exited due to prefix * config */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS 1191 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS 1192 /*! cursor: cached cursor count */ -#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1192 +#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1193 /*! cursor: cursor bulk loaded cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT_BULK 1193 +#define WT_STAT_CONN_CURSOR_INSERT_BULK 1194 /*! cursor: cursor close calls that result in cache */ -#define WT_STAT_CONN_CURSOR_CACHE 1194 +#define WT_STAT_CONN_CURSOR_CACHE 1195 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1195 +#define WT_STAT_CONN_CURSOR_CREATE 1196 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1196 +#define WT_STAT_CONN_CURSOR_INSERT 1197 /*! cursor: cursor insert key and value bytes */ -#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1197 +#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1198 /*! cursor: cursor modify calls */ -#define WT_STAT_CONN_CURSOR_MODIFY 1198 +#define WT_STAT_CONN_CURSOR_MODIFY 1199 /*! cursor: cursor modify key and value bytes affected */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1199 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1200 /*! cursor: cursor modify value bytes modified */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1200 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1201 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1201 +#define WT_STAT_CONN_CURSOR_NEXT 1202 /*! * cursor: cursor next calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE 1202 +#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE 1203 /*! * cursor: cursor next calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1203 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1204 /*! cursor: cursor next calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1204 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1205 /*! cursor: cursor operation restarted */ -#define WT_STAT_CONN_CURSOR_RESTART 1205 +#define WT_STAT_CONN_CURSOR_RESTART 1206 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1206 +#define WT_STAT_CONN_CURSOR_PREV 1207 /*! * cursor: cursor prev calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1207 +#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1208 /*! * cursor: cursor prev calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1208 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1209 /*! cursor: cursor prev calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1209 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1210 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1210 +#define WT_STAT_CONN_CURSOR_REMOVE 1211 /*! cursor: cursor remove key bytes removed */ -#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1211 +#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1212 /*! cursor: cursor reserve calls */ -#define WT_STAT_CONN_CURSOR_RESERVE 1212 +#define WT_STAT_CONN_CURSOR_RESERVE 1213 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1213 +#define WT_STAT_CONN_CURSOR_RESET 1214 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1214 +#define WT_STAT_CONN_CURSOR_SEARCH 1215 /*! cursor: cursor search history store calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_HS 1215 +#define WT_STAT_CONN_CURSOR_SEARCH_HS 1216 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1216 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1217 /*! cursor: cursor sweep buckets */ -#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1217 +#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1218 /*! cursor: cursor sweep cursors closed */ -#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1218 +#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1219 /*! cursor: cursor sweep cursors examined */ -#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1219 +#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1220 /*! cursor: cursor sweeps */ -#define WT_STAT_CONN_CURSOR_SWEEP 1220 +#define WT_STAT_CONN_CURSOR_SWEEP 1221 /*! cursor: cursor truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1221 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1222 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1222 +#define WT_STAT_CONN_CURSOR_UPDATE 1223 /*! cursor: cursor update key and value bytes */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1223 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1224 /*! cursor: cursor update value size change */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1224 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1225 /*! cursor: cursors reused from cache */ -#define WT_STAT_CONN_CURSOR_REOPEN 1225 +#define WT_STAT_CONN_CURSOR_REOPEN 1226 /*! cursor: open cursor count */ -#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1226 +#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1227 /*! data-handle: connection data handle size */ -#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1227 +#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1228 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1228 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1229 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1229 +#define WT_STAT_CONN_DH_SWEEP_REF 1230 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1230 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1231 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1231 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1232 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1232 +#define WT_STAT_CONN_DH_SWEEP_TOD 1233 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1233 +#define WT_STAT_CONN_DH_SWEEPS 1234 /*! * data-handle: connection sweeps skipped due to checkpoint gathering * handles */ -#define WT_STAT_CONN_DH_SWEEP_SKIP_CKPT 1234 +#define WT_STAT_CONN_DH_SWEEP_SKIP_CKPT 1235 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1235 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1236 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1236 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1237 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1237 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1238 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1238 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1239 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1239 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1240 /*! lock: dhandle lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1240 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1241 /*! lock: dhandle lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1241 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1242 /*! lock: dhandle read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1242 +#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1243 /*! lock: dhandle write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1243 +#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1244 /*! * lock: durable timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1244 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1245 /*! * lock: durable timestamp queue lock internal thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1245 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1246 /*! lock: durable timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1246 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1247 /*! lock: durable timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1247 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1248 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1248 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1249 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1249 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1250 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1250 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1251 /*! * lock: read timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1251 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1252 /*! lock: read timestamp queue lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1252 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1253 /*! lock: read timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1253 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1254 /*! lock: read timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1254 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1255 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1255 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1256 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1256 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1257 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1257 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1258 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1258 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1259 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1259 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1260 /*! lock: table read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1260 +#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1261 /*! lock: table write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1261 +#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1262 /*! lock: txn global lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1262 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1263 /*! lock: txn global lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1263 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1264 /*! lock: txn global read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1264 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1265 /*! lock: txn global write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1265 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1266 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1266 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1267 /*! log: force archive time sleeping (usecs) */ -#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1267 +#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1268 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1268 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1269 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1269 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1270 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1270 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1271 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1271 +#define WT_STAT_CONN_LOG_FLUSH 1272 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1272 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1273 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1273 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1274 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1274 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1275 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1275 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1276 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1276 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1277 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1277 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1278 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1278 +#define WT_STAT_CONN_LOG_SCANS 1279 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1279 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1280 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1280 +#define WT_STAT_CONN_LOG_WRITE_LSN 1281 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1281 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1282 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1282 +#define WT_STAT_CONN_LOG_SYNC 1283 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1283 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1284 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1284 +#define WT_STAT_CONN_LOG_SYNC_DIR 1285 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1285 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1286 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1286 +#define WT_STAT_CONN_LOG_WRITES 1287 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1287 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1288 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1288 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1289 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1289 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1290 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1290 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1291 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1291 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1292 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1292 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1293 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1293 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1294 /*! log: slot close lost race */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1294 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1295 /*! log: slot close unbuffered waits */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1295 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1296 /*! log: slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1296 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1297 /*! log: slot join atomic update races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1297 +#define WT_STAT_CONN_LOG_SLOT_RACES 1298 /*! log: slot join calls atomic updates raced */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1298 +#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1299 /*! log: slot join calls did not yield */ -#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1299 +#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1300 /*! log: slot join calls found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1300 +#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1301 /*! log: slot join calls slept */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1301 +#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1302 /*! log: slot join calls yielded */ -#define WT_STAT_CONN_LOG_SLOT_YIELD 1302 +#define WT_STAT_CONN_LOG_SLOT_YIELD 1303 /*! log: slot join found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1303 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1304 /*! log: slot joins yield time (usecs) */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1304 +#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1305 /*! log: slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1305 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1306 /*! log: slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1306 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1307 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1307 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1308 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1308 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1309 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1309 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1310 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1310 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1311 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1311 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1312 /*! perf: file system read latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1312 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1313 /*! perf: file system read latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1313 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1314 /*! perf: file system read latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1314 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1315 /*! perf: file system read latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1315 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1316 /*! perf: file system read latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1316 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1317 /*! perf: file system read latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1317 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1318 /*! perf: file system write latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1318 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1319 /*! perf: file system write latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1319 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1320 /*! perf: file system write latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1320 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1321 /*! perf: file system write latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1321 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1322 /*! perf: file system write latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1322 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1323 /*! perf: file system write latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1323 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1324 /*! perf: operation read latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1324 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1325 /*! perf: operation read latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1325 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1326 /*! perf: operation read latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1326 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1327 /*! perf: operation read latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1327 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1328 /*! perf: operation read latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1328 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1329 /*! perf: operation write latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1329 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1330 /*! perf: operation write latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1330 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1331 /*! perf: operation write latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1331 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1332 /*! perf: operation write latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1332 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1333 /*! perf: operation write latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1333 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1334 /*! reconciliation: approximate byte size of timestamps in pages written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1334 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1335 /*! * reconciliation: approximate byte size of transaction IDs in pages * written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1335 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1336 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1336 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1337 /*! reconciliation: internal-page overflow keys */ -#define WT_STAT_CONN_REC_OVERFLOW_KEY_INTERNAL 1337 +#define WT_STAT_CONN_REC_OVERFLOW_KEY_INTERNAL 1338 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_CONN_REC_OVERFLOW_KEY_LEAF 1338 +#define WT_STAT_CONN_REC_OVERFLOW_KEY_LEAF 1339 /*! reconciliation: maximum seconds spent in a reconciliation call */ -#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1339 +#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1340 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1340 +#define WT_STAT_CONN_REC_PAGES 1341 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1341 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1342 /*! * reconciliation: page reconciliation calls that resulted in values with * prepared transaction metadata */ -#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1342 +#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1343 /*! * reconciliation: page reconciliation calls that resulted in values with * timestamps */ -#define WT_STAT_CONN_REC_PAGES_WITH_TS 1343 +#define WT_STAT_CONN_REC_PAGES_WITH_TS 1344 /*! * reconciliation: page reconciliation calls that resulted in values with * transaction ids */ -#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1344 +#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1345 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1345 +#define WT_STAT_CONN_REC_PAGE_DELETE 1346 /*! * reconciliation: pages written including an aggregated newest start * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1346 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1347 /*! * reconciliation: pages written including an aggregated newest stop * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1347 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1348 /*! * reconciliation: pages written including an aggregated newest stop * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1348 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1349 /*! * reconciliation: pages written including an aggregated newest stop * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1349 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1350 /*! * reconciliation: pages written including an aggregated newest * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_TXN 1350 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_TXN 1351 /*! * reconciliation: pages written including an aggregated oldest start * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1351 +#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1352 /*! reconciliation: pages written including an aggregated prepare */ -#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1352 +#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1353 /*! reconciliation: pages written including at least one prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1353 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1354 /*! * reconciliation: pages written including at least one start durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1354 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1355 /*! reconciliation: pages written including at least one start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1355 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1356 /*! * reconciliation: pages written including at least one start transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1356 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1357 /*! * reconciliation: pages written including at least one stop durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1357 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1358 /*! reconciliation: pages written including at least one stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1358 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1359 /*! * reconciliation: pages written including at least one stop transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1359 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1360 /*! reconciliation: records written including a prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1360 +#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1361 /*! reconciliation: records written including a start durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1361 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1362 /*! reconciliation: records written including a start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1362 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1363 /*! reconciliation: records written including a start transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1363 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1364 /*! reconciliation: records written including a stop durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1364 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1365 /*! reconciliation: records written including a stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1365 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1366 /*! reconciliation: records written including a stop transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1366 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1367 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1367 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1368 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1368 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1369 /*! session: flush state races */ -#define WT_STAT_CONN_FLUSH_STATE_RACES 1369 +#define WT_STAT_CONN_FLUSH_STATE_RACES 1370 /*! session: flush_tier operation calls */ -#define WT_STAT_CONN_FLUSH_TIER 1370 +#define WT_STAT_CONN_FLUSH_TIER 1371 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1371 +#define WT_STAT_CONN_SESSION_OPEN 1372 /*! session: session query timestamp calls */ -#define WT_STAT_CONN_SESSION_QUERY_TS 1372 +#define WT_STAT_CONN_SESSION_QUERY_TS 1373 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1373 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1374 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1374 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1375 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1375 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1376 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1376 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1377 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1377 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1378 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1378 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1379 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1379 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1380 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1380 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1381 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1381 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1382 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1382 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1383 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1383 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1384 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1384 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1385 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1385 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1386 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1386 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1387 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1387 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1388 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1388 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1389 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1389 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1390 /*! session: tiered operations dequeued and processed */ -#define WT_STAT_CONN_TIERED_WORK_UNITS_DEQUEUED 1390 +#define WT_STAT_CONN_TIERED_WORK_UNITS_DEQUEUED 1391 /*! session: tiered operations scheduled */ -#define WT_STAT_CONN_TIERED_WORK_UNITS_CREATED 1391 +#define WT_STAT_CONN_TIERED_WORK_UNITS_CREATED 1392 /*! session: tiered storage local retention time (secs) */ -#define WT_STAT_CONN_TIERED_RETENTION 1392 +#define WT_STAT_CONN_TIERED_RETENTION 1393 /*! session: tiered storage object size */ -#define WT_STAT_CONN_TIERED_OBJECT_SIZE 1393 +#define WT_STAT_CONN_TIERED_OBJECT_SIZE 1394 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1394 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1395 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1395 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1396 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1396 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1397 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1397 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1398 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1398 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1399 /*! * thread-yield: connection close blocked waiting for transaction state * stabilization */ -#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1399 +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1400 /*! thread-yield: connection close yielded for lsm manager shutdown */ -#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1400 +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1401 /*! thread-yield: data handle lock yielded */ -#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1401 +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1402 /*! * thread-yield: get reference for page index and slot time sleeping * (usecs) */ -#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1402 -/*! thread-yield: log server sync yielded for log write */ -#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1403 +#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1403 /*! thread-yield: page access yielded due to prepare state change */ #define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1404 /*! thread-yield: page acquire busy blocked */ @@ -6152,16 +6134,14 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1476 /*! transaction: transaction rollback to stable currently running */ #define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1477 -/*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1478 /*! transaction: transaction walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_WALK_SESSIONS 1479 +#define WT_STAT_CONN_TXN_WALK_SESSIONS 1478 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1480 +#define WT_STAT_CONN_TXN_COMMIT 1479 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1481 +#define WT_STAT_CONN_TXN_ROLLBACK 1480 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1482 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1481 /*! * @} @@ -6738,14 +6718,14 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); * reconciliation: pages written including at least one start transaction * ID */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TXN 2187 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TXN 2189 /*! * reconciliation: pages written including at least one stop durable * timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 2188 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 2190 /*! reconciliation: pages written including at least one stop timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TS 2189 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TS 2191 /*! * reconciliation: pages written including at least one stop transaction * ID diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index bfa252ce4b8..6b136c6cc84 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -269,35 +269,6 @@ __wt_log_flush_lsn(WT_SESSION_IMPL *session, WT_LSN *lsn, bool start) } /* - * __wt_log_background -- - * Record the given LSN as the background LSN and signal the thread as needed. - */ -void -__wt_log_background(WT_SESSION_IMPL *session, WT_LSN *lsn) -{ - WT_CONNECTION_IMPL *conn; - WT_LOG *log; - - conn = S2C(session); - log = conn->log; - /* - * If a thread already set the LSN to a bigger LSN, we're done. - */ - if (__wt_log_cmp(&session->bg_sync_lsn, lsn) > 0) - return; - WT_ASSIGN_LSN(&session->bg_sync_lsn, lsn); - - /* - * Advance the logging subsystem background sync LSN if needed. - */ - __wt_spin_lock(session, &log->log_sync_lock); - if (__wt_log_cmp(lsn, &log->bg_sync_lsn) > 0) - WT_ASSIGN_LSN(&log->bg_sync_lsn, lsn); - __wt_spin_unlock(session, &log->log_sync_lock); - __wt_cond_signal(session, conn->log_file_cond); -} - -/* * __wt_log_force_sync -- * Force a sync of the log and files. */ @@ -2715,12 +2686,6 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, ui __wt_cond_wait(session, log->log_sync_cond, 10000, NULL); } - /* - * Advance the background sync LSN if needed. - */ - if (LF_ISSET(WT_LOG_BACKGROUND)) - __wt_log_background(session, &lsn); - err: if (ret == 0 && lsnp != NULL) *lsnp = lsn; @@ -2836,9 +2801,7 @@ __wt_log_flush(WT_SESSION_IMPL *session, uint32_t flags) * If the user wants write-no-sync, there is nothing more to do. If the user wants background * sync, set the LSN and we're done. If the user wants sync, force it now. */ - if (LF_ISSET(WT_LOG_BACKGROUND)) - __wt_log_background(session, &lsn); - else if (LF_ISSET(WT_LOG_FSYNC)) + if (LF_ISSET(WT_LOG_FSYNC)) WT_RET(__wt_log_force_sync(session, &lsn)); return (0); } diff --git a/src/third_party/wiredtiger/src/log/log_sys.c b/src/third_party/wiredtiger/src/log/log_sys.c index a922cb9b56a..56a831f7275 100644 --- a/src/third_party/wiredtiger/src/log/log_sys.c +++ b/src/third_party/wiredtiger/src/log/log_sys.c @@ -128,7 +128,6 @@ __wt_verbose_dump_log(WT_SESSION_IMPL *session) WT_RET(__wt_msg(session, "Current log file number: %" PRIu32, log->fileid)); WT_RET(__wt_msg(session, "Current log version number: %" PRIu16, log->log_version)); WT_RET(WT_LSN_MSG(&log->alloc_lsn, "Next allocation")); - WT_RET(WT_LSN_MSG(&log->bg_sync_lsn, "Last background sync")); WT_RET(WT_LSN_MSG(&log->ckpt_lsn, "Last checkpoint")); WT_RET(WT_LSN_MSG(&log->sync_dir_lsn, "Last directory sync")); WT_RET(WT_LSN_MSG(&log->sync_lsn, "Last sync")); diff --git a/src/third_party/wiredtiger/src/reconcile/rec_col.c b/src/third_party/wiredtiger/src/reconcile/rec_col.c index 417bafebb50..9518a3b63c3 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_col.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_col.c @@ -573,17 +573,20 @@ __wt_rec_col_var( WT_CELL *cell; WT_CELL_UNPACK_KV *vpack, _vpack; WT_COL *cip; + WT_CURSOR *hs_cursor; WT_CURSOR_BTREE *cbt; WT_DECL_ITEM(orig); WT_DECL_RET; WT_INSERT *ins; + WT_ITEM hs_recno_key; WT_PAGE *page; WT_TIME_WINDOW clear_tw, *twp; WT_UPDATE *upd; WT_UPDATE_SELECT upd_select; uint64_t n, nrepeat, repeat_count, rle, skip, src_recno; uint32_t i, size; - bool deleted, orig_deleted, update_no_copy; + uint8_t *p, hs_recno_key_buf[WT_INTPACK64_MAXSIZE]; + bool deleted, hs_clear, orig_deleted, update_no_copy; const void *data; btree = S2BT(session); @@ -603,6 +606,23 @@ __wt_rec_col_var( WT_TIME_WINDOW_INIT(&last.tw); last.deleted = false; + /* + * When removing a key due to a tombstone with a durable timestamp of "none", also remove the + * history store contents associated with that key. It's safe to do even if we fail + * reconciliation after the removal, the history store content must be obsolete in order for us + * to consider removing the key. + * + * Ignore if this is metadata, as metadata doesn't have any history. + * + * Some code paths, such as schema removal, involve deleting keys in metadata and assert that + * they shouldn't open new dhandles. In those cases we won't ever need to blow away history + * store content, so we can skip this. + */ + hs_cursor = NULL; + hs_clear = F_ISSET(S2C(session), WT_CONN_HS_OPEN) && + !F_ISSET(session, WT_SESSION_NO_DATA_HANDLES) && !WT_IS_HS(btree->dhandle) && + !WT_IS_METADATA(btree->dhandle); + WT_RET(__wt_rec_split_init(session, r, page, pageref->ref_recno, btree->maxleafpage_precomp)); WT_RET(__wt_scr_alloc(session, 0, &orig)); @@ -789,8 +809,30 @@ record_loop: size = upd->size; break; case WT_UPDATE_TOMBSTONE: - twp = &clear_tw; + /* + * When removing a key due to a tombstone with a durable timestamp of "none", + * also remove the history store contents associated with that key. + */ + if (twp->durable_stop_ts == WT_TS_NONE && hs_clear) { + p = hs_recno_key_buf; + WT_ERR(__wt_vpack_uint(&p, 0, src_recno)); + hs_recno_key.data = hs_recno_key_buf; + hs_recno_key.size = WT_PTRDIFF(p, hs_recno_key_buf); + + /* Open a history store cursor if we don't yet have one. */ + if (hs_cursor == NULL) + WT_ERR(__wt_curhs_open(session, NULL, &hs_cursor)); + + /* From WT_TS_NONE to delete all the history store content of the key. */ + WT_ERR(__wt_hs_delete_key_from_ts(session, hs_cursor, btree->id, + &hs_recno_key, WT_TS_NONE, false, F_ISSET(r, WT_REC_CHECKPOINT_RUNNING))); + + WT_STAT_CONN_INCR(session, cache_hs_key_truncate_onpage_removal); + WT_STAT_DATA_INCR(session, cache_hs_key_truncate_onpage_removal); + } + deleted = true; + twp = &clear_tw; break; default: WT_ERR(__wt_illegal_value(session, upd->type)); @@ -799,19 +841,21 @@ record_loop: compare: /* - * If we have a record against which to compare, and the records compare equal, - * increment the rle counter and continue. If the records don't compare equal, output - * the last record and swap the last and current buffers: do NOT update the starting - * record number, we've been doing that all along. + * If we have a record against which to compare and the records compare equal, increment + * the RLE and continue. If the records don't compare equal, output the last record and + * swap the last and current buffers: do NOT update the starting record number, we've + * been doing that all along. */ if (rle != 0) { - if (WT_TIME_WINDOWS_EQUAL(twp, &last.tw) && + if (WT_TIME_WINDOWS_EQUAL(&last.tw, twp) && ((deleted && last.deleted) || (!deleted && !last.deleted && last.value->size == size && - memcmp(last.value->data, data, size) == 0))) { + (size == 0 || memcmp(last.value->data, data, size) == 0)))) { + /* The time window for deleted keys must be empty. */ WT_ASSERT( session, (!deleted && !last.deleted) || WT_TIME_WINDOW_IS_EMPTY(&last.tw)); + rle += repeat_count; continue; } @@ -951,13 +995,13 @@ compare: if (rle != 0) { if (WT_TIME_WINDOWS_EQUAL(&last.tw, twp) && ((deleted && last.deleted) || - (!deleted && !last.deleted && size != 0 && last.value->size == size && - memcmp(last.value->data, data, size) == 0))) { - /* - * The time window for deleted keys must be empty. - */ + (!deleted && !last.deleted && last.value->size == size && + (size == 0 || memcmp(last.value->data, data, size) == 0)))) { + + /* The time window for deleted keys must be empty. */ WT_ASSERT( session, (!deleted && !last.deleted) || WT_TIME_WINDOW_IS_EMPTY(&last.tw)); + ++rle; goto next; } @@ -1012,6 +1056,8 @@ next: ret = __wt_rec_split_finish(session, r); err: + if (hs_cursor != NULL) + WT_TRET(hs_cursor->close(hs_cursor)); __wt_scr_free(session, &orig); return (ret); } diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 513b466d0a9..62464af3c6a 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -203,8 +203,6 @@ __session_clear(WT_SESSION_IMPL *session) */ memset(session, 0, WT_SESSION_CLEAR_SIZE); - WT_INIT_LSN(&session->bg_sync_lsn); - session->hazard_inuse = 0; session->nhazard = 0; } @@ -808,9 +806,7 @@ __session_log_flush(WT_SESSION *wt_session, const char *config) WT_ERR_MSG(session, EINVAL, "logging not enabled"); WT_ERR(__wt_config_gets_def(session, cfg, "sync", 0, &cval)); - if (WT_STRING_MATCH("background", cval.str, cval.len)) - flags = WT_LOG_BACKGROUND; - else if (WT_STRING_MATCH("off", cval.str, cval.len)) + if (WT_STRING_MATCH("off", cval.str, cval.len)) flags = WT_LOG_FLUSH; else if (WT_STRING_MATCH("on", cval.str, cval.len)) flags = WT_LOG_FSYNC; @@ -1783,114 +1779,6 @@ err: } /* - * __transaction_sync_run_chk -- - * Check to decide if the transaction sync call should continue running. - */ -static bool -__transaction_sync_run_chk(WT_SESSION_IMPL *session) -{ - WT_CONNECTION_IMPL *conn; - - conn = S2C(session); - - return (FLD_ISSET(conn->server_flags, WT_CONN_SERVER_LOG)); -} - -/* - * __session_transaction_sync -- - * WT_SESSION->transaction_sync method. - */ -static int -__session_transaction_sync(WT_SESSION *wt_session, const char *config) -{ - WT_CONFIG_ITEM cval; - WT_CONNECTION_IMPL *conn; - WT_DECL_RET; - WT_LOG *log; - WT_SESSION_IMPL *session; - uint64_t remaining_usec, timeout_ms, waited_ms; - uint64_t time_start, time_stop; - - session = (WT_SESSION_IMPL *)wt_session; - SESSION_API_CALL_PREPARE_NOT_ALLOWED(session, transaction_sync, config, cfg); - WT_STAT_CONN_INCR(session, txn_sync); - - conn = S2C(session); - WT_ERR(__wt_txn_context_check(session, false)); - - /* - * If logging is not enabled there is nothing to do. - */ - if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) - WT_ERR_MSG(session, EINVAL, "logging not enabled"); - - log = conn->log; - - /* - * If there is no background sync LSN in this session, there is nothing to do. - */ - if (WT_IS_INIT_LSN(&session->bg_sync_lsn)) - goto err; - - /* - * If our LSN is smaller than the current sync LSN then our transaction is stable. We're done. - */ - if (__wt_log_cmp(&session->bg_sync_lsn, &log->sync_lsn) <= 0) - goto err; - - /* - * Our LSN is not yet stable. Wait and check again depending on the timeout. - */ - WT_ERR(__wt_config_gets_def(session, cfg, "timeout_ms", (int)WT_SESSION_BG_SYNC_MSEC, &cval)); - timeout_ms = (uint64_t)cval.val; - - if (timeout_ms == 0) - WT_ERR(ETIMEDOUT); - - /* - * Keep checking the LSNs until we find it is stable or we reach our timeout, or there's some - * other reason to quit. - */ - time_start = __wt_clock(session); - while (__wt_log_cmp(&session->bg_sync_lsn, &log->sync_lsn) > 0) { - if (!__transaction_sync_run_chk(session)) - WT_ERR(ETIMEDOUT); - - __wt_cond_signal(session, conn->log_file_cond); - time_stop = __wt_clock(session); - waited_ms = WT_CLOCKDIFF_MS(time_stop, time_start); - if (waited_ms < timeout_ms) { - remaining_usec = (timeout_ms - waited_ms) * WT_THOUSAND; - __wt_cond_wait(session, log->log_sync_cond, remaining_usec, __transaction_sync_run_chk); - } else - WT_ERR(ETIMEDOUT); - } - -err: - API_END_RET(session, ret); -} - -/* - * __session_transaction_sync_readonly -- - * WT_SESSION->transaction_sync method; readonly version. - */ -static int -__session_transaction_sync_readonly(WT_SESSION *wt_session, const char *config) -{ - WT_DECL_RET; - WT_SESSION_IMPL *session; - - WT_UNUSED(config); - - session = (WT_SESSION_IMPL *)wt_session; - SESSION_API_CALL_NOCONF(session, transaction_sync); - - ret = __wt_session_notsup(session); -err: - API_END_RET(session, ret); -} - -/* * __session_checkpoint -- * WT_SESSION->checkpoint method. */ @@ -2009,8 +1897,7 @@ __open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const __session_truncate, __session_upgrade, __session_verify, __session_begin_transaction, __session_commit_transaction, __session_prepare_transaction, __session_reset_snapshot, __session_rollback_transaction, __session_timestamp_transaction, __session_query_timestamp, - __session_checkpoint, __session_transaction_pinned_range, __session_transaction_sync, - __wt_session_breakpoint}, + __session_checkpoint, __session_transaction_pinned_range, __wt_session_breakpoint}, stds_readonly = {NULL, NULL, __session_close, __session_reconfigure, __session_flush_tier, __wt_session_strerror, __session_open_cursor, __session_alter_readonly, __session_create_readonly, __wt_session_compact_readonly, __session_drop_readonly, @@ -2020,8 +1907,7 @@ __open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const __session_begin_transaction, __session_commit_transaction, __session_prepare_transaction_readonly, __session_reset_snapshot, __session_rollback_transaction, __session_timestamp_transaction, __session_query_timestamp, - __session_checkpoint_readonly, __session_transaction_pinned_range, - __session_transaction_sync_readonly, __wt_session_breakpoint}; + __session_checkpoint_readonly, __session_transaction_pinned_range, __wt_session_breakpoint}; WT_DECL_RET; WT_SESSION_IMPL *session, *session_ret; uint32_t i; diff --git a/src/third_party/wiredtiger/src/support/global.c b/src/third_party/wiredtiger/src/support/global.c index 10bb0e93c7a..dc038c2b53c 100644 --- a/src/third_party/wiredtiger/src/support/global.c +++ b/src/third_party/wiredtiger/src/support/global.c @@ -52,7 +52,7 @@ __global_calibrate_ticks(void) __wt_process.tsc_nsec_ratio = WT_TSC_DEFAULT_RATIO; __wt_process.use_epochtime = true; -#if defined(__i386) || defined(__amd64) +#if defined(__i386) || defined(__amd64) || defined(__aarch64__) { struct timespec start, stop; double ratio; diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index a7f7c70f9c6..bc13dd8aab7 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -1066,6 +1066,7 @@ static const char *const __stats_connection_desc[] = { "cache: forced eviction - pages evicted that were clean time (usecs)", "cache: forced eviction - pages evicted that were dirty count", "cache: forced eviction - pages evicted that were dirty time (usecs)", + "cache: forced eviction - pages selected because of a large number of updates to a single item", "cache: forced eviction - pages selected because of too many deleted items count", "cache: forced eviction - pages selected count", "cache: forced eviction - pages selected unable to be evicted count", @@ -1391,7 +1392,6 @@ static const char *const __stats_connection_desc[] = { "thread-yield: connection close yielded for lsm manager shutdown", "thread-yield: data handle lock yielded", "thread-yield: get reference for page index and slot time sleeping (usecs)", - "thread-yield: log server sync yielded for log write", "thread-yield: page access yielded due to prepare state change", "thread-yield: page acquire busy blocked", "thread-yield: page acquire eviction blocked", @@ -1468,7 +1468,6 @@ static const char *const __stats_connection_desc[] = { "transaction: transaction range of timestamps pinned by the oldest timestamp", "transaction: transaction read timestamp of the oldest active reader", "transaction: transaction rollback to stable currently running", - "transaction: transaction sync calls", "transaction: transaction walk of concurrent sessions", "transaction: transactions committed", "transaction: transactions rolled back", @@ -1597,6 +1596,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_force_clean_time = 0; stats->cache_eviction_force_dirty = 0; stats->cache_eviction_force_dirty_time = 0; + stats->cache_eviction_force_long_update_list = 0; stats->cache_eviction_force_delete = 0; stats->cache_eviction_force = 0; stats->cache_eviction_force_fail = 0; @@ -1916,7 +1916,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->conn_close_blocked_lsm = 0; stats->dhandle_lock_blocked = 0; stats->page_index_slot_ref_blocked = 0; - stats->log_server_sync_blocked = 0; stats->prepared_transition_blocked_page = 0; stats->page_busy_blocked = 0; stats->page_forcible_evict_blocked = 0; @@ -1991,7 +1990,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) /* not clearing txn_pinned_timestamp_oldest */ /* not clearing txn_timestamp_oldest_active_read */ /* not clearing txn_rollback_to_stable_running */ - stats->txn_sync = 0; stats->txn_walk_sessions = 0; stats->txn_commit = 0; stats->txn_rollback = 0; @@ -2104,6 +2102,8 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->cache_eviction_force_clean_time += WT_STAT_READ(from, cache_eviction_force_clean_time); to->cache_eviction_force_dirty += WT_STAT_READ(from, cache_eviction_force_dirty); to->cache_eviction_force_dirty_time += WT_STAT_READ(from, cache_eviction_force_dirty_time); + to->cache_eviction_force_long_update_list += + WT_STAT_READ(from, cache_eviction_force_long_update_list); to->cache_eviction_force_delete += WT_STAT_READ(from, cache_eviction_force_delete); to->cache_eviction_force += WT_STAT_READ(from, cache_eviction_force); to->cache_eviction_force_fail += WT_STAT_READ(from, cache_eviction_force_fail); @@ -2449,7 +2449,6 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->conn_close_blocked_lsm += WT_STAT_READ(from, conn_close_blocked_lsm); to->dhandle_lock_blocked += WT_STAT_READ(from, dhandle_lock_blocked); to->page_index_slot_ref_blocked += WT_STAT_READ(from, page_index_slot_ref_blocked); - to->log_server_sync_blocked += WT_STAT_READ(from, log_server_sync_blocked); to->prepared_transition_blocked_page += WT_STAT_READ(from, prepared_transition_blocked_page); to->page_busy_blocked += WT_STAT_READ(from, page_busy_blocked); to->page_forcible_evict_blocked += WT_STAT_READ(from, page_forcible_evict_blocked); @@ -2528,7 +2527,6 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->txn_pinned_timestamp_oldest += WT_STAT_READ(from, txn_pinned_timestamp_oldest); to->txn_timestamp_oldest_active_read += WT_STAT_READ(from, txn_timestamp_oldest_active_read); to->txn_rollback_to_stable_running += WT_STAT_READ(from, txn_rollback_to_stable_running); - to->txn_sync += WT_STAT_READ(from, txn_sync); to->txn_walk_sessions += WT_STAT_READ(from, txn_walk_sessions); to->txn_commit += WT_STAT_READ(from, txn_commit); to->txn_rollback += WT_STAT_READ(from, txn_rollback); diff --git a/src/third_party/wiredtiger/src/tiered/tiered_handle.c b/src/third_party/wiredtiger/src/tiered/tiered_handle.c index af7a81bf341..5cfd583b1c9 100644 --- a/src/third_party/wiredtiger/src/tiered/tiered_handle.c +++ b/src/third_party/wiredtiger/src/tiered/tiered_handle.c @@ -577,7 +577,6 @@ __tiered_open(WT_SESSION_IMPL *session, const char *cfg[]) if (0) { err: __wt_free(session, tiered->obj_config); - __wt_free(session, tiered->tiers); __wt_free(session, metaconf); } __wt_verbose(session, WT_VERB_TIERED, "TIERED_OPEN: Done ret %d", ret); diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 9876159c4be..68e0e45a8ea 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -874,14 +874,24 @@ __txn_commit_timestamps_usage_check(WT_SESSION_IMPL *session, WT_TXN_OP *op, WT_ #ifdef HAVE_DIAGNOSTIC prev_op_durable_ts = upd->prev_durable_ts; + /* + * Exit abnormally as the key consistency mode dictates all updates must use timestamps once + * they have been used. + */ if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_KEY_CONSISTENT) && prev_op_durable_ts != WT_TS_NONE && - !txn_has_ts) + !txn_has_ts) { WT_RET(__wt_msg(session, WT_COMMIT_TS_VERB_PREFIX "no timestamp provided for an update to a " "table configured to always use timestamps once they are first used")); + WT_ASSERT(session, false); + } - if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_ORDERED) && txn_has_ts && prev_op_durable_ts > op_ts) + /* + * Exit abnormally as we don't allow out of order timestamps on a table configured for strict + * ordering. + */ + if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_ORDERED) && txn_has_ts && prev_op_durable_ts > op_ts) { WT_RET(__wt_msg(session, WT_COMMIT_TS_VERB_PREFIX "committing a transaction that updates a " @@ -889,18 +899,27 @@ __txn_commit_timestamps_usage_check(WT_SESSION_IMPL *session, WT_TXN_OP *op, WT_ "update (%s) on a table configured for strict ordering", __wt_timestamp_to_string(op_ts, ts_string[0]), __wt_timestamp_to_string(prev_op_durable_ts, ts_string[1]))); + WT_ASSERT(session, false); + } + /* + * Exit abnormally as we don't allow an update without a timestamp if the previous update had an + * associated timestamp. This applies to both tables configured for strict and mixed mode + * orderings. + */ if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_ORDERED) && prev_op_durable_ts != WT_TS_NONE && - !txn_has_ts) + !txn_has_ts) { WT_RET( __wt_msg(session, WT_COMMIT_TS_VERB_PREFIX "committing a transaction that updates a value without " "a timestamp while the previous update (%s) is timestamped " "on a table configured for strict ordering", __wt_timestamp_to_string(prev_op_durable_ts, ts_string[1]))); + WT_ASSERT(session, false); + } if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_MIXED_MODE) && F_ISSET(txn, WT_TXN_HAS_TS_COMMIT) && - op_ts != WT_TS_NONE && prev_op_durable_ts > op_ts) + op_ts != WT_TS_NONE && prev_op_durable_ts > op_ts) { WT_RET(__wt_msg(session, WT_COMMIT_TS_VERB_PREFIX "committing a transaction that updates a " @@ -908,6 +927,8 @@ __txn_commit_timestamps_usage_check(WT_SESSION_IMPL *session, WT_TXN_OP *op, WT_ "update (%s) on a table configured for mixed mode ordering", __wt_timestamp_to_string(op_ts, ts_string[0]), __wt_timestamp_to_string(prev_op_durable_ts, ts_string[1]))); + WT_ASSERT(session, false); + } #else WT_UNUSED(prev_op_durable_ts); #endif @@ -1052,16 +1073,18 @@ __txn_search_prepared_op( static int __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, WT_CURSOR **cursorp) { + WT_BTREE *btree; WT_CURSOR *hs_cursor; WT_CURSOR_BTREE *cbt; WT_DECL_RET; + WT_ITEM hs_recno_key; WT_TXN *txn; WT_UPDATE *fix_upd, *tombstone, *upd; #ifdef HAVE_DIAGNOSTIC WT_UPDATE *head_upd; #endif size_t not_used; - uint32_t hs_btree_id; + uint8_t *p, hs_recno_key_buf[WT_INTPACK64_MAXSIZE]; char ts_string[3][WT_TS_INT_STRING_SIZE]; bool upd_appended; @@ -1118,17 +1141,24 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, (upd->type != WT_UPDATE_TOMBSTONE || (!commit && upd->next != NULL && upd->durable_ts == upd->next->durable_ts && upd->txnid == upd->next->txnid && upd->start_ts == upd->next->start_ts))) { + btree = S2BT(session); cbt = (WT_CURSOR_BTREE *)(*cursorp); - hs_btree_id = S2BT(session)->id; - /* Open a history store table cursor. */ - WT_ERR(__wt_curhs_open(session, NULL, &hs_cursor)); - F_SET(hs_cursor, WT_CURSTD_HS_READ_COMMITTED); /* - * Scan the history store for the given btree and key with maximum start timestamp to let - * the search point to the last version of the key. + * Open a history store table cursor and scan the history store for the given btree and key + * with maximum start timestamp to let the search point to the last version of the key. */ - hs_cursor->set_key(hs_cursor, 4, hs_btree_id, &op->u.op_row.key, WT_TS_MAX, UINT64_MAX); + WT_ERR(__wt_curhs_open(session, NULL, &hs_cursor)); + F_SET(hs_cursor, WT_CURSTD_HS_READ_COMMITTED); + if (btree->type == BTREE_ROW) + hs_cursor->set_key(hs_cursor, 4, btree->id, &cbt->iface.key, WT_TS_MAX, UINT64_MAX); + else { + p = hs_recno_key_buf; + WT_ERR(__wt_vpack_uint(&p, 0, cbt->recno)); + hs_recno_key.data = hs_recno_key_buf; + hs_recno_key.size = WT_PTRDIFF(p, hs_recno_key_buf); + hs_cursor->set_key(hs_cursor, 4, btree->id, &hs_recno_key, WT_TS_MAX, UINT64_MAX); + } WT_ERR_NOTFOUND_OK(__wt_curhs_search_near_before(session, hs_cursor), true); if (ret == WT_NOTFOUND && !commit) { /* @@ -1139,12 +1169,15 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, WT_ERR(__wt_upd_alloc_tombstone(session, &tombstone, ¬_used)); #ifdef HAVE_DIAGNOSTIC WT_WITH_BTREE(session, op->btree, - ret = __wt_row_modify( - cbt, &cbt->iface.key, NULL, tombstone, WT_UPDATE_INVALID, false, false)); + ret = btree->type == BTREE_ROW ? + __wt_row_modify( + cbt, &cbt->iface.key, NULL, tombstone, WT_UPDATE_INVALID, false, false) : + __wt_col_modify(cbt, cbt->recno, NULL, tombstone, WT_UPDATE_INVALID, false, false)); #else WT_WITH_BTREE(session, op->btree, - ret = - __wt_row_modify(cbt, &cbt->iface.key, NULL, tombstone, WT_UPDATE_INVALID, false)); + ret = btree->type == BTREE_ROW ? + __wt_row_modify(cbt, &cbt->iface.key, NULL, tombstone, WT_UPDATE_INVALID, false) : + __wt_col_modify(cbt, cbt->recno, NULL, tombstone, WT_UPDATE_INVALID, false)); #endif WT_ERR(ret); tombstone = NULL; @@ -1537,9 +1570,7 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) */ if (F_ISSET(txn, WT_TXN_SYNC_SET)) WT_ERR_MSG(session, EINVAL, "Sync already set during begin_transaction"); - if (WT_STRING_MATCH("background", cval.str, cval.len)) - txn->txn_logsync = WT_LOG_BACKGROUND; - else if (WT_STRING_MATCH("off", cval.str, cval.len)) + if (WT_STRING_MATCH("off", cval.str, cval.len)) txn->txn_logsync = 0; /* * We don't need to check for "on" here because that is the default to inherit from the diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index d6b683dd4f2..1a7090cd2c6 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -1991,9 +1991,6 @@ __wt_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) !WT_IS_METADATA(session->dhandle) || FLD_ISSET(session->lock_flags, WT_SESSION_LOCKED_METADATA)); - /* Discard the cached checkpoint list when checkpointing a single file by itself. */ - __wt_meta_saved_ckptlist_free(session); - WT_RET(__wt_config_gets_def(session, cfg, "force", 0, &cval)); force = cval.val != 0; WT_SAVE_DHANDLE(session, ret = __checkpoint_lock_dirty_tree(session, true, force, true, cfg)); @@ -2066,9 +2063,6 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) (!F_ISSET(S2C(session), WT_CONN_FILE_CLOSE_SYNC) && !metadata))) return (__wt_set_return(session, EBUSY)); - /* Discard the cached checkpoint list when checkpointing a single file by itself. */ - __wt_meta_saved_ckptlist_free(session); - /* * Make sure there isn't a potential race between backup copying the metadata and a checkpoint * changing the metadata. Backup holds both the checkpoint and schema locks. Checkpoint should @@ -2096,7 +2090,7 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) if (ret == 0 && !F_ISSET(btree, WT_BTREE_SKIP_CKPT)) ret = __checkpoint_tree(session, false, NULL); - /* Do not store the cached checkpoint list when checkpointing a single file alone. */ + /* Do not store the cached checkpoint list when closing the handle. */ __wt_meta_saved_ckptlist_free(session); if (need_tracking) diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index 0a6b7e52cf9..5bc0f245497 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -339,11 +339,11 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *page WT_CELL *kcell; WT_CELL_UNPACK_KV *unpack, _unpack; WT_CURSOR *hs_cursor; + WT_DECL_ITEM(full_value); WT_DECL_ITEM(hs_key); WT_DECL_ITEM(hs_value); WT_DECL_ITEM(key); WT_DECL_RET; - WT_ITEM full_value; WT_TIME_WINDOW *hs_tw; WT_UPDATE *tombstone, *upd; wt_timestamp_t hs_durable_ts, hs_start_ts, hs_stop_durable_ts, newer_hs_durable_ts; @@ -371,7 +371,6 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *page tombstone = upd = NULL; hs_durable_ts = hs_start_ts = hs_stop_durable_ts = WT_TS_NONE; hs_btree_id = S2BT(session)->id; - WT_CLEAR(full_value); valid_update_found = false; #ifdef HAVE_DIAGNOSTIC first_record = true; @@ -392,18 +391,19 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *page } else { /* Unpack a column cell. */ WT_ERR(__wt_scr_alloc(session, WT_INTPACK64_MAXSIZE, &key)); + memp = key->mem; + WT_ERR(__wt_vpack_uint(&memp, 0, recno)); + key->size = WT_PTRDIFF(memp, key->data); /* Get the full update value from the data store. */ unpack = &_unpack; kcell = WT_COL_PTR(page, cip); __wt_cell_unpack_kv(session, page->dsk, kcell, unpack); - memp = key->mem; - WT_ERR(__wt_vpack_uint(&memp, 0, recno)); - key->size = WT_PTRDIFF(memp, key->data); } - WT_ERR(__wt_page_cell_data_ref(session, page, unpack, &full_value)); - WT_ERR(__wt_buf_set(session, &full_value, full_value.data, full_value.size)); + WT_ERR(__wt_scr_alloc(session, 0, &full_value)); + WT_ERR(__wt_page_cell_data_ref(session, page, unpack, full_value)); + WT_ERR(__wt_buf_set(session, full_value, full_value->data, full_value->size)); newer_hs_durable_ts = unpack->tw.durable_start_ts; /* Open a history store table cursor. */ @@ -447,10 +447,10 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *page if (hs_start_ts <= unpack->tw.start_ts || unpack->tw.prepare) { if (type == WT_UPDATE_MODIFY) WT_ERR(__wt_modify_apply_item( - session, S2BT(session)->value_format, &full_value, hs_value->data)); + session, S2BT(session)->value_format, full_value, hs_value->data)); else { WT_ASSERT(session, type == WT_UPDATE_STANDARD); - WT_ERR(__wt_buf_set(session, &full_value, hs_value->data, hs_value->size)); + WT_ERR(__wt_buf_set(session, full_value, hs_value->data, hs_value->size)); } } else __wt_verbose(session, WT_VERB_RECOVERY_RTS(session), @@ -532,7 +532,7 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *page __wt_hs_upd_time_window(hs_cursor, &hs_tw); WT_ASSERT(session, hs_tw->start_ts < unpack->tw.start_ts || hs_tw->start_txn < unpack->tw.start_txn); - WT_ERR(__wt_upd_alloc(session, &full_value, WT_UPDATE_STANDARD, &upd, NULL)); + WT_ERR(__wt_upd_alloc(session, full_value, WT_UPDATE_STANDARD, &upd, NULL)); /* * Set the transaction id of updates to WT_TXN_NONE when called from recovery, because the @@ -624,10 +624,10 @@ err: WT_ASSERT(session, tombstone == NULL || upd == tombstone); __wt_free_update_list(session, &upd); } + __wt_scr_free(session, &full_value); __wt_scr_free(session, &hs_key); __wt_scr_free(session, &hs_value); __wt_scr_free(session, &key); - __wt_buf_free(session, &full_value); if (hs_cursor != NULL) WT_TRET(hs_cursor->close(hs_cursor)); return (ret); @@ -643,8 +643,8 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, W { WT_CELL *kcell; WT_CELL_UNPACK_KV *vpack, _vpack; + WT_DECL_ITEM(tmp); WT_DECL_RET; - WT_ITEM buf; WT_PAGE *page; WT_UPDATE *upd; char ts_string[5][WT_TS_INT_STRING_SIZE]; @@ -652,7 +652,6 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, W page = ref->page; vpack = &_vpack; - WT_CLEAR(buf); upd = NULL; /* @@ -712,12 +711,6 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, W (vpack->tw.durable_stop_ts > rollback_timestamp || __rollback_check_if_txnid_non_committed(session, vpack->tw.stop_txn) || prepared)) { /* - * Clear the remove operation from the key by inserting the original on-disk value as a - * standard update. - */ - WT_RET(__wt_page_cell_data_ref(session, page, vpack, &buf)); - - /* * For prepared transactions, it is possible that both the on-disk key start and stop time * windows can be the same. To abort these updates, check for any stable update from history * store or remove the key. @@ -738,7 +731,16 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, W WT_STAT_CONN_DATA_INCR(session, txn_rts_keys_removed); } } else { - WT_ERR(__wt_upd_alloc(session, &buf, WT_UPDATE_STANDARD, &upd, NULL)); + /* + * Clear the remove operation from the key by inserting the original on-disk value as a + * standard update. + */ + WT_RET(__wt_scr_alloc(session, 0, &tmp)); + if ((ret = __wt_page_cell_data_ref(session, page, vpack, tmp)) == 0) + ret = __wt_upd_alloc(session, tmp, WT_UPDATE_STANDARD, &upd, NULL); + __wt_scr_free(session, &tmp); + WT_RET(ret); + /* * Set the transaction id of updates to WT_TXN_NONE when called from recovery, because * the connections write generation will be initialized after rollback to stable and the @@ -774,10 +776,9 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, W WT_ERR(__rollback_row_modify(session, page, rip, upd)); else WT_ERR(__rollback_col_modify(session, ref, upd, recno)); - upd = NULL; + return (0); err: - __wt_buf_free(session, &buf); __wt_free(session, upd); return (ret); } @@ -1672,7 +1673,13 @@ __rollback_to_stable(WT_SESSION_IMPL *session, bool no_ckpt) if (retries == WT_RTS_EVICT_MAX_RETRIES) { WT_ERR(__wt_msg( session, "timed out waiting for eviction to quiesce, running rollback to stable")); - WT_ASSERT(session, false && "Timed out waiting for eviction to quiesce prior to rts"); + /* + * FIXME: WT-7877 RTS fails when there are active transactions running in parallel to it. + * Waiting in a loop for eviction to quiesce is not efficient in some scenarios where the + * cache is not cleared in 2 minutes. Enable the following assert and + * test_rollback_to_stable22.py when the cache issue is addressed. + */ + /* WT_ASSERT(session, false && "Timed out waiting for eviction to quiesce prior to rts"); */ } /* diff --git a/src/third_party/wiredtiger/src/utilities/util_list.c b/src/third_party/wiredtiger/src/utilities/util_list.c index e0953c0abfe..49e8ae0b199 100644 --- a/src/third_party/wiredtiger/src/utilities/util_list.c +++ b/src/third_party/wiredtiger/src/utilities/util_list.c @@ -258,7 +258,8 @@ list_print_checkpoint(WT_SESSION *session, const char *key) /* Decode the checkpoint block. */ if (ckpt->raw.data == NULL) continue; - if ((ret = __wt_block_ckpt_decode(session, block, ckpt->raw.data, &ci)) == 0) { + if ((ret = __wt_block_ckpt_decode(session, block, ckpt->raw.data, ckpt->raw.size, &ci)) == + 0) { printf( "\t\t" "file-size: "); diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index c1eefaa2349..5222df53fc0 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -40,6 +40,25 @@ functions: params: path: "WT_TEST-big-endian.tgz" destination: "wiredtiger/build_posix/test/format" + "fetch artifacts from many-coll" : + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + remote_file: dbpath_100k_500.tgz + bucket: build_external + local_file: dbpath_100k_500.tgz + - command: shell.exec + params: + shell: bash + script: | + set -o errexit + set -o verbose + mkdir -p mongo-tests/largescale/many-collection/dbpath + - command: archive.targz_extract + params: + path: "dbpath_100k_500.tgz" + destination: "mongo-tests/largescale/many-collection/dbpath/." "fetch mongo-tests repo" : command: shell.exec params: @@ -416,15 +435,15 @@ functions: "upload artifact": - command: archive.targz_pack params: - target: "wiredtiger.tgz" - source_dir: "wiredtiger" + target: ${upload_filename|wiredtiger.tgz} + source_dir: ${upload_source_dir|wiredtiger} include: - "./**" - command: s3.put params: aws_secret: ${aws_secret} aws_key: ${aws_key} - local_file: wiredtiger.tgz + local_file: ${upload_filename|wiredtiger.tgz} bucket: build_external permissions: public-read content_type: application/tar @@ -2132,18 +2151,31 @@ tasks: script: | set -o errexit set -o verbose - - GCOV=/opt/mongodbtoolchain/v3/bin/gcov gcovr -r .. -e '.*/bt_(debug|dump|misc|salvage|vrfy).*' -e '.*/(log|progress|verify_build|strerror|env_msg|err_file|cur_config|os_abort)\..*' -e '.*_stat\..*' --html -o ../coverage_report.html + mkdir -p ../coverage_report + GCOV=/opt/mongodbtoolchain/v3/bin/gcov gcovr -r .. -f ../src -e '.*/bt_(debug|dump|misc|salvage|vrfy).*' -e '.*/(log|progress|verify_build|strerror|env_msg|err_file|cur_config|os_abort)\..*' -e '.*_stat\..*' -e 'bench' -e 'examples' -e 'test' -e 'ext' -e 'dist' -e 'tools' -j 4 --html-details --html-self-contained -o ../coverage_report/2_coverage_report.html - command: s3.put params: aws_secret: ${aws_secret} aws_key: ${aws_key} - local_file: wiredtiger/coverage_report.html + local_files_include_filter: wiredtiger/coverage_report/* bucket: build_external permissions: public-read content_type: text/html - display_name: Coverage report - remote_file: wiredtiger/${build_variant}/${revision}/coverage_report/coverage_report_${build_id}-${execution}.html + remote_file: wiredtiger/${build_variant}/${revision}/coverage_report_${build_id}-${execution}/ + - command: s3.put + params: + aws_secret: ${aws_secret} + aws_key: ${aws_key} + local_file: wiredtiger/coverage_report/2_coverage_report.html + bucket: build_external + permissions: public-read + content_type: text/html + # Ensure that the first character of the display_name is a space + # This will ensure that it sorts before the per-file report pages which also get a space + # at the start of their display name (why this happens is not yet clear). + display_name: " 1 Coverage report main page" + remote_file: wiredtiger/${build_variant}/${revision}/coverage_report_${build_id}-${execution}/1_coverage_report_main.html + - name: spinlock-gcc-test commands: @@ -2576,6 +2608,10 @@ tasks: - func: "import wiredtiger into mongo" - func: "compile mongodb" - func: "fetch mongo-tests repo" + # After WT-7868 we can use the following function to retrieve an existing database and remove + # the "clean-and-populate" option from the script arguments to skip the initial populating + # step. + # - func: "fetch artifacts from many-coll" - command: shell.exec params: working_dir: mongo-tests/largescale @@ -2583,8 +2619,6 @@ tasks: script: | set -o errexit set -o verbose - sudo su - ulimit -n 1000000 export "PATH=/opt/mongodbtoolchain/v3/bin:$PATH" virtualenv -p python3 venv source venv/bin/activate @@ -2596,21 +2630,26 @@ tasks: - func: "get project" - command: shell.exec params: - working_dir: "wiredtiger/src" + working_dir: "wiredtiger" shell: bash script: | set -o verbose + # Install Metrix++, ensuring it is outside the 'src' directory git clone https://github.com/metrixplusplus/metrixplusplus metrixplusplus - python "metrixplusplus/metrix++.py" collect --std.code.lines.code --std.code.complexity.cyclomatic - python "metrixplusplus/metrix++.py" view + + # We only want complexity measures for the 'src' directory + cd src + + python "../metrixplusplus/metrix++.py" collect --std.code.lines.code --std.code.complexity.cyclomatic + python "../metrixplusplus/metrix++.py" view # Set the cyclomatic complexity limit to 20 - python "metrixplusplus/metrix++.py" limit --max-limit=std.code.complexity:cyclomatic:20 + python "../metrixplusplus/metrix++.py" limit --max-limit=std.code.complexity:cyclomatic:20 # Fail if there are functions with cyclomatic complexity larger than 95 set -o errexit - python "metrixplusplus/metrix++.py" limit --max-limit=std.code.complexity:cyclomatic:95 + python "../metrixplusplus/metrix++.py" limit --max-limit=std.code.complexity:cyclomatic:95 buildvariants: @@ -2791,6 +2830,8 @@ buildvariants: --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH + upload_source_dir: mongo-tests/largescale/many-collection/dbpath/diagnostic.data + upload_filename: diagnostic.data.tgz tasks: - name: many-dhandle-stress-test - name: many-collection-test @@ -2880,7 +2921,7 @@ buildvariants: - name: code-statistics display_name: "Code statistics" - batchtime: 1440 # 1 day + batchtime: 10080 # 7 days run_on: - ubuntu1804-test expansions: @@ -2930,6 +2971,7 @@ buildvariants: windows_configure_flags: -vcvars_bat "'C:\Program Files (x86)\Microsoft Visual Studio\2017\Professional\VC\Auxiliary\Build\vcvars64.bat'" tasks: - name: compile + - name: make-check-test - name: unit-test - name: macos-1014 diff --git a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh index 521a9787145..b686b6a90ba 100755 --- a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh +++ b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh @@ -53,44 +53,134 @@ build_branch() } ############################################################# +# get_config_file_name: +# arg1: branch name +############################################################# +get_config_file_name() +{ + local file_name="" + branch_name=$1 + if [ "${wt_standalone}" = true ] || [ $older = true ] ; then + file_name="${branch_name}/test/format/CONFIG_default" + echo $file_name + return + fi + file_name="CONFIG_${branch_name}" + + echo $file_name +} + +############################################################# +# create_configs: +# arg1: branch name +############################################################# +create_configs() +{ + branch_name=$1 + + file_name=$(get_config_file_name $branch_name) + + if [ -f $file_name ] ; then + echo " WARNING - ${file_name} already exists, overwriting it." + fi + + echo "##################################################" > $file_name + echo "runs.type=row" >> $file_name # WT-7379 - Temporarily disable column store tests + echo "btree.prefix=0" >> $file_name # WT-7579 - Prefix testing isn't portable between releases + echo "cache=80" >> $file_name # Medium cache so there's eviction + echo "checksum=on" >> $file_name # WT-7851 Fix illegal checksum configuration + echo "checkpoints=1" >> $file_name # Force periodic writes + echo "compression=snappy" >> $file_name # We only built with snappy, force the choice + echo "data_source=table" >> $file_name + echo "huffman_key=0" >> $file_name # WT-6893 - Not supoprted by newer releases + echo "in_memory=0" >> $file_name # Interested in the on-disk format + echo "leak_memory=1" >> $file_name # Faster runs + echo "logging=1" >> $file_name # Test log compatibility + echo "logging_compression=snappy" >> $file_name # We only built with snappy, force the choice + echo "rows=1000000" >> $file_name + echo "salvage=0" >> $file_name # Faster runs + echo "timer=4" >> $file_name + echo "verify=1" >> $file_name # Faster runs + + # Append older release configs for newer compatibility release test + if [ $newer = true ]; then + for i in "${compatible_upgrade_downgrade_release_branches[@]}" + do + if [ "$i" == "$branch_name" ] ; then + echo "transaction.isolation=snapshot" >> $file_name # WT-7545 - Older releases can't do lower isolation levels + echo "transaction.timestamps=1" >> $file_name # WT-7545 - Older releases can't do non-timestamp transactions + break + fi + done + fi + echo "##################################################" >> $file_name +} + +############################################################# +# create_default_configs: +# This function will create the default configs for older and standalone +# release branches. +############################################################# +create_default_configs() +{ + # Iterate over the release branches and create configuration files + for b in `ls`; do + if [ -d "$b" ]; then + (create_configs $b) + fi + done +} + +############################################################# +# create_configs_for_newer_release_branches: +############################################################# +create_configs_for_newer_release_branches() +{ + # Create configs for all the newer releases + for b in ${newer_release_branches[@]}; do + (create_configs $b) + done + + # Copy per-release configs in the newer release branches + for b in ${newer_release_branches[@]}; do + cp -rf CONFIG* $b/test/format/ + done + + # Delete configs from the top folder + rm -rf CONFIG* +} + +############################################################# # run_format: # arg1: branch name # arg2: access methods list ############################################################# run_format() { + branch_name=$1 echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - echo "Running format in branch: \"$1\"" + echo "Running format in branch: \"$branch_name\"" echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - cd "$1/test/format" - - flags="-1q $(bflag $1)" - - args="" - args+="btree.prefix=0 " # Prefix testing isn't portable between releases - args+="cache=80 " # Medium cache so there's eviction - args+="checkpoints=1 " # Force periodic writes - args+="checksum=on " # Force checksums. - args+="compression=snappy " # We only built with snappy, force the choice - args+="data_source=table " - args+="huffman_key=0 " # Not supoprted by newer releases - args+="in_memory=0 " # Interested in the on-disk format - args+="leak_memory=1 " # Faster runs - args+="logging=1 " # Test log compatibility - args+="logging_compression=snappy " # We only built with snappy, force the choice - args+="rows=1000000 " - args+="runs.type=row " # Temporarily disable column store tests - args+="salvage=0 " # Faster runs - args+="timer=4 " - args+="transaction.isolation=snapshot " # Older releases can't do lower isolation levels - args+="transaction.timestamps=1 " # Older releases can't do non-timestamp transactions - args+="verify=0 " # Faster runs + cd "$branch_name/test/format" + flags="-1q $(bflag $branch_name)" + + config_file="" + + # Compatibility test for newer releases will have CONFIG file for each release + # branches for the upgrade/downgrade testing. + # + # Compatibility test for older and standalone releases will have the default config. + if [ "$newer" = true ]; then + config_file="-c CONFIG_${branch_name}" + else + config_file="-c CONFIG_default" + fi for am in $2; do dir="RUNDIR.$am" echo "./t running $am access method..." - ./t $flags -h $dir "file_type=$am" $args + ./t $flags ${config_file} -h $dir "file_type=$am" # Remove the version string from the base configuration file. (MongoDB does not create # a base configuration file, but format does, so we need to remove its version string @@ -145,6 +235,9 @@ upgrade_downgrade() echo "Upgrade/downgrade testing with \"$1\" and \"$2\"" echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + cfg_file_branch1=$(get_config_file_name $1) + cfg_file_branch2=$(get_config_file_name $2) + # Alternate running each branch format test program on the second branch's build. # Loop twice, that is, run format twice using each branch. top="$PWD" @@ -152,13 +245,13 @@ upgrade_downgrade() for reps in {1..2}; do echo "$1 format running on $2 access method $am..." cd "$top/$1/test/format" - flags="-1qR $(bflag $1)" - ./t $flags -h "$top/$2/test/format/RUNDIR.$am" timer=2 + flags="-1Rq $(bflag $1)" + ./t $flags -c "$top/$2/test/format/${cfg_file_branch1}" -h "$top/$2/test/format/RUNDIR.$am" timer=2 echo "$2 format running on $2 access method $am..." cd "$top/$2/test/format" - flags="-1qR $(bflag $2)" - ./t $flags -h "RUNDIR.$am" timer=2 + flags="-1Rq $(bflag $2)" + ./t $flags -c $cfg_file_branch2 -h "RUNDIR.$am" timer=2 done done } @@ -178,6 +271,11 @@ wt_standalone=false newer_release_branches=(develop mongodb-5.0 mongodb-4.4 mongodb-4.2) older_release_branches=(mongodb-4.2 mongodb-4.0 mongodb-3.6) +# This array is used to generate compatible configuration files between releases, because +# upgrade/downgrade test runs each build's format test program on the second build's +# configuration file. +compatible_upgrade_downgrade_release_branches=(mongodb-4.4 mongodb-4.2) + declare -A scopes scopes[newer]="newer stable release branches" scopes[older]="older stable release branches" @@ -229,6 +327,7 @@ top="test-compatibility-run" rm -rf "$top" && mkdir "$top" cd "$top" + # Build the branches. if [ "$newer" = true ]; then for b in ${newer_release_branches[@]}; do @@ -253,6 +352,13 @@ if [ "${wt_standalone}" = true ]; then (build_branch "$wt2") fi +if [ "$newer" = true ]; then + create_configs_for_newer_release_branches +else + create_default_configs +fi + + # Run format in each branch for supported access methods. if [ "$newer" = true ]; then for b in ${newer_release_branches[@]}; do @@ -267,8 +373,8 @@ if [ "$older" = true ]; then fi if [ "${wt_standalone}" = true ]; then - (run_format "$wt1" "fix row var") - (run_format "$wt2" "fix row var") + (run_format "$wt1" "row") + (run_format "$wt2" "row") fi # Verify backward compatibility for supported access methods. diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 9ea615cb519..81d21111ec3 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -149,6 +149,7 @@ tinfo_init(void) tinfo->session = NULL; tinfo->cursor = NULL; + memset(tinfo->insert_list, 0, sizeof(tinfo->insert_list)); tinfo->insert_list_cnt = 0; tinfo->state = TINFO_RUNNING; @@ -685,7 +686,7 @@ ops(void *arg) WT_SESSION *session; iso_level_t iso_level; thread_op op; - uint64_t reset_op, session_op, truncate_op; + uint64_t max_rows, reset_op, session_op, truncate_op; uint32_t range, rnd; u_int i, j; const char *iso_config; @@ -814,8 +815,12 @@ ops(void *arg) op = UPDATE; } - /* Select a row. */ - tinfo->keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows); + /* + * Select a row. Column-store extends the object, explicitly read the maximum row count and + * then use a local variable so the value won't change inside the loop. + */ + max_rows = (volatile uint64_t)g.rows; + tinfo->keyno = mmrand(&tinfo->rnd, 1, (u_int)max_rows); /* * Inserts, removes and updates can be done following a cursor set-key, or based on a cursor @@ -938,7 +943,7 @@ ops(void *arg) } if (!positioned) - tinfo->keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows); + tinfo->keyno = mmrand(&tinfo->rnd, 1, (u_int)max_rows); /* * Truncate up to 5% of the table. If the range overlaps the beginning/end of the table, @@ -950,7 +955,7 @@ ops(void *arg) * (truncating from lower keys to higher keys or vice-versa). */ greater_than = mmrand(&tinfo->rnd, 0, 1) == 1; - range = g.rows < 20 ? 0 : mmrand(&tinfo->rnd, 0, (u_int)g.rows / 20); + range = max_rows < 20 ? 0 : mmrand(&tinfo->rnd, 0, (u_int)max_rows / 20); tinfo->last = tinfo->keyno; if (greater_than) { if (g.c_reverse) { @@ -960,13 +965,13 @@ ops(void *arg) tinfo->last -= range; } else { tinfo->last += range; - if (tinfo->last > g.rows) + if (tinfo->last > max_rows) tinfo->last = 0; } } else { if (g.c_reverse) { tinfo->keyno += range; - if (tinfo->keyno > g.rows) + if (tinfo->keyno > max_rows) tinfo->keyno = 0; } else { if (tinfo->keyno <= range) @@ -1700,7 +1705,7 @@ col_insert_resolve(TINFO *tinfo) */ do { WT_ORDERED_READ(v, g.rows); - for (i = 0, p = tinfo->insert_list; i < WT_ELEMENTS(tinfo->insert_list); ++i) { + for (i = 0, p = tinfo->insert_list; i < WT_ELEMENTS(tinfo->insert_list); ++i, ++p) { if (*p == v + 1) { testutil_assert(__wt_atomic_casv64(&g.rows, v, v + 1)); *p = 0; @@ -1746,6 +1751,7 @@ col_insert(TINFO *tinfo, WT_CURSOR *cursor) else cursor->set_value(cursor, tinfo->value); + /* Create a record, then add the key to our list of new records for later resolution. */ if ((ret = cursor->insert(cursor)) != 0) return (ret); diff --git a/src/third_party/wiredtiger/test/manydbs/manydbs.c b/src/third_party/wiredtiger/test/manydbs/manydbs.c index 2dc0e5a1f8f..1dec47c5f16 100644 --- a/src/third_party/wiredtiger/test/manydbs/manydbs.c +++ b/src/third_party/wiredtiger/test/manydbs/manydbs.c @@ -54,6 +54,27 @@ static const char *const uri = "table:main"; #define MAX_KV 100 #define MAX_VAL 128 +/* + * Maximum expected condition variable wakeups. POSIX allows arbitrarily many spurious wakeups to + * happen, so we need to be able to adjust this expectation per-platform. There are two cases: when + * completely idle, and when running a light workload. The latter is expressed as a fraction of the + * total number of condition variable sleeps; the former is a constant. + */ +#if defined(__NetBSD__) || defined(_WIN32) +/* + * NetBSD should never generate spurious wakeups, but does: see https://gnats.netbsd.org/56275. + * Windows can also generate spurious wakeups: + * https://docs.microsoft.com/en-us/windows/win32/sync/condition-variables These values allow the + * test to complete in spite of that. + */ +#define CV_RESET_THRESHOLD_IDLE 20 +#define CV_RESET_THRESHOLD_DENOM 10 +#else +/* Default values: should be no wakeups when idle and allow 1/20 otherwise. */ +#define CV_RESET_THRESHOLD_IDLE 0 +#define CV_RESET_THRESHOLD_DENOM 20 +#endif + static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void usage(void) @@ -208,13 +229,13 @@ main(int argc, char *argv[]) * On an idle workload there should be no resets of condition variables during the idle * period. Even with a light workload, resets should not be very common. We look for 5%. */ - if (idle && cond_reset != cond_reset_orig[i]) - testutil_die( - ERANGE, "condition reset on idle connection %d of %" PRIu64, i, cond_reset); - if (!idle && cond_reset > cond_wait / 20) + if (idle && cond_reset > cond_reset_orig[i] + CV_RESET_THRESHOLD_IDLE) + testutil_die(ERANGE, "condition reset on idle connection %d of %" PRIu64 " exceeds %d", + i, cond_reset, CV_RESET_THRESHOLD_IDLE); + if (!idle && cond_reset > cond_wait / CV_RESET_THRESHOLD_DENOM) testutil_die(ERANGE, - "connection %d condition reset %" PRIu64 " exceeds 5%% of %" PRIu64, i, cond_reset, - cond_wait); + "connection %d condition reset %" PRIu64 " exceeds %d%% of %" PRIu64, i, cond_reset, + 100 / CV_RESET_THRESHOLD_DENOM, cond_wait); testutil_check(connections[i]->close(connections[i], NULL)); } diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py index 86aafdb261c..4e1971cbcbe 100755 --- a/src/third_party/wiredtiger/test/suite/run.py +++ b/src/third_party/wiredtiger/test/suite/run.py @@ -131,7 +131,8 @@ Options:\n\ -p | --preserve preserve output files in WT_TEST/<testname>\n\ -r N | --random-sample N randomly sort scenarios to be run, then\n\ execute every Nth (2<=N<=1000) scenario.\n\ - -s N | --scenario N use scenario N (N can be number or symbolic)\n\ + -s N | --scenario N use scenario N (N can be symbolic, number, or\n\ + list of numbers and ranges in the form 1,3-5,7)\n\ -t | --timestamp name WT_TEST according to timestamp\n\ -v N | --verbose N set verboseness to N (0<=N<=3, default=1)\n\ -i | --ignore-stdout dont fail on unexpected stdout or stderr\n\ @@ -178,16 +179,40 @@ def show_env(verbose, envvar): # e.g. test_util03 -> util reCatname = re.compile(r"test_([^0-9]+)[0-9]*") +# Look for a list of the form 0-9,11,15-17. +def parse_int_list(str): + # Use a dictionary as the result set to avoid repeated list scans. + # (Only the keys are used; the values are ignored.) + ret = {} + # Divide the input into ranges separated by commas. + for r in str.split(","): + # Split the range we got (if it is one). + bounds = r.split("-") + if len(bounds) == 1 and bounds[0].isdigit(): + # It's a single number with no dash. + scenario = int(bounds[0]) + ret[scenario] = True + continue + if len(bounds) == 2 and bounds[0].isdigit() and bounds[1].isdigit(): + # It's two numbers separated by a dash. + for scenario in range(int(bounds[0]), int(bounds[1]) + 1): + ret[scenario] = True + continue + # It's not valid syntax; give up. + return None + return ret + def restrictScenario(testcases, restrict): if restrict == '': return testcases - elif restrict.isdigit(): - s = int(restrict) - return [t for t in testcases - if hasattr(t, 'scenario_number') and t.scenario_number == s] else: - return [t for t in testcases - if hasattr(t, 'scenario_name') and t.scenario_name == restrict] + scenarios = parse_int_list(restrict) + if scenarios is not None: + return [t for t in testcases + if hasattr(t, 'scenario_number') and t.scenario_number in scenarios] + else: + return [t for t in testcases + if hasattr(t, 'scenario_name') and t.scenario_name == restrict] def addScenarioTests(tests, loader, testname, scenario): loaded = loader.loadTestsFromName(testname) diff --git a/src/third_party/wiredtiger/test/suite/test_assert01.py b/src/third_party/wiredtiger/test/suite/test_assert01.py index a91fd1700e0..4793ef26e72 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert01.py +++ b/src/third_party/wiredtiger/test/suite/test_assert01.py @@ -32,9 +32,7 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): base = 'assert01' @@ -43,13 +41,18 @@ class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): uri_def = base_uri + '.def.wt' uri_never = base_uri + '.never.wt' uri_none = base_uri + '.none.wt' - cfg = 'key_format=S,value_format=S,' cfg_always = 'verbose=[write_timestamp],write_timestamp_usage=always,assert=(write_timestamp=on)' cfg_def = '' cfg_never = 'verbose=(write_timestamp=true),write_timestamp_usage=never,assert=(write_timestamp=on)' cfg_none = 'assert=(write_timestamp=off)' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + count = 1 # # Commit a k/v pair making sure that it detects an error if needed, when @@ -57,13 +60,13 @@ class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): # def insert_check(self, uri, use_ts): c = self.session.open_cursor(uri) - key = 'key' + str(self.count) + key = 'key' + str(self.count) if self.usestrings else self.count val = 'value' + str(self.count) # Commit with a timestamp self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(self.count)) + 'commit_timestamp=' + self.timestamp_str(self.count)) c[key] = val # All settings other than never should commit successfully if (use_ts != 'never'): @@ -77,7 +80,7 @@ class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): self.count += 1 # Commit without a timestamp - key = 'key' + str(self.count) + key = 'key' + str(self.count) if self.usestrings else self.count val = 'value' + str(self.count) c = self.session.open_cursor(uri) self.session.begin_transaction() @@ -94,11 +97,13 @@ class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): c.close() def test_commit_timestamp(self): + cfg = 'key_format={},value_format=S,'.format(self.key_format) + # Create a data item at a timestamp - self.session.create(self.uri_always, self.cfg + self.cfg_always) - self.session.create(self.uri_def, self.cfg + self.cfg_def) - self.session.create(self.uri_never, self.cfg + self.cfg_never) - self.session.create(self.uri_none, self.cfg + self.cfg_none) + self.session.create(self.uri_always, cfg + self.cfg_always) + self.session.create(self.uri_def, cfg + self.cfg_def) + self.session.create(self.uri_never, cfg + self.cfg_never) + self.session.create(self.uri_none, cfg + self.cfg_none) # Check inserting into each table self.insert_check(self.uri_always, 'always') diff --git a/src/third_party/wiredtiger/test/suite/test_assert02.py b/src/third_party/wiredtiger/test/suite/test_assert02.py index ba871eb01c9..d3d5e885e87 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert02.py +++ b/src/third_party/wiredtiger/test/suite/test_assert02.py @@ -32,13 +32,17 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert02(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + def test_read_timestamp(self): #if not wiredtiger.diagnostic_build(): # self.skipTest('requires a diagnostic build') @@ -50,29 +54,32 @@ class test_assert02(wttest.WiredTigerTestCase, suite_subprocess): uri_never = base_uri + '.never.wt' uri_none = base_uri + '.none.wt' - cfg = 'key_format=S,value_format=S' + cfg = 'key_format={},value_format=S'.format(self.key_format) cfg_always = cfg + ',write_timestamp_usage=always,assert=(read_timestamp=always)' cfg_def = cfg cfg_never = cfg + ',assert=(read_timestamp=never)' cfg_none = cfg + ',assert=(read_timestamp=none)' - # Create a data item at a timestamp + # Create a data item at a timestamp. self.session.create(uri_always, cfg_always) self.session.create(uri_def, cfg_def) self.session.create(uri_never, cfg_never) self.session.create(uri_none, cfg_none) + # Make a key. + key1 = 'key1' if self.usestrings else 1 + # Insert a data item at timestamp 1. This should work for all. c_always = self.session.open_cursor(uri_always) c_def = self.session.open_cursor(uri_def) c_never = self.session.open_cursor(uri_never) c_none = self.session.open_cursor(uri_none) self.session.begin_transaction() - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(1)) - c_always['key1'] = 'value1' - c_def['key1'] = 'value1' - c_never['key1'] = 'value1' - c_none['key1'] = 'value1' + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(1)) + c_always[key1] = 'value1' + c_def[key1] = 'value1' + c_never[key1] = 'value1' + c_none[key1] = 'value1' self.session.commit_transaction() c_always.close() c_def.close() @@ -86,12 +93,12 @@ class test_assert02(wttest.WiredTigerTestCase, suite_subprocess): c_never = self.session.open_cursor(uri_never) c_none = self.session.open_cursor(uri_none) - c_always.set_key('key1') - c_def.set_key('key1') - c_never.set_key('key1') - c_none.set_key('key1') + c_always.set_key(key1) + c_def.set_key(key1) + c_never.set_key(key1) + c_none.set_key(key1) - self.session.begin_transaction('read_timestamp=' + timestamp_str(1)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(1)) c_always.search() c_def.search() c_none.search() @@ -114,10 +121,10 @@ class test_assert02(wttest.WiredTigerTestCase, suite_subprocess): c_never = self.session.open_cursor(uri_never) c_none = self.session.open_cursor(uri_none) - c_always.set_key('key1') - c_def.set_key('key1') - c_never.set_key('key1') - c_none.set_key('key1') + c_always.set_key(key1) + c_def.set_key(key1) + c_never.set_key(key1) + c_none.set_key(key1) self.session.begin_transaction() c_never.search() diff --git a/src/third_party/wiredtiger/test/suite/test_assert03.py b/src/third_party/wiredtiger/test/suite/test_assert03.py index d1ac3926269..51b219b2fac 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert03.py +++ b/src/third_party/wiredtiger/test/suite/test_assert03.py @@ -32,24 +32,36 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest +from wtscenario import make_scenarios class test_assert03(wttest.WiredTigerTestCase, suite_subprocess): conn_config = 'log=(enabled)' base_uri = 'file:assert03.wt' - cfg = 'key_format=S,value_format=S' always = 'write_timestamp_usage=always,assert=(write_timestamp=on)' never = 'write_timestamp_usage=never,assert=(write_timestamp=on)' none = 'assert=(write_timestamp=off)' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + def test_assert03(self): #if not wiredtiger.diagnostic_build(): # self.skipTest('requires a diagnostic build') + cfg = 'key_format={},value_format=S'.format(self.key_format) + key0 = 'key0' if self.usestrings else 17 + key1 = 'key1' if self.usestrings else 18 + key2 = 'key2' if self.usestrings else 19 + key3 = 'key3' if self.usestrings else 20 + # Create a data item at the default setting - self.session.create(self.base_uri, self.cfg) + self.session.create(self.base_uri, cfg) c = self.session.open_cursor(self.base_uri) self.session.begin_transaction() - c['key0'] = 'value0' + c[key0] = 'value0' self.session.commit_transaction() c.close() @@ -58,7 +70,7 @@ class test_assert03(wttest.WiredTigerTestCase, suite_subprocess): self.session.alter(self.base_uri, self.always) c = self.session.open_cursor(self.base_uri) self.session.begin_transaction() - c['key1'] = 'value1' + c[key1] = 'value1' msg = "/none set on this transaction/" self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda:self.assertEquals(self.session.commit_transaction(), 0), msg) @@ -68,14 +80,14 @@ class test_assert03(wttest.WiredTigerTestCase, suite_subprocess): self.session.alter(self.base_uri, self.never) c = self.session.open_cursor(self.base_uri) self.session.begin_transaction() - c['key2'] = 'value2' + c[key2] = 'value2' self.session.commit_transaction() c.close() self.session.alter(self.base_uri, self.none) c = self.session.open_cursor(self.base_uri) self.session.begin_transaction() - c['key3'] = 'value3' + c[key3] = 'value3' self.session.commit_transaction() c.close() diff --git a/src/third_party/wiredtiger/test/suite/test_assert04.py b/src/third_party/wiredtiger/test/suite/test_assert04.py index 711e1d02b06..6c5d4623d4d 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert04.py +++ b/src/third_party/wiredtiger/test/suite/test_assert04.py @@ -32,13 +32,17 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + def test_timestamp_alter(self): base = 'assert04' uri = 'file:' + base @@ -50,14 +54,17 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # Create the table without the key consistency checking turned on. # Create a few items breaking the rules. Then alter the setting and # verify the inconsistent usage is detected. - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + + key_nots = 'key_nots' if self.usestrings else 5 + key_ts1 = 'key_ts1' if self.usestrings else 16 # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)) - c['key_ts1'] = 'value2' + 'commit_timestamp=' + self.timestamp_str(2)) + c[key_ts1] = 'value2' self.session.commit_transaction() c.close() @@ -65,8 +72,8 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(1)) - c['key_ts1'] = 'value1' + 'commit_timestamp=' + self.timestamp_str(1)) + c[key_ts1] = 'value1' self.session.commit_transaction() c.close() @@ -74,27 +81,27 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # again modify without a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots' + c[key_nots] = 'value_nots' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)) - c['key_nots'] = 'value2' + 'commit_timestamp=' + self.timestamp_str(2)) + c[key_nots] = 'value2' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots2' + c[key_nots] = 'value_nots2' self.session.commit_transaction() c.close() # We must move the oldest timestamp forward in order to alter. # Otherwise alter closing the file will fail with EBUSY. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(2)) # Now alter the setting and make sure we detect incorrect usage. self.session.alter(uri, cfg_on) @@ -103,16 +110,16 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(5)) - c['key_ts1'] = 'value5' + 'commit_timestamp=' + self.timestamp_str(5)) + c[key_ts1] = 'value5' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(4)) - c['key_ts1'] = 'value4' + 'commit_timestamp=' + self.timestamp_str(4)) + c[key_ts1] = 'value4' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() @@ -120,7 +127,7 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # Detect not using a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value_nots3' + c[key_ts1] = 'value_nots3' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() @@ -131,41 +138,41 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # following transaction. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots3' + c[key_nots] = 'value_nots3' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(3)) - c['key_nots'] = 'value3' + 'commit_timestamp=' + self.timestamp_str(3)) + c[key_nots] = 'value3' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts1'], 'value5') - self.assertEquals(c['key_nots'], 'value_nots3') + self.assertEquals(c[key_ts1], 'value5') + self.assertEquals(c[key_nots], 'value_nots3') c.close() # Now alter the setting again and detection is off. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5)) self.session.alter(uri, cfg_off) # Detection is off we can successfully change the same key with and # without a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots4' + c[key_nots] = 'value_nots4' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(6)) - c['key_nots'] = 'value6' + 'commit_timestamp=' + self.timestamp_str(6)) + c[key_nots] = 'value6' self.session.commit_transaction() c.close() @@ -179,14 +186,22 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # That checking will verify any individual key is always or never # used with a timestamp. And if it is used with a timestamp that # the timestamps are in increasing order for that key. - self.session.create(uri, 'key_format=S,value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)') + self.session.create(uri, 'key_format={},value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)'.format(self.key_format)) + + key_nots = 'key_nots' if self.usestrings else 5 + key_ts1 = 'key_ts1' if self.usestrings else 16 + key_ts2 = 'key_ts2' if self.usestrings else 17 + key_ts3 = 'key_ts3' if self.usestrings else 18 + key_ts4 = 'key_ts4' if self.usestrings else 19 + key_ts5 = 'key_ts5' if self.usestrings else 20 + key_ts6 = 'key_ts6' if self.usestrings else 21 # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)) - c['key_ts1'] = 'value2' + 'commit_timestamp=' + self.timestamp_str(2)) + c[key_ts1] = 'value2' self.session.commit_transaction() c.close() @@ -194,8 +209,8 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(1)) - c['key_ts1'] = 'value1' + 'commit_timestamp=' + self.timestamp_str(1)) + c[key_ts1] = 'value1' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() @@ -204,8 +219,8 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(1)) - c['key_ts2'] = 'value1' + 'commit_timestamp=' + self.timestamp_str(1)) + c[key_ts2] = 'value1' self.session.commit_transaction() c.close() @@ -218,28 +233,28 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(10)) - c['key_ts3'] = 'value10' + 'commit_timestamp=' + self.timestamp_str(10)) + c[key_ts3] = 'value10' self.session.commit_transaction() self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(15)) - c['key_ts4'] = 'value15' + 'commit_timestamp=' + self.timestamp_str(15)) + c[key_ts4] = 'value15' self.session.commit_transaction() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(13)) - c['key_ts3'] = 'value13' - c['key_ts4'] = 'value13' + 'commit_timestamp=' + self.timestamp_str(13)) + c[key_ts3] = 'value13' + c[key_ts4] = 'value13' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts3'], 'value10') - self.assertEquals(c['key_ts4'], 'value15') + self.assertEquals(c[key_ts3], 'value10') + self.assertEquals(c[key_ts4], 'value15') c.close() # @@ -249,15 +264,15 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(13)) - c['key_ts3'] = 'value13' + 'commit_timestamp=' + self.timestamp_str(13)) + c[key_ts3] = 'value13' self.session.commit_transaction() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(13)) - c['key_ts4'] = 'value13' + 'commit_timestamp=' + self.timestamp_str(13)) + c[key_ts4] = 'value13' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() @@ -268,43 +283,43 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(14)) - c['key_ts4'] = 'value14' + 'commit_timestamp=' + self.timestamp_str(14)) + c[key_ts4] = 'value14' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value15') + self.assertEquals(c[key_ts4], 'value15') c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(16)) - c['key_ts4'] = 'value16' + 'commit_timestamp=' + self.timestamp_str(16)) + c[key_ts4] = 'value16' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value16') + self.assertEquals(c[key_ts4], 'value16') c.close() # Now try to modify a key previously used with timestamps without # one. We should get the inconsistent usage message. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value_nots' + c[key_ts4] = 'value_nots' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value_nots' + c[key_ts4] = 'value_nots' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value16') + self.assertEquals(c[key_ts4], 'value16') c.close() # Now confirm the other way. Create a key without a timestamp and then @@ -312,82 +327,82 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # makes sense here is the inconsistent usage. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots' + c[key_nots] = 'value_nots' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(16)) - c['key_nots'] = 'value16' + 'commit_timestamp=' + self.timestamp_str(16)) + c[key_nots] = 'value16' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots1' + c[key_nots] = 'value_nots1' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(17)) - c['key_nots'] = 'value17' + 'commit_timestamp=' + self.timestamp_str(17)) + c[key_nots] = 'value17' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_nots'], 'value_nots1') + self.assertEquals(c[key_nots], 'value_nots1') c.close() # Confirm it is okay to set the timestamp in the middle or end of the # transaction. That should set the timestamp for the whole thing. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts5'] = 'value_notsyet' + c[key_ts5] = 'value_notsyet' self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(20)) - c['key_ts5'] = 'value20' + 'commit_timestamp=' + self.timestamp_str(20)) + c[key_ts5] = 'value20' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts5'], 'value20') + self.assertEquals(c[key_ts5], 'value20') c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts6'] = 'value_notsyet' - c['key_ts6'] = 'value21_after' + c[key_ts6] = 'value_notsyet' + c[key_ts6] = 'value21_after' self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(21)) + 'commit_timestamp=' + self.timestamp_str(21)) self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts6'], 'value21_after') + self.assertEquals(c[key_ts6], 'value21_after') c.close() # Confirm it is okay to set the timestamp on the commit call. # That should set the timestamp for the whole thing. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts6'] = 'value_committs1' - c['key_ts6'] = 'value22' + c[key_ts6] = 'value_committs1' + c[key_ts6] = 'value22' self.session.commit_transaction('commit_timestamp=' + - timestamp_str(22)) + self.timestamp_str(22)) c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value23' + c[key_nots] = 'value23' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(23)), msg_usage) + 'commit_timestamp=' + self.timestamp_str(23)), msg_usage) c.close() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_assert05.py b/src/third_party/wiredtiger/test/suite/test_assert05.py index b5fe9205bd9..afa2acbcbfb 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert05.py +++ b/src/third_party/wiredtiger/test/suite/test_assert05.py @@ -32,9 +32,7 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): base = 'assert05' @@ -44,12 +42,17 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): uri_def = base_uri + '.def.wt' uri_never = base_uri + '.never.wt' uri_none = base_uri + '.none.wt' - cfg = 'key_format=S,value_format=S,' cfg_always = 'verbose=(write_timestamp=true),write_timestamp_usage=always,assert=(write_timestamp=on)' cfg_def = '' cfg_never = 'write_timestamp_usage=never,assert=(write_timestamp=on)' cfg_none = 'assert=(write_timestamp=off)' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + count = 1 # # Commit a k/v pair making sure that it detects an error if needed, when @@ -57,18 +60,18 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): # def insert_check(self, uri, use_ts): c = self.session.open_cursor(uri) - key = 'key' + str(self.count) + key = 'key' + str(self.count) if self.usestrings else self.count val = 'value' + str(self.count) # Commit with a timestamp self.session.begin_transaction() c[key] = val self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(self.count)) + 'prepare_timestamp=' + self.timestamp_str(self.count)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(self.count)) + 'commit_timestamp=' + self.timestamp_str(self.count)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(self.count)) + 'durable_timestamp=' + self.timestamp_str(self.count)) # All settings other than never should commit successfully if (use_ts != 'never'): self.session.commit_transaction() @@ -86,17 +89,17 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): self.count += 1 # Commit without a timestamp - key = 'key' + str(self.count) + key = 'key' + str(self.count) if self.usestrings else self.count val = 'value' + str(self.count) c = self.session.open_cursor(uri) self.session.begin_transaction() c[key] = val if (use_ts == 'always'): self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(self.count)) + 'prepare_timestamp=' + self.timestamp_str(self.count)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(self.count)) + 'commit_timestamp=' + self.timestamp_str(self.count)) # All settings other than always should commit successfully if (use_ts != 'always' and use_ts != 'never'): self.session.commit_transaction() @@ -114,11 +117,13 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): c.close() def test_durable_timestamp(self): + cfg = 'key_format={},value_format=S,'.format(self.key_format) + # Create a data item at a timestamp - self.session.create(self.uri_always, self.cfg + self.cfg_always) - self.session.create(self.uri_def, self.cfg + self.cfg_def) - self.session.create(self.uri_never, self.cfg + self.cfg_never) - self.session.create(self.uri_none, self.cfg + self.cfg_none) + self.session.create(self.uri_always, cfg + self.cfg_always) + self.session.create(self.uri_def, cfg + self.cfg_def) + self.session.create(self.uri_never, cfg + self.cfg_never) + self.session.create(self.uri_none, cfg + self.cfg_none) # Check inserting into each table self.insert_check(self.uri_always, 'always') diff --git a/src/third_party/wiredtiger/test/suite/test_assert06.py b/src/third_party/wiredtiger/test/suite/test_assert06.py index 24891445ab0..3ea620e638e 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert06.py +++ b/src/third_party/wiredtiger/test/suite/test_assert06.py @@ -32,20 +32,24 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + def apply_timestamps(self, timestamp): self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(timestamp)) + 'prepare_timestamp=' + self.timestamp_str(timestamp)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(timestamp)) + 'commit_timestamp=' + self.timestamp_str(timestamp)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(timestamp)) + 'durable_timestamp=' + self.timestamp_str(timestamp)) def test_timestamp_alter(self): base = 'assert06' @@ -55,14 +59,17 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): msg_ooo='/out of order/' msg_usage='/used inconsistently/' + key_nots = 'key_nots' if self.usestrings else 5 + key_ts1 = 'key_ts1' if self.usestrings else 16 + # Create the table without the key consistency checking turned on. # Create a few items breaking the rules. Then alter the setting and # verify the inconsistent usage is detected. - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value2' + c[key_ts1] = 'value2' self.apply_timestamps(2) self.session.commit_transaction() c.close() @@ -70,7 +77,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # Modify the data item at timestamp 1. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value1' + c[key_ts1] = 'value1' self.apply_timestamps(1) self.session.commit_transaction() c.close() @@ -79,26 +86,26 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # again modify without a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots' + c[key_nots] = 'value_nots' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value2' + c[key_nots] = 'value2' self.apply_timestamps(2) self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots2' + c[key_nots] = 'value_nots2' self.session.commit_transaction() c.close() # We must move the oldest timestamp forward in order to alter. # Otherwise alter closing the file will fail with EBUSY. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(2)) # Now alter the setting and make sure we detect incorrect usage. self.session.alter(uri, cfg_on) @@ -106,7 +113,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # Detect decreasing timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value5' + c[key_ts1] = 'value5' self.apply_timestamps(5) self.session.commit_transaction() c.close() @@ -116,7 +123,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value4' + c[key_ts1] = 'value4' self.apply_timestamps(4) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) @@ -126,7 +133,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # Detect not using a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value_nots3' + c[key_ts1] = 'value_nots3' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() @@ -137,7 +144,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # following transaction. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots3' + c[key_nots] = 'value_nots3' self.session.commit_transaction() c.close() @@ -146,7 +153,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value3' + c[key_nots] = 'value3' self.apply_timestamps(3) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) @@ -155,26 +162,26 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): ''' c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts1'], 'value5') - self.assertEquals(c['key_nots'], 'value_nots3') + self.assertEquals(c[key_ts1], 'value5') + self.assertEquals(c[key_nots], 'value_nots3') c.close() # Test to make sure that key consistency can be turned off # after turning it on. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5)) self.session.alter(uri, cfg_off) # Detection is off we can successfully change the same key with and # without a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots4' + c[key_nots] = 'value_nots4' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value6' + c[key_nots] = 'value6' self.apply_timestamps(6) self.session.commit_transaction() c.close() @@ -185,16 +192,24 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): msg_ooo='/out of order/' msg_usage='/used inconsistently/' + key_nots = 'key_nots' if self.usestrings else 5 + key_ts1 = 'key_ts1' if self.usestrings else 16 + key_ts2 = 'key_ts2' if self.usestrings else 17 + key_ts3 = 'key_ts3' if self.usestrings else 18 + key_ts4 = 'key_ts4' if self.usestrings else 19 + key_ts5 = 'key_ts5' if self.usestrings else 20 + key_ts6 = 'key_ts6' if self.usestrings else 21 + # Create the table with the key consistency checking turned on. # That checking will verify any individual key is always or never # used with a timestamp. And if it is used with a timestamp that # the timestamps are in increasing order for that key. - self.session.create(uri, 'key_format=S,value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)') + self.session.create(uri, 'key_format={},value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)'.format(self.key_format)) # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value2' + c[key_ts1] = 'value2' self.apply_timestamps(2) self.session.commit_transaction() c.close() @@ -205,7 +220,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # Modify the data item at timestamp 1. We should detect it is wrong. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value1' + c[key_ts1] = 'value1' self.apply_timestamps(1) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) @@ -215,7 +230,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # Make sure we can successfully add a different key at timestamp 1. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts2'] = 'value1' + c[key_ts2] = 'value1' self.apply_timestamps(1) self.session.commit_transaction() c.close() @@ -228,11 +243,11 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts3'] = 'value10' + c[key_ts3] = 'value10' self.apply_timestamps(10) self.session.commit_transaction() self.session.begin_transaction() - c['key_ts4'] = 'value15' + c[key_ts4] = 'value15' self.apply_timestamps(15) self.session.commit_transaction() @@ -241,8 +256,8 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts3'] = 'value13' - c['key_ts4'] = 'value13' + c[key_ts3] = 'value13' + c[key_ts4] = 'value13' self.apply_timestamps(13) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) @@ -250,8 +265,8 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): ''' c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts3'], 'value10') - self.assertEquals(c['key_ts4'], 'value15') + self.assertEquals(c[key_ts3], 'value10') + self.assertEquals(c[key_ts4], 'value15') c.close() # @@ -260,7 +275,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts3'] = 'value13' + c[key_ts3] = 'value13' self.apply_timestamps(13) self.session.commit_transaction() @@ -269,7 +284,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value13' + c[key_ts4] = 'value13' self.apply_timestamps(13) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) @@ -284,43 +299,43 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # in the update chain are not considered for the timestamp check. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value14' + c[key_ts4] = 'value14' self.apply_timestamps(14) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value15') + self.assertEquals(c[key_ts4], 'value15') c.close() ''' c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value16' + c[key_ts4] = 'value16' self.apply_timestamps(16) self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value16') + self.assertEquals(c[key_ts4], 'value16') c.close() # Now try to modify a key previously used with timestamps without # one. We should get the inconsistent usage message. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value_nots' + c[key_ts4] = 'value_nots' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value_nots' + c[key_ts4] = 'value_nots' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value16') + self.assertEquals(c[key_ts4], 'value16') c.close() # Now confirm the other way. Create a key without a timestamp and then @@ -328,7 +343,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # makes sense here is the inconsistent usage. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots' + c[key_nots] = 'value_nots' self.session.commit_transaction() c.close() @@ -337,7 +352,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value16' + c[key_nots] = 'value16' self.apply_timestamps(16) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) @@ -346,7 +361,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots1' + c[key_nots] = 'value_nots1' self.session.commit_transaction() c.close() @@ -355,7 +370,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value17' + c[key_nots] = 'value17' self.apply_timestamps(17) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) @@ -363,47 +378,47 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): ''' c = self.session.open_cursor(uri) - self.assertEquals(c['key_nots'], 'value_nots1') + self.assertEquals(c[key_nots], 'value_nots1') c.close() # Confirm it is okay to set the timestamp in the middle or end of the # transaction. That should set the timestamp for the whole thing. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts5'] = 'value_notsyet' - c['key_ts5'] = 'value20' + c[key_ts5] = 'value_notsyet' + c[key_ts5] = 'value20' self.apply_timestamps(20) self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts5'], 'value20') + self.assertEquals(c[key_ts5], 'value20') c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts6'] = 'value_notsyet' - c['key_ts6'] = 'value21_after' + c[key_ts6] = 'value_notsyet' + c[key_ts6] = 'value21_after' self.apply_timestamps(21) self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts6'], 'value21_after') + self.assertEquals(c[key_ts6], 'value21_after') c.close() # Confirm it is okay to set the durable timestamp on the commit call. # That should set the timestamp for the whole thing. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts6'] = 'value_committs1' - c['key_ts6'] = 'value22' + c[key_ts6] = 'value_committs1' + c[key_ts6] = 'value22' self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(22)) + 'prepare_timestamp=' + self.timestamp_str(22)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(22)) + 'commit_timestamp=' + self.timestamp_str(22)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(22)) + 'durable_timestamp=' + self.timestamp_str(22)) self.session.commit_transaction() c.close() @@ -412,14 +427,14 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value23' + c[key_nots] = 'value23' self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(23)) + 'prepare_timestamp=' + self.timestamp_str(23)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(23)) + 'commit_timestamp=' + self.timestamp_str(23)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'durable_timestamp=' + timestamp_str(23)), msg_usage) + 'durable_timestamp=' + self.timestamp_str(23)), msg_usage) c.close() ''' diff --git a/src/third_party/wiredtiger/test/suite/test_assert07.py b/src/third_party/wiredtiger/test/suite/test_assert07.py index e79dd1a27dc..687174b4796 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert07.py +++ b/src/third_party/wiredtiger/test/suite/test_assert07.py @@ -33,95 +33,101 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert07(wttest.WiredTigerTestCase, suite_subprocess): + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + def apply_timestamps(self, timestamp): self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(timestamp)) + 'prepare_timestamp=' + self.timestamp_str(timestamp)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(timestamp)) + 'commit_timestamp=' + self.timestamp_str(timestamp)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(timestamp)) + 'durable_timestamp=' + self.timestamp_str(timestamp)) def test_timestamp_alter(self): base = 'assert07' uri = 'file:' + base + key_ts1 = 'key_ts1' if self.usestrings else 1 + # No reserved, single update. - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) c = self.session.open_cursor(uri) self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value1' + c[key_ts1] = 'value1' self.apply_timestamps(1) self.session.commit_transaction() # Reserved at the start of the chain, with one update. self.session.begin_transaction('isolation=snapshot') - c.set_key('key_ts1') + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value2' + c[key_ts1] = 'value2' self.apply_timestamps(2) self.session.commit_transaction() # Reserved at the end of the chain, with one update. self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value3' - c.set_key('key_ts1') + c[key_ts1] = 'value3' + c.set_key(key_ts1) c.reserve() self.apply_timestamps(3) self.session.commit_transaction() # Reserved at the start of the chain, with multiple. self.session.begin_transaction('isolation=snapshot') - c.set_key('key_ts1') + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value4' - c['key_ts1'] = 'value5' + c[key_ts1] = 'value4' + c[key_ts1] = 'value5' self.apply_timestamps(4) self.session.commit_transaction() # Reserved at the end of the chain, with multiple updates. self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value6' - c['key_ts1'] = 'value7' - c.set_key('key_ts1') + c[key_ts1] = 'value6' + c[key_ts1] = 'value7' + c.set_key(key_ts1) c.reserve() self.apply_timestamps(5) self.session.commit_transaction() # Reserved between two updates. self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value8' - c.set_key('key_ts1') + c[key_ts1] = 'value8' + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value9' + c[key_ts1] = 'value9' self.apply_timestamps(6) self.session.commit_transaction() # Reserved update with multiple extra updates. self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value10' - c.set_key('key_ts1') + c[key_ts1] = 'value10' + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value11' - c['key_ts1'] = 'value12' - c['key_ts1'] = 'value13' + c[key_ts1] = 'value11' + c[key_ts1] = 'value12' + c[key_ts1] = 'value13' self.apply_timestamps(7) self.session.commit_transaction() # Reserved updates with multiple extra updates. self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value14' - c.set_key('key_ts1') + c[key_ts1] = 'value14' + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value15' - c['key_ts1'] = 'value16' - c.set_key('key_ts1') + c[key_ts1] = 'value15' + c[key_ts1] = 'value16' + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value17' + c[key_ts1] = 'value17' self.apply_timestamps(8) self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_backup08.py b/src/third_party/wiredtiger/test/suite/test_backup08.py index 1c674df5a54..95e50971408 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup08.py +++ b/src/third_party/wiredtiger/test/suite/test_backup08.py @@ -34,9 +34,6 @@ import os, shutil import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_backup08(wttest.WiredTigerTestCase): conn_config = 'config_base=false,create,log=(enabled)' session_config = 'isolation=snapshot' @@ -85,12 +82,12 @@ class test_backup08(wttest.WiredTigerTestCase): curs[i] = i self.pr("i: " + str(i)) self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(i)) + 'commit_timestamp=' + self.timestamp_str(i)) # Set the oldest and stable timestamp a bit earlier than the data # we inserted. Take a checkpoint to the stable timestamp. self.pr("stable ts: " + str(ts)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(ts) + - ',stable_timestamp=' + timestamp_str(ts)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(ts) + + ',stable_timestamp=' + self.timestamp_str(ts)) # This forces a different checkpoint timestamp for each table. # Default is to use the stable timestamp. expected = ts @@ -102,7 +99,7 @@ class test_backup08(wttest.WiredTigerTestCase): ckpt_config = 'use_timestamp=true' self.session.checkpoint(ckpt_config) q = self.conn.query_timestamp('get=last_checkpoint') - self.assertTimestampsEqual(q, timestamp_str(expected)) + self.assertTimestampsEqual(q, self.timestamp_str(expected)) return expected def backup_and_recover(self, expected_rec_ts): @@ -123,7 +120,7 @@ class test_backup08(wttest.WiredTigerTestCase): backup_conn = self.wiredtiger_open(self.dir) q = backup_conn.query_timestamp('get=recovery') self.pr("query recovery ts: " + q) - self.assertTimestampsEqual(q, timestamp_str(expected_rec_ts)) + self.assertTimestampsEqual(q, self.timestamp_str(expected_rec_ts)) def test_timestamp_backup(self): # Add some data and checkpoint using the timestamp or not diff --git a/src/third_party/wiredtiger/test/suite/test_backup15.py b/src/third_party/wiredtiger/test/suite/test_backup15.py index 6aa2f854b95..009fdf85e40 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup15.py +++ b/src/third_party/wiredtiger/test/suite/test_backup15.py @@ -25,13 +25,8 @@ # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. - -import wiredtiger -import os, shutil +import os from wtbackup import backup_base -from wtdataset import simple_key -from wtscenario import make_scenarios -import glob # test_backup15.py # Test cursor backup with a block-based incremental cursor. @@ -118,6 +113,3 @@ class test_backup15(backup_base): self.take_full_backup(self.home_full) self.compare_backups(self.uri, self.home_full, self.home_incr, str(self.bkup_id)) self.setup_directories(self.home_incr, self.home_full) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug022.py b/src/third_party/wiredtiger/test/suite/test_bug022.py index fe7c2c32f28..31fb2387662 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug022.py +++ b/src/third_party/wiredtiger/test/suite/test_bug022.py @@ -31,9 +31,6 @@ import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_bug022(wttest.WiredTigerTestCase): uri = 'file:test_bug022' conn_config = 'cache_size=50MB' @@ -41,21 +38,21 @@ class test_bug022(wttest.WiredTigerTestCase): def test_apply_modifies_on_onpage_tombstone(self): self.session.create(self.uri, 'key_format=S,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) value = 'a' * 500 for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Apply tombstones for every key. for i in range(1, 10000): self.session.begin_transaction() cursor.set_key(str(i)) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint03.py b/src/third_party/wiredtiger/test/suite/test_checkpoint03.py index 0f4fba34186..10f6138c2c0 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint03.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint03.py @@ -39,9 +39,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_checkpoint03(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_checkpoint03' conn_config = 'statistics=(all)' diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint04.py b/src/third_party/wiredtiger/test/suite/test_checkpoint04.py index 413f3333428..eda62c33f66 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint04.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint04.py @@ -33,9 +33,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - class test_checkpoint04(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' session_config = 'isolation=snapshot' diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint06.py b/src/third_party/wiredtiger/test/suite/test_checkpoint06.py index edd22434228..06795a3cd27 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint06.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint06.py @@ -29,9 +29,6 @@ import time import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - # test_checkpoint06.py # Verify that we rollback the truncation that is committed after stable # timestamp in the checkpoint. @@ -44,14 +41,14 @@ class test_checkpoint06(wttest.WiredTigerTestCase): self.session.create(self.uri, 'key_format=i,value_format=S') value = "abcdefghijklmnopqrstuvwxyz" * 3 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) self.session.begin_transaction() # Setup: Insert some data for i in range(10000): cursor[i] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(2)) # Flush everything to disk self.reopen_conn() @@ -63,7 +60,7 @@ class test_checkpoint06(wttest.WiredTigerTestCase): end = self.session.open_cursor(self.uri) end.set_key(9995) self.session.truncate(None, start, None, None) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Do a checkpoint self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint07.py b/src/third_party/wiredtiger/test/suite/test_checkpoint07.py index c1c11dbad91..a876d1ac807 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint07.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint07.py @@ -33,9 +33,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - class test_checkpoint07(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' session_config = 'isolation=snapshot' diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint08.py b/src/third_party/wiredtiger/test/suite/test_checkpoint08.py index 5a8e8fe647e..49de9c5f163 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint08.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint08.py @@ -37,9 +37,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - class test_checkpoint08(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' session_config = 'isolation=snapshot' @@ -61,8 +58,8 @@ class test_checkpoint08(wttest.WiredTigerTestCase): self.session.create(self.uri2, 'key_format=i,value_format=i') # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) # Setup: Insert some data and checkpoint it. Then modify only # the data in the first table and checkpoint. Verify the clean skip @@ -73,23 +70,23 @@ class test_checkpoint08(wttest.WiredTigerTestCase): self.session.begin_transaction() c1[1] = 1 c2[1] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() c1[1] = 10 c2[1] = 10 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(3)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(3)) self.session.checkpoint(None) # Modify the both tables and reverify. self.session.begin_transaction() c1[3] = 3 c2[3] = 3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(4)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(4)) self.session.checkpoint(None) val = self.get_stat(self.uri1) @@ -100,13 +97,13 @@ class test_checkpoint08(wttest.WiredTigerTestCase): self.assertNotEqual(hsval, 0) # Modify the both tables and reverify when oldest timestamp moved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(4)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() c1[4] = 4 c2[4] = 4 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) self.session.checkpoint(None) val = self.get_stat(self.uri1) diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot01.py b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot01.py index ce839469ed1..0bc38abb5d2 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot01.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot01.py @@ -42,7 +42,7 @@ from wiredtiger import stat # class test_checkpoint_snapshot01(wttest.WiredTigerTestCase): - conn_config = 'cache_size=50MB,statistics=(fast)' + conn_config = 'cache_size=50MB' # Create a table. uri = "table:test_checkpoint_snapshot01" diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py index 2f93fef82b2..9f796b28089 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py @@ -38,9 +38,6 @@ from wiredtiger import stat # This test is to run checkpoint and eviction in parallel with timing # stress for checkpoint and let eviction write more data than checkpoint. # - -def timestamp_str(t): - return '%x' % t class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): # Create a table. @@ -68,7 +65,7 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): if commit_ts == 0: session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, check_value, uri, nrows, read_ts): @@ -76,7 +73,7 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): if read_ts == 0: session.begin_transaction() else: - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -137,8 +134,8 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): valuea = "aaaaa" * 100 # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) self.large_updates(self.uri, valuea, ds, self.nrows, 20) self.check(valuea, self.uri, self.nrows, 20) @@ -151,10 +148,10 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): cursor1.set_key(ds.key(i)) cursor1.set_value(valuea) self.assertEqual(cursor1.insert(), 0) - session1.timestamp_transaction('commit_timestamp=' + timestamp_str(30)) + session1.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) # Set stable timestamp to 40 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) # Create a checkpoint thread done = threading.Event() @@ -191,8 +188,8 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): valueb = "bbbbb" * 100 # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) session1 = self.conn.open_session() session1.begin_transaction() @@ -208,10 +205,10 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): cursor2.set_key(ds.key(i)) cursor2.set_value(valuea) self.assertEqual(cursor2.insert(), 0) - session1.timestamp_transaction('commit_timestamp=' + timestamp_str(30)) + session1.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) # Set stable timestamp to 40 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) # Create a checkpoint thread done = threading.Event() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot04.py b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot04.py index 70ce703c333..0bc908c26fc 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot04.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot04.py @@ -50,7 +50,7 @@ class test_checkpoint_snapshot04(backup_base): scenarios = make_scenarios(target_backup) def conn_config(self): - config = 'cache_size=25MB' + config = 'cache_size=200MB' return config def large_updates(self, uri, value, ds, nrows): diff --git a/src/third_party/wiredtiger/test/suite/test_compress02.py b/src/third_party/wiredtiger/test/suite/test_compress02.py index a88b8f54304..e2ce0dd3a33 100644 --- a/src/third_party/wiredtiger/test/suite/test_compress02.py +++ b/src/third_party/wiredtiger/test/suite/test_compress02.py @@ -39,11 +39,7 @@ from wiredtiger import stat # we are using zstd as the block compressor. Tables created before reconfiguration # will still use the previous compression level. # - -def timestamp_str(t): - return '%x' % t class test_compress02(wttest.WiredTigerTestCase): - # Create a table. uri = "table:test_compress02" nrows = 1000 diff --git a/src/third_party/wiredtiger/test/suite/test_config07.py b/src/third_party/wiredtiger/test/suite/test_config07.py index 6539178cb7e..33789d518a0 100644 --- a/src/third_party/wiredtiger/test/suite/test_config07.py +++ b/src/third_party/wiredtiger/test/suite/test_config07.py @@ -83,7 +83,7 @@ class test_config07(wttest.WiredTigerTestCase): msg = '/invalid log extend length/' config = 'log=(enabled,file_max=1M),file_extend=' + self.log_extend_len - configarg = 'create,statistics=(fast)' + ',' + config + configarg = 'create,' + config # Expect an error when an invalid log extend size is provided. if self.expected_log_size is None: diff --git a/src/third_party/wiredtiger/test/suite/test_cursor15.py b/src/third_party/wiredtiger/test/suite/test_cursor15.py index ab136333682..1a950b7fc51 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor15.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor15.py @@ -37,7 +37,7 @@ class test_cursor15(wttest.WiredTigerTestCase): tablename = 'test_read_once' uri = 'table:' + tablename - conn_config = 'cache_size=1M,statistics=(all)' + conn_config = 'cache_size=1M' def test_cursor15(self): # This test is configured to use 1MB of cache. It will insert 20 @@ -51,7 +51,7 @@ class test_cursor15(wttest.WiredTigerTestCase): cursor[key] = '1' * (100 * 1024) cursor.close() - # Restart the database to clear the cache and reset statistics. + # Restart the database to clear the cache. self.reopen_conn() # We don't restart the database between runs to exercise that read_once diff --git a/src/third_party/wiredtiger/test/suite/test_cursor17.py b/src/third_party/wiredtiger/test/suite/test_cursor17.py index f068279f8f1..60cad85a5e2 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor17.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor17.py @@ -30,9 +30,6 @@ import wiredtiger, wttest from wtscenario import make_scenarios from wiredtiger import stat, WT_NOTFOUND -def timestamp_str(t): - return '%x' % t - # test_cursor17.py # Test the cursor traversal optimization for delete heavy workloads. This optimization enables # cursor traversal mechanism to skip pages where all records on the page are deleted with a @@ -60,8 +57,8 @@ class test_cursor17(wttest.WiredTigerTestCase): total_keys = 40000 # Keep the oldest and the stable timestamp pinned. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(2)) cursor = self.session.open_cursor(uri) commit_timestamp = 3 @@ -70,7 +67,7 @@ class test_cursor17(wttest.WiredTigerTestCase): for key in range(total_keys): self.session.begin_transaction() cursor[key] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_timestamp)) commit_timestamp += 1 # Delete everything on the table except for the first and the last KV pair. @@ -78,13 +75,13 @@ class test_cursor17(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(key) self.assertEqual(cursor.remove(),0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_timestamp)) commit_timestamp += 1 # Take a checkpoint to reconcile the pages. self.session.checkpoint() - self.session.begin_transaction('read_timestamp=' + timestamp_str(commit_timestamp)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(commit_timestamp)) # Position the cursor on the first record. cursor.set_key(0) self.assertEqual(cursor.search(), 0) @@ -107,11 +104,11 @@ class test_cursor17(wttest.WiredTigerTestCase): # Update a key in the middle of the table. self.session.begin_transaction() cursor[total_keys // 2] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_timestamp)) commit_timestamp += 1 # Make sure we can reach a the record we updated in the middle of the table. - self.session.begin_transaction('read_timestamp=' + timestamp_str(commit_timestamp)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(commit_timestamp)) # Position the cursor on the first record. cursor.set_key(0) self.assertEqual(cursor.search(), 0) diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode05.py b/src/third_party/wiredtiger/test/suite/test_debug_mode05.py index f322dd98f7a..bb480138997 100644 --- a/src/third_party/wiredtiger/test/suite/test_debug_mode05.py +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode05.py @@ -28,9 +28,6 @@ import wiredtiger, wttest -def timestamp_str(t): - return '%x' %t - # test_debug_mode05.py # As per WT-5046, the debug table logging settings prevent rollback to # stable in the presence of prepared transactions. @@ -44,7 +41,7 @@ class test_debug_mode05(wttest.WiredTigerTestCase): def test_table_logging_rollback_to_stable(self): self.session.create(self.uri, 'key_format=i,value_format=u,log=(enabled=false)') - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100)) self.session.checkpoint() # Try doing a normal prepared txn and then rollback to stable. @@ -53,11 +50,11 @@ class test_debug_mode05(wttest.WiredTigerTestCase): for i in range(1, 50): cursor[i] = b'a' * 100 self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(150)) + 'prepare_timestamp=' + self.timestamp_str(150)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(200)) + 'commit_timestamp=' + self.timestamp_str(200)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(250)) + 'durable_timestamp=' + self.timestamp_str(250)) self.session.commit_transaction() cursor.close() @@ -70,11 +67,11 @@ class test_debug_mode05(wttest.WiredTigerTestCase): # Therefore, we're specifically not doing any operations here. self.session.begin_transaction() self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(300)) + 'prepare_timestamp=' + self.timestamp_str(300)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(350)) + 'commit_timestamp=' + self.timestamp_str(350)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(400)) + 'durable_timestamp=' + self.timestamp_str(400)) self.session.commit_transaction() # The aforementioned bug resulted in a failure in rollback to stable. @@ -88,7 +85,7 @@ class test_debug_mode05(wttest.WiredTigerTestCase): for i in range(1, 50): cursor[i] = b'b' * 100 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(450)) + 'commit_timestamp=' + self.timestamp_str(450)) cursor.close() self.conn.rollback_to_stable() diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode07.py b/src/third_party/wiredtiger/test/suite/test_debug_mode07.py index c957fe57799..6c1432937fe 100644 --- a/src/third_party/wiredtiger/test/suite/test_debug_mode07.py +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode07.py @@ -31,7 +31,7 @@ import wttest, wiredtiger # test_debug_mode07.py # Test the debug mode settings. Test realloc_exact use (from WT-4919). class test_debug_mode07(wttest.WiredTigerTestCase): - conn_config = 'log=(enabled=true),debug_mode=(realloc_exact=true),statistics=(all)' + conn_config = 'log=(enabled=true),debug_mode=(realloc_exact=true)' uri = 'file:test_debug_mode07' # Insert some data to ensure setting/unsetting the flag does not diff --git a/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py b/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py index aa5ed0aae65..dfad0d8869d 100644 --- a/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py @@ -32,9 +32,6 @@ from suite_subprocess import suite_subprocess from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' %t - # test_durable_rollback_to_stable.py # Checking visibility and durability of updates with durable_timestamp and # with rollback to stable. @@ -44,8 +41,7 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces keyfmt = [ ('row-string', dict(keyfmt='S')), ('row-int', dict(keyfmt='i')), - # The commented columnar tests needs to be enabled once rollback to stable for columnar is fixed in (WT-5548). - # ('column-store', dict(keyfmt='r')), + ('column-store', dict(keyfmt='r')), ] types = [ ('file', dict(uri='file', ds=SimpleDataSet)), @@ -78,7 +74,7 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces cursor = session.open_cursor(uri, None) # Set stable timestamp to checkpoint initial data set. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100)) self.session.checkpoint() # Update all values with value 111 i.e. first update value. @@ -89,15 +85,15 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(220)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(200)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(220)) session.commit_transaction() # Check the values read are correct with different timestamps. # Read the initial dataset. self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(150)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(150)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(i)) @@ -106,7 +102,7 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces # Read the first update value with timestamp. self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(200)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(200)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(111)) @@ -123,7 +119,7 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces session.commit_transaction() # Set a stable timestamp so that first update value is durable. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) # Update all values with value 222 i.e. second update value. self.assertEquals(cursor.reset(), 0) @@ -134,12 +130,12 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(200)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(200)) # Commit timestamp is earlier to stable timestamp but durable timestamp # is later than stable timestamp. Hence second update value is not durable. - session.timestamp_transaction('commit_timestamp=' + timestamp_str(240)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(300)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(240)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(300)) session.commit_transaction() # Checkpoint so that first update value will be visible and durable, @@ -160,8 +156,8 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces self.conn.rollback_to_stable() session = self.conn.open_session(self.session_config) cursor = session.open_cursor(uri, None) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(250)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(111)) diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts01.py b/src/third_party/wiredtiger/test/suite/test_durable_ts01.py index 7195aa85569..cd20fbf6b35 100644 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts01.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts01.py @@ -31,9 +31,6 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' %t - # test_durable_ts01.py # Checking visibility and durability of updates with durable_timestamp and # with restart. @@ -43,8 +40,7 @@ class test_durable_ts01(wttest.WiredTigerTestCase): keyfmt = [ ('row-string', dict(keyfmt='S')), ('row-int', dict(keyfmt='i')), - # The commented columnar tests needs to be enabled once rollback to stable for columnar is fixed in (WT-5548). - # ('column-store', dict(keyfmt='r')), + ('column-store', dict(keyfmt='r')), ] types = [ ('file', dict(uri='file', ds=SimpleDataSet)), @@ -77,7 +73,7 @@ class test_durable_ts01(wttest.WiredTigerTestCase): cursor = session.open_cursor(uri, None) # Set stable timestamp to checkpoint initial data set. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100)) self.session.checkpoint() # Update all values with value 111 i.e. first update value. @@ -88,15 +84,15 @@ class test_durable_ts01(wttest.WiredTigerTestCase): self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(220)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(200)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(220)) session.commit_transaction() # Check the values read are correct with different timestamps. # Read the initial dataset. self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(150)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(150)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(i)) @@ -105,7 +101,7 @@ class test_durable_ts01(wttest.WiredTigerTestCase): # Read the first update value with timestamp. self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(200)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(200)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(111)) @@ -122,7 +118,7 @@ class test_durable_ts01(wttest.WiredTigerTestCase): session.commit_transaction() # Set a stable timestamp so that first update value is durable. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) # Update all values with value 222 i.e. second update value. self.assertEquals(cursor.reset(), 0) @@ -133,12 +129,12 @@ class test_durable_ts01(wttest.WiredTigerTestCase): self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(200)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(200)) # Commit timestamp is earlier to stable timestamp but durable timestamp # is later than stable timestamp. Hence second update value is not durable. - session.timestamp_transaction('commit_timestamp=' + timestamp_str(240)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(300)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(240)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(300)) session.commit_transaction() # Checkpoint so that first update value will be visible and durable, @@ -159,8 +155,8 @@ class test_durable_ts01(wttest.WiredTigerTestCase): self.reopen_conn() session = self.conn.open_session(self.session_config) cursor = session.open_cursor(uri, None) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(250)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(111)) diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts02.py b/src/third_party/wiredtiger/test/suite/test_durable_ts02.py index b71743c232c..88dea1291a2 100644 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts02.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts02.py @@ -31,9 +31,6 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' %t - # test_durable_ts03.py # Checking visibility and durability of updates with durable_timestamp class test_durable_ts03(wttest.WiredTigerTestCase): @@ -75,7 +72,7 @@ class test_durable_ts03(wttest.WiredTigerTestCase): cursor = session.open_cursor(uri, None) # Set stable timestamp to checkpoint initial data set. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100)) self.session.checkpoint() ''' @@ -90,15 +87,15 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) msg = "/is less than the commit timestamp/" # Check for error when commit timestamp > durable timestamp. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: session.commit_transaction('commit_timestamp=' +\ - timestamp_str(200) + ',durable_timestamp=' + timestamp_str(180)), msg) + self.timestamp_str(200) + ',durable_timestamp=' + self.timestamp_str(180)), msg) # Set a stable timestamp so that first update value is durable. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) # Scenario: 2 # Check to see durable timestamp < stable timestamp, returns error. @@ -111,13 +108,13 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) msg = "/is less than the stable timestamp/" # Check that error is returned when durable timestamp < stable timestamp. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: session.commit_transaction('commit_timestamp=' +\ - timestamp_str(200) + ',durable_timestamp=' + timestamp_str(240)), msg) + self.timestamp_str(200) + ',durable_timestamp=' + self.timestamp_str(240)), msg) ''' if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py index 298c607904d..deea2f3dfc5 100755 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py @@ -29,9 +29,6 @@ from helper import copy_wiredtiger_home import wiredtiger, wttest -def timestamp_str(t): - return '%x' %t - # test_durable_ts03.py # Check that the checkpoint honors the durable timestamp of updates. class test_durable_ts03(wttest.WiredTigerTestCase): @@ -48,8 +45,8 @@ class test_durable_ts03(wttest.WiredTigerTestCase): valueC = b"ccccc" * 100 # Start with setting a stable and oldest timestamp. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1) + \ - ',oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1) + \ + ',oldest_timestamp=' + self.timestamp_str(1)) # Load the data into the table. session = self.conn.open_session(self.session_config) @@ -57,12 +54,12 @@ class test_durable_ts03(wttest.WiredTigerTestCase): for i in range(0, nrows): session.begin_transaction() cursor[i] = valueA - session.commit_transaction('commit_timestamp=' + timestamp_str(50)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) cursor.close() # Set the stable and the oldest timestamp to checkpoint initial data. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100) + \ - ',oldest_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100) + \ + ',oldest_timestamp=' + self.timestamp_str(100)) self.session.checkpoint() # Update all the values within transaction. Commit the transaction with @@ -71,9 +68,9 @@ class test_durable_ts03(wttest.WiredTigerTestCase): for i in range(0, nrows): session.begin_transaction() cursor[i] = valueB - session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(220)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(200)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(220)) session.commit_transaction() # Check the checkpoint wrote only the durable updates. @@ -83,14 +80,14 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.assertEqual(value, valueA) self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(150)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(150)) for key, value in cursor: self.assertEqual(value, valueA) session.commit_transaction() # Read the updated data to confirm that it is visible. self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(210)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(210)) for key, value in cursor: self.assertEqual(value, valueB) session.commit_transaction() @@ -102,8 +99,8 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.reopen_conn() session = self.conn.open_session(self.session_config) cursor = session.open_cursor(uri, None) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(210) + \ - ',oldest_timestamp=' + timestamp_str(210)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(210) + \ + ',oldest_timestamp=' + self.timestamp_str(210)) for key, value in cursor: self.assertEqual(value, valueA) @@ -111,12 +108,12 @@ class test_durable_ts03(wttest.WiredTigerTestCase): for i in range(0, nrows): session.begin_transaction() cursor[i] = valueC - session.prepare_transaction('prepare_timestamp=' + timestamp_str(220)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(230)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(240)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(220)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(230)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(240)) session.commit_transaction() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) self.session.checkpoint() cursor.close() session.close() @@ -124,8 +121,8 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.reopen_conn() session = self.conn.open_session(self.session_config) cursor = session.open_cursor(uri, None) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250) + \ - ',oldest_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250) + \ + ',oldest_timestamp=' + self.timestamp_str(250)) for key, value in cursor: self.assertEqual(value, valueC) diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt08.py b/src/third_party/wiredtiger/test/suite/test_encrypt08.py index e15dd851908..5e4e065d3f3 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt08.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt08.py @@ -84,11 +84,10 @@ class test_encrypt08(wttest.WiredTigerTestCase): def test_encrypt(self): sysconfig = 'encryption=(name=sodium,{0}),'.format(self.sys_encrypt) - with self.expectedStdoutPattern('Failed wiredtiger_open'): - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: - self.reopen_conn(config = sysconfig), - self.msg) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: + self.reopen_conn(config = sysconfig), + self.msg) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_gc01.py b/src/third_party/wiredtiger/test/suite/test_gc01.py index 950c1e4ce54..8cec5461881 100755 --- a/src/third_party/wiredtiger/test/suite/test_gc01.py +++ b/src/third_party/wiredtiger/test/suite/test_gc01.py @@ -36,11 +36,7 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet from wiredtiger import stat -def timestamp_str(t): - return '%x' % t - # test_gc01.py - # Shared base class used by gc tests. class test_gc_base(wttest.WiredTigerTestCase): @@ -51,7 +47,7 @@ class test_gc_base(wttest.WiredTigerTestCase): for i in range(0, nrows): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def large_modifies(self, uri, value, ds, location, nbytes, nrows, commit_ts): @@ -63,12 +59,12 @@ class test_gc_base(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify(value, location, nbytes)] self.assertEqual(cursor.modify(mods), 0) - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, check_value, uri, nrows, read_ts): session = self.session - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -100,8 +96,8 @@ class test_gc01(test_gc_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 bigvalue2 = "ddddd" * 100 @@ -119,8 +115,8 @@ class test_gc01(test_gc_base): self.check(bigvalue, uri, nrows, 10) # Pin oldest and stable to timestamp 100. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100) + - ',stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100) + + ',stable_timestamp=' + self.timestamp_str(100)) # Checkpoint to ensure that the history store is cleaned. self.session.checkpoint() @@ -152,8 +148,8 @@ class test_gc01(test_gc_base): self.check(bigvalue2, uri, nrows, 100) # Pin oldest and stable to timestamp 200. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(200) + - ',stable_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(200) + + ',stable_timestamp=' + self.timestamp_str(200)) # Checkpoint to ensure that the history store is cleaned. self.session.checkpoint() @@ -185,8 +181,8 @@ class test_gc01(test_gc_base): self.check(bigvalue, uri, nrows, 200) # Pin oldest and stable to timestamp 300. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(300) + - ',stable_timestamp=' + timestamp_str(300)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(300) + + ',stable_timestamp=' + self.timestamp_str(300)) # Checkpoint to ensure that the history store is cleaned. self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_gc02.py b/src/third_party/wiredtiger/test/suite/test_gc02.py index b42d3e23a47..7af18d02186 100755 --- a/src/third_party/wiredtiger/test/suite/test_gc02.py +++ b/src/third_party/wiredtiger/test/suite/test_gc02.py @@ -30,9 +30,6 @@ from test_gc01 import test_gc_base from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - # test_gc02.py # Test that checkpoint cleans the obsolete history store internal pages. class test_gc02(test_gc_base): @@ -49,8 +46,8 @@ class test_gc02(test_gc_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 bigvalue2 = "ddddd" * 100 @@ -76,8 +73,8 @@ class test_gc02(test_gc_base): c.close() # Pin oldest and stable to timestamp 100. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100) + - ',stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100) + + ',stable_timestamp=' + self.timestamp_str(100)) # Check that the new updates are only seen after the update timestamp. self.check(bigvalue2, uri, nrows, 100) @@ -114,8 +111,8 @@ class test_gc02(test_gc_base): self.check(bigvalue, uri, nrows, 200) # Pin oldest and stable to timestamp 200. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(200) + - ',stable_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(200) + + ',stable_timestamp=' + self.timestamp_str(200)) # Checkpoint to ensure that the history store is cleaned. self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_gc03.py b/src/third_party/wiredtiger/test/suite/test_gc03.py index c605a07963e..386a7bea801 100755 --- a/src/third_party/wiredtiger/test/suite/test_gc03.py +++ b/src/third_party/wiredtiger/test/suite/test_gc03.py @@ -30,9 +30,6 @@ from test_gc01 import test_gc_base from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - # test_gc03.py # Test that checkpoint cleans the obsolete history store pages that are in-memory. class test_gc03(test_gc_base): @@ -61,8 +58,8 @@ class test_gc03(test_gc_base): ds_extra.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 bigvalue2 = "ddddd" * 100 @@ -84,8 +81,8 @@ class test_gc03(test_gc_base): self.assertGreater(self.get_stat(stat.conn.cc_pages_visited), 0) # Pin oldest and stable to timestamp 100. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100) + - ',stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100) + + ',stable_timestamp=' + self.timestamp_str(100)) # Check that the new updates are only seen after the update timestamp. self.check(bigvalue2, uri, nrows, 100) @@ -113,8 +110,8 @@ class test_gc03(test_gc_base): self.check(bigvalue2, uri, nrows, 100) # Pin oldest and stable to timestamp 200. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(200) + - ',stable_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(200) + + ',stable_timestamp=' + self.timestamp_str(200)) # Update on extra table. self.large_updates(uri_extra, bigvalue, ds_extra, 100, 210) @@ -129,8 +126,8 @@ class test_gc03(test_gc_base): self.check(bigvalue, uri, nrows, 200) # Pin oldest and stable to timestamp 300. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(300) + - ',stable_timestamp=' + timestamp_str(300)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(300) + + ',stable_timestamp=' + self.timestamp_str(300)) self.large_updates(uri_extra, bigvalue, ds_extra, 100, 310) self.large_updates(uri_extra, bigvalue2, ds_extra, 100, 320) diff --git a/src/third_party/wiredtiger/test/suite/test_gc04.py b/src/third_party/wiredtiger/test/suite/test_gc04.py index 1db0cceb24f..4504bd3f75e 100755 --- a/src/third_party/wiredtiger/test/suite/test_gc04.py +++ b/src/third_party/wiredtiger/test/suite/test_gc04.py @@ -30,9 +30,6 @@ from test_gc01 import test_gc_base from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - # test_gc04.py # Test that checkpoint must not clean the pages that are not obsolete. class test_gc04(test_gc_base): @@ -55,8 +52,8 @@ class test_gc04(test_gc_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 bigvalue2 = "ddddd" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_gc05.py b/src/third_party/wiredtiger/test/suite/test_gc05.py index 1d34ce34dc6..75bf3d253e8 100755 --- a/src/third_party/wiredtiger/test/suite/test_gc05.py +++ b/src/third_party/wiredtiger/test/suite/test_gc05.py @@ -29,9 +29,6 @@ from test_gc01 import test_gc_base from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - # test_gc05.py # Verify a locked checkpoint is not removed during garbage collection. class test_gc05(test_gc_base): @@ -52,8 +49,8 @@ class test_gc05(test_gc_base): ds.populate() # Set the oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) # Insert values with varying timestamps. self.large_updates(uri, value_x, ds, nrows, 20) @@ -70,8 +67,8 @@ class test_gc05(test_gc_base): ckpt_cursor = self.session.open_cursor(uri, None, "checkpoint=checkpoint_one") # Move the oldest and stable timestamps to 40. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(40) + - ',stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(40) + + ',stable_timestamp=' + self.timestamp_str(40)) # Insert values with varying timestamps. self.large_updates(uri, value_z, ds, nrows, 50) @@ -79,8 +76,8 @@ class test_gc05(test_gc_base): self.large_updates(uri, value_x, ds, nrows, 70) # Move the oldest and stable timestamps to 70. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(70) + - ',stable_timestamp=' + timestamp_str(70)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(70) + + ',stable_timestamp=' + self.timestamp_str(70)) # Perform a checkpoint. self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_hs01.py b/src/third_party/wiredtiger/test/suite/test_hs01.py index 90a7be1cd0b..52c68cb699d 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs01.py +++ b/src/third_party/wiredtiger/test/suite/test_hs01.py @@ -28,72 +28,70 @@ from helper import copy_wiredtiger_home import wiredtiger, wttest +from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs01.py -# Smoke tests to ensure history store tables are working. +# Test that update and modify operations are durable across crash and recovery. +# Additionally test that checkpoint inserts content into the history store. class test_hs01(wttest.WiredTigerTestCase): - # Force a small cache. - conn_config = 'cache_size=50MB' + conn_config = 'cache_size=200MB,statistics=(all)' session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), - ('string', dict(key_format='S')) + ('row_integer', dict(key_format='i')), + ('row_string', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) + def get_stat(self, stat): + stat_cursor = self.session.open_cursor('statistics:') + val = stat_cursor[stat][2] + stat_cursor.close() + return val + def large_updates(self, session, uri, value, ds, nrows, timestamp=False): - # Update a large number of records, we'll hang if the history store table - # isn't doing its thing. cursor = session.open_cursor(uri) - for i in range(1, 10000): + for i in range(1, nrows): if timestamp == True: session.begin_transaction() - cursor.set_key(ds.key(nrows + i)) + cursor.set_key(ds.key(i)) cursor.set_value(value) self.assertEqual(cursor.update(), 0) if timestamp == True: - session.commit_transaction('commit_timestamp=' + timestamp_str(i + 1)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(i + 1)) cursor.close() def large_modifies(self, session, uri, offset, ds, nrows, timestamp=False): - # Modify a large number of records, we'll hang if the history store table - # isn't doing its thing. cursor = session.open_cursor(uri) - for i in range(1, 10000): - if timestamp == True: - session.begin_transaction() - cursor.set_key(ds.key(nrows + i)) + for i in range(1, nrows): + # Unlike inserts and updates, modify operations do not implicitly start/commit a transaction. + # Hence, we begin/commit transaction manually. + session.begin_transaction() + cursor.set_key(ds.key(i)) mods = [] mod = wiredtiger.Modify('A', offset, 1) mods.append(mod) - self.assertEqual(cursor.modify(mods), 0) + if timestamp == True: - session.commit_transaction('commit_timestamp=' + timestamp_str(i + 1)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(i + 1)) + else: + session.commit_transaction() cursor.close() - def durable_check(self, check_value, uri, ds, nrows): - # Checkpoint and backup so as to simulate recovery - self.session.checkpoint() + def durable_check(self, check_value, uri, ds): + # Simulating recovery. newdir = "BACKUP" copy_wiredtiger_home(self, '.', newdir, True) - conn = self.setUpConnectionOpen(newdir) session = self.setUpSessionOpen(conn) cursor = session.open_cursor(uri, None) - # Skip the initial rows, which were not updated - for i in range(0, nrows+1): - self.assertEqual(cursor.next(), 0) - if check_value != cursor.get_value(): - session.breakpoint() + + cursor.next() self.assertTrue(check_value == cursor.get_value(), - "for key " + str(i) + ", expected " + str(check_value) + + "for key " + str(1) + ", expected " + str(check_value) + ", got " + str(cursor.get_value())) cursor.close() session.close() @@ -102,15 +100,15 @@ class test_hs01(wttest.WiredTigerTestCase): def test_hs(self): # Create a small table. uri = "table:test_hs01" - nrows = 100 - ds = SimpleDataSet(self, uri, nrows, key_format=self.key_format, value_format='u') + ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format='u') ds.populate() - bigvalue = b"aaaaa" * 100 - # Initially load huge data. + # Initially insert a lot of data. + nrows = 10000 + bigvalue = b"aaaaa" * 100 cursor = self.session.open_cursor(uri) - for i in range(1, 10000): - cursor.set_key(ds.key(nrows + i)) + for i in range(1, nrows): + cursor.set_key(ds.key(i)) cursor.set_value(bigvalue) self.assertEqual(cursor.insert(), 0) cursor.close() @@ -119,11 +117,19 @@ class test_hs01(wttest.WiredTigerTestCase): # Scenario: 1 # Check to see if the history store is working with the old reader. bigvalue2 = b"ccccc" * 100 + # Open session 2. session2 = self.conn.open_session() session2.begin_transaction('isolation=snapshot') + # Large updates with session 1. self.large_updates(self.session, uri, bigvalue2, ds, nrows) - # Check to see the value after recovery. - self.durable_check(bigvalue2, uri, ds, nrows) + + # Checkpoint and then assert that the (nrows-1) insertions were moved to history store from data store. + self.session.checkpoint() + hs_writes = self.get_stat(stat.conn.cache_hs_insert) + self.assertEqual(hs_writes, nrows-1) + + # Check to see the latest updated value after recovery. + self.durable_check(bigvalue2, uri, ds) session2.rollback_transaction() session2.close() @@ -131,34 +137,48 @@ class test_hs01(wttest.WiredTigerTestCase): # Check to see the history store working with modify operations. bigvalue3 = b"ccccc" * 100 bigvalue3 = b'AA' + bigvalue3[2:] + # Open session 2. session2 = self.conn.open_session() session2.begin_transaction('isolation=snapshot') - # Apply two modify operations - replacing the first two items with 'A'. - self.session.begin_transaction() + # Apply two modify operations (session1)- replacing the first two items with 'A'. self.large_modifies(self.session, uri, 0, ds, nrows) self.large_modifies(self.session, uri, 1, ds, nrows) - self.session.commit_transaction() - # Check to see the value after recovery. - self.durable_check(bigvalue3, uri, ds, nrows) + + # Checkpoint and then assert if updates (nrows-1) and first large modifies (nrows-1) were moved to history store. + self.session.checkpoint() + hs_writes = self.get_stat(stat.conn.cache_hs_insert) + # The updates in data store: nrows-1 + # The first modifies in cache: nrows-1 + # The stats was already set at: nrows-1 (previous hs stats) + # Total: (nrows-1)*3 + self.assertEqual(hs_writes, (nrows-1) * 3) + + # Check to see the modified value after recovery. + self.durable_check(bigvalue3, uri, ds) session2.rollback_transaction() session2.close() - # FIXME-WT-7120: Rollback to stable support for column store is not implemented, and it - # fails only when it is used with timestamps. - if self.key_format == 'r': - return - # Scenario: 3 # Check to see if the history store is working with the old timestamp. bigvalue4 = b"ddddd" * 100 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) self.large_updates(self.session, uri, bigvalue4, ds, nrows, timestamp=True) - # Check to see data can be see only till the stable_timestamp - self.durable_check(bigvalue3, uri, ds, nrows) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(i + 1)) + self.session.checkpoint() + # Check if the (nrows-1) modifications were moved to history store from data store. + # The stats was already set at: (nrows-1)*3 (previous hs stats) + # Total: (nrows-1)*4 + hs_writes = self.get_stat(stat.conn.cache_hs_insert) + self.assertEqual(hs_writes, (nrows-1) * 4) + + # Check to see data can be see only till the stable_timestamp. + self.durable_check(bigvalue3, uri, ds) + + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(i + 1)) + # No need to check history store stats here as nothing will be moved. + self.session.checkpoint() # Check that the latest data can be seen. - self.durable_check(bigvalue4, uri, ds, nrows) + self.durable_check(bigvalue4, uri, ds) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_hs02.py b/src/third_party/wiredtiger/test/suite/test_hs02.py index 6f92f422161..42cd3b327ce 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs02.py +++ b/src/third_party/wiredtiger/test/suite/test_hs02.py @@ -31,9 +31,6 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs02.py # Test that truncate with history store entries and timestamps gives expected results. class test_hs02(wttest.WiredTigerTestCase): @@ -42,9 +39,8 @@ class test_hs02(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ - ('string', dict(key_format='S')), - # FIXME-WT-7120: Uncomment the column store scenario when rollback to stable support is complete. - # ('column', dict(key_format='r')) + ('string-row', dict(key_format='S')), + ('column', dict(key_format='r')) ] scenarios = make_scenarios(key_format_values) @@ -55,12 +51,12 @@ class test_hs02(wttest.WiredTigerTestCase): for i in range(1, nrows + 1): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, check_value, uri, nrows, read_ts): session = self.session - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -84,8 +80,8 @@ class test_hs02(wttest.WiredTigerTestCase): ds2.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 self.large_updates(uri, bigvalue, ds, nrows // 3, 1) @@ -110,7 +106,7 @@ class test_hs02(wttest.WiredTigerTestCase): end.set_key(ds.key(nrows // 2)) self.session.truncate(None, None, end) end.close() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(200)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(200)) # Check that the truncate is visible after commit self.check(bigvalue2, uri, nrows // 2, 200) diff --git a/src/third_party/wiredtiger/test/suite/test_hs03.py b/src/third_party/wiredtiger/test/suite/test_hs03.py index 6980859ced6..54a54bf1f0f 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs03.py +++ b/src/third_party/wiredtiger/test/suite/test_hs03.py @@ -32,9 +32,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs03.py # Ensure checkpoints don't read too unnecessary history store entries. class test_hs03(wttest.WiredTigerTestCase): @@ -43,8 +40,8 @@ class test_hs03(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), - ('string', dict(key_format='S')) + ('integer-row', dict(key_format='i')), + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -61,7 +58,7 @@ class test_hs03(wttest.WiredTigerTestCase): for i in range(nrows + 1, nrows + nops + 1): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(i)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(i)) cursor.close() def test_checkpoint_hs_reads(self): @@ -81,7 +78,7 @@ class test_hs03(wttest.WiredTigerTestCase): # Check to see the history store working with old timestamp. bigvalue2 = b"ddddd" * 100 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) hs_writes_start = self.get_stat(stat.conn.cache_write_hs) self.large_updates(self.session, uri, bigvalue2, ds, nrows, 10000) @@ -91,7 +88,7 @@ class test_hs03(wttest.WiredTigerTestCase): self.assertGreaterEqual(hs_writes, 0) for ts in range(2, 4): - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(ts)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(ts)) # Now just update one record and checkpoint again. self.large_updates(self.session, uri, bigvalue2, ds, nrows, 1) diff --git a/src/third_party/wiredtiger/test/suite/test_hs05.py b/src/third_party/wiredtiger/test/suite/test_hs05.py index 5a66d04f546..6f869e83aaa 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs05.py +++ b/src/third_party/wiredtiger/test/suite/test_hs05.py @@ -32,9 +32,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs05.py # Verify hs_score reflects cache pressure due to history # even if we're not yet actively pushing into the history store file. @@ -47,8 +44,8 @@ class test_hs05(wttest.WiredTigerTestCase): stable = 1 key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), - ('string', dict(key_format='S')) + ('integer-row', dict(key_format='i')), + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -66,7 +63,7 @@ class test_hs05(wttest.WiredTigerTestCase): for i in range(nrows + 1, nrows + nops + 1): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(self.stable + i)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(self.stable + i)) cursor.close() score_end = self.get_stat(stat.conn.cache_hs_score) score_diff = score_end - score_start @@ -93,7 +90,7 @@ class test_hs05(wttest.WiredTigerTestCase): self.session.checkpoint() # Pin the oldest timestamp so that all history has to stay. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) # Loop a couple times, partly filling the cache but not # overfilling it to see the history store score value change # even if the history store is not yet in use. @@ -103,7 +100,7 @@ class test_hs05(wttest.WiredTigerTestCase): loop_start = self.get_stat(stat.conn.cache_hs_score) for i in range(1, 9): bigvalue2 = valstr[i].encode() * 50 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(self.stable)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(self.stable)) entries_start = self.get_stat(stat.conn.cache_hs_insert) score_start = self.get_stat(stat.conn.cache_hs_score) self.pr("Update iteration: " + str(i) + " Value: " + str(bigvalue2)) @@ -127,14 +124,14 @@ class test_hs05(wttest.WiredTigerTestCase): # By moving the oldest after updating we should see the score drop # to zero. score_start = loop_end - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(self.stable)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(self.stable)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(self.stable)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(self.stable)) for i in range(9, 11): bigvalue2 = valstr[i].encode() * 50 self.pr("Update iteration with oldest: " + str(i) + " Value: " + str(bigvalue2)) self.large_updates(self.session, uri, bigvalue2, ds, nrows, nrows) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(self.stable)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(self.stable)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(self.stable)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(self.stable)) self.stable += nrows score_end = self.get_stat(stat.conn.cache_hs_score) self.assertLess(score_end, score_start) diff --git a/src/third_party/wiredtiger/test/suite/test_hs06.py b/src/third_party/wiredtiger/test/suite/test_hs06.py index a7747e281e4..1c2f18f09f2 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs06.py +++ b/src/third_party/wiredtiger/test/suite/test_hs06.py @@ -30,9 +30,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs06.py # Verify that triggering history store usage does not cause a spike in memory usage # to form an update chain from the history store contents. @@ -48,10 +45,11 @@ class test_hs06(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), - ('string', dict(key_format='S')) + ('integer-row', dict(key_format='i')), + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) + nrows = 2000 def get_stat(self, stat): stat_cursor = self.session.open_cursor('statistics:') @@ -77,21 +75,21 @@ class test_hs06(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Load 1Mb of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load another 1Mb of data with a later timestamp. self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Write a version of the data to disk. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(2)) self.session.checkpoint() # Check the checkpoint wrote the expected values. @@ -102,7 +100,7 @@ class test_hs06(wttest.WiredTigerTestCase): # # cursor2 = self.session.open_cursor(uri, None, 'checkpoint=WiredTigerCheckpoint') cursor2 = self.session.open_cursor(uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) for key, value in cursor2: self.assertEqual(value, value1) self.session.commit_transaction() @@ -113,8 +111,8 @@ class test_hs06(wttest.WiredTigerTestCase): # Whenever we request something out of cache of timestamp 2, we should # be reading it straight from the history store without initialising a full # update chain of every version of the data. - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) - for i in range(1, 2000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) + for i in range(1, self.nrows): self.assertEqual(cursor[self.create_key(i)], value1) self.session.rollback_transaction() @@ -143,34 +141,34 @@ class test_hs06(wttest.WiredTigerTestCase): value2 = 'd' * 500 # Load 1Mb of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load a slight modification with a later timestamp. self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor.set_key(self.create_key(i)) mods = [wiredtiger.Modify('B', 100, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # And another. self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor.set_key(self.create_key(i)) mods = [wiredtiger.Modify('C', 200, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Now write something completely different. self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Now the latest version will get written to the data file. self.session.checkpoint() @@ -187,8 +185,8 @@ class test_hs06(wttest.WiredTigerTestCase): # t4: full update in las # t3: (reverse delta in las) <= We're querying for t4 so we begin here. # t2: value2 (full update in las) - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) - for i in range(1, 2000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) + for i in range(1, self.nrows): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() @@ -203,17 +201,12 @@ class test_hs06(wttest.WiredTigerTestCase): # t4: full update in las <= We're querying for t4 and we return. # t3: (reverse delta in las) # t2: value2 (full update in las) - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) - for i in range(1, 2000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) + for i in range(1, self.nrows): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() def test_hs_prepare_reads(self): - # FIXME-WT-6061: Prepare reads currently not supported with columnar store. - # Remove this once prepare reads is supported. - if self.key_format == 'r': - return - # Create a small table. uri = "table:test_hs06" create_params = 'key_format={},value_format=S'.format(self.key_format) @@ -222,12 +215,12 @@ class test_hs06(wttest.WiredTigerTestCase): value1 = 'a' * 500 value2 = 'b' * 500 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - for i in range(1, 2000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load prepared data and leave it in a prepared state. prepare_session = self.conn.open_session(self.session_config) @@ -236,20 +229,20 @@ class test_hs06(wttest.WiredTigerTestCase): for i in range(1, 11): prepare_cursor[self.create_key(i)] = value2 prepare_session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(3)) + 'prepare_timestamp=' + self.timestamp_str(3)) # Write some more to cause eviction of the prepared data. - for i in range(11, 2000): + for i in range(11, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.checkpoint() # Try to read every key of the prepared data again. # Ensure that we read the history store to find the prepared update and # return a prepare conflict as appropriate. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): cursor.set_key(self.create_key(i)) self.assertRaisesException( @@ -259,9 +252,9 @@ class test_hs06(wttest.WiredTigerTestCase): self.session.rollback_transaction() prepare_session.commit_transaction( - 'commit_timestamp=' + timestamp_str(5) + ',durable_timestamp=' + timestamp_str(6)) + 'commit_timestamp=' + self.timestamp_str(5) + ',durable_timestamp=' + self.timestamp_str(6)) - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) for i in range(1, 11): self.assertEquals(value2, cursor[self.create_key(i)]) self.session.rollback_transaction() @@ -278,12 +271,12 @@ class test_hs06(wttest.WiredTigerTestCase): value4 = 'd' * 500 # Load 1Mb of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - for i in range(1, 2000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Do two different updates to the same key with the same timestamp. # We want to make sure that the second value is the one that is visible even after eviction. @@ -291,16 +284,16 @@ class test_hs06(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor[self.create_key(i)] = value2 cursor[self.create_key(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Write a newer value on top. - for i in range(1, 2000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value4 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Ensure that we see the last of the two updates that got applied. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): self.assertEquals(cursor[self.create_key(i)], value3) self.session.rollback_transaction() @@ -315,12 +308,12 @@ class test_hs06(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Load 1Mb of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - for i in range(1, 2000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Apply three sets of modifies. # They specifically need to be in separate modify calls. @@ -330,7 +323,7 @@ class test_hs06(wttest.WiredTigerTestCase): self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) expected = list(value1) expected[100] = 'B' @@ -339,13 +332,13 @@ class test_hs06(wttest.WiredTigerTestCase): expected = str().join(expected) # Write a newer value on top. - for i in range(1, 2000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Go back and read. We should get the initial value with the 3 modifies applied on top. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() @@ -361,25 +354,25 @@ class test_hs06(wttest.WiredTigerTestCase): # Load 5Mb of data. self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) for i in range(1, 10000): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Apply three sets of modifies. for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Since the stable timestamp is still at 1, there will be no birthmark record. # History store instantiation should choose this update since it is the most recent. @@ -388,7 +381,7 @@ class test_hs06(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Make a bunch of updates to another table to flush everything out of cache. uri2 = 'table:test_hs06_extra' @@ -397,7 +390,7 @@ class test_hs06(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor2[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(6)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(6)) expected = list(value1) expected[100] = 'B' @@ -406,7 +399,7 @@ class test_hs06(wttest.WiredTigerTestCase): expected = str().join(expected) # Go back and read. We should get the initial value with the 3 modifies applied on top. - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) for i in range(1, 11): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() @@ -422,7 +415,7 @@ class test_hs06(wttest.WiredTigerTestCase): # Load 5Mb of data. self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) # The base update is at timestamp 1. # When we history store evict these pages, the base update is the only thing behind @@ -431,26 +424,26 @@ class test_hs06(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) # Apply three sets of modifies. for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Make a bunch of updates to another table to flush everything out of cache. uri2 = 'table:test_hs06_extra' @@ -459,7 +452,7 @@ class test_hs06(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor2[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) expected = list(value1) expected[100] = 'B' @@ -468,7 +461,7 @@ class test_hs06(wttest.WiredTigerTestCase): expected = str().join(expected) # Go back and read. - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) for i in range(1, 11): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() @@ -483,44 +476,44 @@ class test_hs06(wttest.WiredTigerTestCase): value2 = 'b' * 500 self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) # Base update. for i in range(1, 10000): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Apply three sets of modifies. for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # This is the one we want to be selected by the checkpoint. for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Apply another update and evict the pages with the modifies out of cache. for i in range(1, 10000): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(6)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(6)) # Checkpoint such that the modifies will be selected. When we grab it from the history # store, we'll need to unflatten it before using it for reconciliation. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) self.session.checkpoint() expected = list(value1) @@ -530,7 +523,7 @@ class test_hs06(wttest.WiredTigerTestCase): expected = str().join(expected) # Check that the correct value is visible after checkpoint. - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) for i in range(1, 11): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs07.py b/src/third_party/wiredtiger/test/suite/test_hs07.py index 58d44374e42..7e2562e5420 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs07.py +++ b/src/third_party/wiredtiger/test/suite/test_hs07.py @@ -32,9 +32,6 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs07.py # Test that the history store sweep cleans the obsolete history store entries and gives expected results. class test_hs07(wttest.WiredTigerTestCase): @@ -45,7 +42,7 @@ class test_hs07(wttest.WiredTigerTestCase): key_format_values = ( ('column', dict(key_format='r')), - ('int', dict(key_format='i')) + ('integer-row', dict(key_format='i')) ) scenarios = make_scenarios(key_format_values) @@ -56,12 +53,12 @@ class test_hs07(wttest.WiredTigerTestCase): for i in range(1, nrows + 1): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, check_value, uri, nrows, read_ts): session = self.session - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -85,8 +82,8 @@ class test_hs07(wttest.WiredTigerTestCase): ds2.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 bigvalue2 = "ddddd" * 100 @@ -102,8 +99,8 @@ class test_hs07(wttest.WiredTigerTestCase): self.check(bigvalue, uri, nrows, 100) # Pin oldest and stable to timestamp 100. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100) + - ',stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100) + + ',stable_timestamp=' + self.timestamp_str(100)) # Sleep here to let that sweep server to trigger cleanup of obsolete entries. time.sleep(10) @@ -118,7 +115,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('A', 10, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(110)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(110)) # Load a slight modification with a later timestamp. self.session.begin_transaction() @@ -126,7 +123,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('B', 20, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(120)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(120)) # Load a slight modification with a later timestamp. self.session.begin_transaction() @@ -134,7 +131,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('C', 30, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(130)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(130)) cursor.close() # Second set of update operations with increased timestamp @@ -147,8 +144,8 @@ class test_hs07(wttest.WiredTigerTestCase): self.check(bigvalue2, uri, nrows, 200) # Pin oldest and stable to timestamp 300. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(200) + - ',stable_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(200) + + ',stable_timestamp=' + self.timestamp_str(200)) # Sleep here to let that sweep server to trigger cleanup of obsolete entries. time.sleep(10) @@ -163,7 +160,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('A', 10, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(210)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(210)) # Load a slight modification with a later timestamp. self.session.begin_transaction() @@ -171,7 +168,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('B', 20, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(220)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(220)) # Load a slight modification with a later timestamp. self.session.begin_transaction() @@ -179,7 +176,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('C', 30, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(230)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(230)) cursor.close() # Third set of update operations with increased timestamp @@ -192,8 +189,8 @@ class test_hs07(wttest.WiredTigerTestCase): self.check(bigvalue, uri, nrows, 300) # Pin oldest and stable to timestamp 400. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(300) + - ',stable_timestamp=' + timestamp_str(300)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(300) + + ',stable_timestamp=' + self.timestamp_str(300)) # Sleep here to let that sweep server to trigger cleanup of obsolete entries. time.sleep(10) diff --git a/src/third_party/wiredtiger/test/suite/test_hs08.py b/src/third_party/wiredtiger/test/suite/test_hs08.py index 0cb26149687..13769f2f19f 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs08.py +++ b/src/third_party/wiredtiger/test/suite/test_hs08.py @@ -30,9 +30,6 @@ import wiredtiger, wttest, time from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs08.py # Verify modify insert into history store logic. class test_hs08(wttest.WiredTigerTestCase): @@ -40,7 +37,7 @@ class test_hs08(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), + ('integer-row', dict(key_format='i')), ] scenarios = make_scenarios(key_format_values) @@ -57,27 +54,27 @@ class test_hs08(wttest.WiredTigerTestCase): self.session.create(uri, create_params) # Insert a full value. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) self.session.begin_transaction() cursor[1] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Insert 3 modifies in separate transactions. self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('A', 1000, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 1001, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 1002, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Call checkpoint. self.session.checkpoint('use_timestamp=true') @@ -89,15 +86,15 @@ class test_hs08(wttest.WiredTigerTestCase): self.assertEqual(squashed_write, 0) # Validate that we see the correct value at each of the timestamps. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) self.assertEqual(cursor[1], value1 + 'A') self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) self.assertEqual(cursor[1], value1 + 'AB') self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) self.assertEqual(cursor[1], value1 + 'ABC') self.session.commit_transaction() @@ -107,12 +104,12 @@ class test_hs08(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 1000, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(7)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(7)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('E', 1001, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(8)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(8)) # Call checkpoint again. self.session.checkpoint('use_timestamp=true') @@ -126,11 +123,11 @@ class test_hs08(wttest.WiredTigerTestCase): # Validate that we see the expected value on the modifies, this # scenario tests the logic that will retrieve a full value for # a modify previously inserted into the history store. - self.session.begin_transaction('read_timestamp=' + timestamp_str(7)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(7)) self.assertEqual(cursor[1], value1 + 'DBC') self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(8)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(8)) self.assertEqual(cursor[1], value1 + 'DEC') self.session.commit_transaction() @@ -140,7 +137,7 @@ class test_hs08(wttest.WiredTigerTestCase): self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('H', 1004, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(9)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(9)) # Call checkpoint again. self.session.checkpoint('use_timestamp=true') @@ -157,13 +154,13 @@ class test_hs08(wttest.WiredTigerTestCase): cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(11)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(11)) # Call checkpoint again. self.session.checkpoint('use_timestamp=true') @@ -179,17 +176,17 @@ class test_hs08(wttest.WiredTigerTestCase): # modify to guarantee we squash zero modifies. self.session.begin_transaction() cursor.set_key(1) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(12)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(12)) self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(13)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(13)) self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0) self.session.commit_transaction() self.session.begin_transaction() cursor.set_key(1) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(14)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(14)) self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(15)) self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0) self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs09.py b/src/third_party/wiredtiger/test/suite/test_hs09.py index 4217da2b02d..1b7993239b2 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs09.py +++ b/src/third_party/wiredtiger/test/suite/test_hs09.py @@ -34,23 +34,21 @@ import wiredtiger, wttest from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs09.py # Verify that we write the newest committed version to data store and the # second newest committed version to history store. class test_hs09(wttest.WiredTigerTestCase): # Force a small cache. - conn_config = 'cache_size=20MB,statistics=(fast)' + conn_config = 'cache_size=20MB' session_config = 'isolation=snapshot' uri = "table:test_hs09" key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), - ('string', dict(key_format='S')), + ('integer-row', dict(key_format='i')), + ('string-row', dict(key_format='S')), ] scenarios = make_scenarios(key_format_values) + nrows = 1000 def create_key(self, i): if self.key_format == 'S': @@ -101,18 +99,18 @@ class test_hs09(wttest.WiredTigerTestCase): value3 = 'c' * 500 # Load 500KB of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load another 500KB of data with a later timestamp. self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Uncommitted changes. self.session.begin_transaction() @@ -122,11 +120,6 @@ class test_hs09(wttest.WiredTigerTestCase): self.check_ckpt_hs(value2, value1, 2, 3) def test_prepared_updates_not_written_to_hs(self): - # FIXME-WT-6061: Prepare reads currently not supported with columnar store. - # Remove this once prepare reads is supported. - if self.key_format == 'r': - return - # Create a small table. create_params = 'key_format={},value_format=S'.format(self.key_format) self.session.create(self.uri, create_params) @@ -136,30 +129,30 @@ class test_hs09(wttest.WiredTigerTestCase): value3 = 'c' * 500 # Load 1MB of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) self.session.begin_transaction() for i in range(1, 2000): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load another 1MB of data with a later timestamp. self.session.begin_transaction() for i in range(1, 2000): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Prepare some updates. self.session.begin_transaction() for i in range(1, 11): cursor[self.create_key(i)] = value3 - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(4)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(4)) # We can expect prepared values to show up in data store if the eviction runs between now # and the time when we open a cursor on the user table. self.check_ckpt_hs(value2, value1, 2, 3, True) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5) + - ',durable_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5) + + ',durable_timestamp=' + self.timestamp_str(5)) def test_write_newest_version_to_data_store(self): # Create a small table. @@ -170,18 +163,18 @@ class test_hs09(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Load 500KB of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load another 500KB of data with a later timestamp. self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) self.check_ckpt_hs(value2, value1, 2, 3) @@ -194,26 +187,26 @@ class test_hs09(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Load 500KB of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load another 500KB of data with a later timestamp. self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Delete records. self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor = self.session.open_cursor(self.uri) cursor.set_key(self.create_key(i)) self.assertEqual(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.check_ckpt_hs(value2, value1, 2, 3) diff --git a/src/third_party/wiredtiger/test/suite/test_hs10.py b/src/third_party/wiredtiger/test/suite/test_hs10.py index 9e3cb4f2699..256ecff4f7b 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs10.py +++ b/src/third_party/wiredtiger/test/suite/test_hs10.py @@ -30,9 +30,6 @@ import wiredtiger, wttest, time from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs10.py # Verify modify read after eviction. class test_hs10(wttest.WiredTigerTestCase): @@ -40,7 +37,7 @@ class test_hs10(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = ( ('column', dict(key_format='r')), - ('int', dict(key_format='i')) + ('integer-row', dict(key_format='i')) ) scenarios = make_scenarios(key_format_values) @@ -61,28 +58,28 @@ class test_hs10(wttest.WiredTigerTestCase): session2.create(uri2, create_params) cursor2 = session2.open_cursor(uri2) # Insert a full value. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) self.session.begin_transaction() cursor[1] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Insert 3 modifies in separate transactions. self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('A', 1000, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 1001, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 1002, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) self.session.checkpoint() # Insert a whole bunch of data into the other table to force wiredtiger to evict data @@ -91,20 +88,20 @@ class test_hs10(wttest.WiredTigerTestCase): cursor2[i] = value2 # Validate that we see the correct value at each of the timestamps. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) cursor.set_key(1) cursor.search() self.assertEqual(cursor[1], value1 + 'A') self.session.commit_transaction() cursor2 = self.session.open_cursor(uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) cursor2.set_key(1) cursor2.search() self.assertEqual(cursor2.get_value(), value1 + 'AB') self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) self.assertEqual(cursor[1], value1 + 'ABC') self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs11.py b/src/third_party/wiredtiger/test/suite/test_hs11.py index 7f0bbfe141a..fd2afc0d977 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs11.py +++ b/src/third_party/wiredtiger/test/suite/test_hs11.py @@ -30,23 +30,22 @@ import wiredtiger, wttest from wtscenario import make_scenarios from wiredtiger import stat -def timestamp_str(t): - return '%x' % t - # test_hs11.py # Ensure that updates without timestamps clear the history store records. class test_hs11(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,statistics=(all)' session_config = 'isolation=snapshot' - key_format_values = ( - ('int', dict(key_format='i')), - ('string', dict(key_format='S')), - ('column', dict(key_format='r')) - ) - scenarios = make_scenarios([ + key_format_values = [ + ('column', dict(key_format='r')), + ('integer-row', dict(key_format='i')), + ('string-row', dict(key_format='S')) + ] + update_type_values = [ ('deletion', dict(update_type='deletion')), - ('update', dict(update_type='update')), - ], key_format_values) + ('update', dict(update_type='update')) + ] + scenarios = make_scenarios(key_format_values, update_type_values) + nrows = 10000 def create_key(self, i): if self.key_format == 'S': @@ -68,19 +67,19 @@ class test_hs11(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Apply a series of updates from timestamps 1-4. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) for ts in range(1, 5): - for i in range(1, 10000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) # Reconcile and flush versions 1-3 to the history store. self.session.checkpoint() # Apply an update without timestamp. - for i in range(1, 10000): + for i in range(1, self.nrows): if i % 2 == 0: if self.update_type == 'deletion': cursor.set_key(self.create_key(i)) @@ -92,20 +91,15 @@ class test_hs11(wttest.WiredTigerTestCase): self.session.checkpoint() # Now apply an update at timestamp 10. - for i in range(1, 10000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) - - # FIXME-WT-7120: Remove for column store until rollback to stable is implemented for column - # store. - if self.key_format == 'r': - return + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Ensure that we blew away history store content. for ts in range(1, 5): - self.session.begin_transaction('read_timestamp=' + timestamp_str(ts)) - for i in range(1, 10000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts)) + for i in range(1, self.nrows): if i % 2 == 0: if self.update_type == 'deletion': cursor.set_key(self.create_key(i)) @@ -129,38 +123,38 @@ class test_hs11(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Apply a series of updates from timestamps 1-4. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) for ts in range(1, 5): - for i in range(1, 10000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) # Reconcile and flush versions 1-3 to the history store. self.session.checkpoint() # Remove the key with timestamp 10. - for i in range(1, 10000): + for i in range(1, self.nrows): if i % 2 == 0: self.session.begin_transaction() cursor.set_key(self.create_key(i)) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Reconcile and remove the obsolete entries. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) self.session.checkpoint() # Now apply an update at timestamp 20. - for i in range(1, 10000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Ensure that we didn't select old history store content even if it is not blew away. - self.session.begin_transaction('read_timestamp=' + timestamp_str(10)) - for i in range(1, 10000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(10)) + for i in range(1, self.nrows): if i % 2 == 0: cursor.set_key(self.create_key(i)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) diff --git a/src/third_party/wiredtiger/test/suite/test_hs12.py b/src/third_party/wiredtiger/test/suite/test_hs12.py index d3be5f40874..e92e338737c 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs12.py +++ b/src/third_party/wiredtiger/test/suite/test_hs12.py @@ -29,19 +29,14 @@ import wiredtiger, wttest, time from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs12.py # Verify we can correctly append modifies to the end of string values class test_hs12(wttest.WiredTigerTestCase): - conn_config = 'cache_size=2MB,statistics=(all),eviction=(threads_max=1)' + conn_config = 'cache_size=2MB,eviction=(threads_max=1)' session_config = 'isolation=snapshot' key_format_values = [ - # FIXME-WT-7120: The commented columnar tests needs to be enabled once columnar - # modify type update is fixed. - # ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), + ('column', dict(key_format='r')), + ('integer-row', dict(key_format='i')), ] scenarios = make_scenarios(key_format_values) @@ -94,12 +89,12 @@ class test_hs12(wttest.WiredTigerTestCase): cursor[1] = value2 self.session.commit_transaction() - # Insert a whole bunch of data into the table to force wiredtiger to evict data - # from the previous table. - self.session.begin_transaction() - for i in range(2, 10000): - cursor[i] = valuebig - self.session.commit_transaction() + # Configure debug behavior on a cursor to evict the positioned page on cursor reset + # and evict the page. + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + evict_cursor.set_key(1) + self.assertEquals(evict_cursor.search(), 0) + evict_cursor.reset() # Try to find the value we saw earlier cursor2.set_key(1) diff --git a/src/third_party/wiredtiger/test/suite/test_hs13.py b/src/third_party/wiredtiger/test/suite/test_hs13.py index ad0992a1656..3781ab200ad 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs13.py +++ b/src/third_party/wiredtiger/test/suite/test_hs13.py @@ -29,19 +29,14 @@ import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs13.py # Verify reverse modify traversal after eviction. class test_hs13(wttest.WiredTigerTestCase): - conn_config = 'cache_size=2MB,statistics=(all),eviction=(threads_max=1)' + conn_config = 'cache_size=2MB,eviction=(threads_max=1)' session_config = 'isolation=snapshot' key_format_values = [ - # FIXME-WT-5550: The commented columnar tests needs to be enabled once columnar modify type - # update is fixed. - # ('column', dict(key_format='r')), - ('integer', dict(key_format='i')) + ('column', dict(key_format='r')), + ('integer-row', dict(key_format='i')) ] scenarios = make_scenarios(key_format_values) @@ -94,12 +89,12 @@ class test_hs13(wttest.WiredTigerTestCase): cursor[1] = value2 self.session.commit_transaction() - # Insert a whole bunch of data into the table to force wiredtiger to evict data - # from the previous table. - self.session.begin_transaction() - for i in range(2, 10000): - cursor[i] = value3 - self.session.commit_transaction() + # Configure debug behavior on a cursor to evict the positioned page on cursor reset + # and evict the page. + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + evict_cursor.set_key(1) + self.assertEquals(evict_cursor.search(), 0) + evict_cursor.reset() # Try to find the value we saw earlier. cursor2.set_key(1) diff --git a/src/third_party/wiredtiger/test/suite/test_hs14.py b/src/third_party/wiredtiger/test/suite/test_hs14.py index 1351dd23e37..7c9d0241d82 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs14.py +++ b/src/third_party/wiredtiger/test/suite/test_hs14.py @@ -29,9 +29,6 @@ import time, wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs14.py # Ensure that point in time reads with few visible history store records don't # damage performance. @@ -40,7 +37,7 @@ class test_hs14(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('string', dict(key_format='S')) + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -52,35 +49,37 @@ class test_hs14(wttest.WiredTigerTestCase): def test_hs14(self): uri = 'table:test_hs14' self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) + nrows = 10000 + value1 = 'a' * 500 value2 = 'b' * 500 value3 = 'c' * 500 value4 = 'd' * 500 value5 = 'e' * 500 - for i in range(1, 10000): + for i in range(1, nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() cursor[self.create_key(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) self.session.begin_transaction() cursor[self.create_key(i)] = value4 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # A checkpoint will ensure that older values are written to the history store. self.session.checkpoint() start = time.time() - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) - for i in range(1, 10000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) + for i in range(1, nrows): self.assertEqual(cursor[self.create_key(i)], value3) self.session.rollback_transaction() end = time.time() @@ -88,21 +87,21 @@ class test_hs14(wttest.WiredTigerTestCase): # The time spent when all history store keys are visible to us. visible_hs_latency = (end - start) - for i in range(1, 10000): + for i in range(1, nrows): self.session.begin_transaction() cursor.set_key(self.create_key(i)) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) self.session.begin_transaction() cursor[self.create_key(i)] = value5 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # A checkpoint will ensure that older values are written to the history store. self.session.checkpoint() start = time.time() - self.session.begin_transaction('read_timestamp=' + timestamp_str(9)) - for i in range(1, 10000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(9)) + for i in range(1, nrows): cursor.set_key(self.create_key(i)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs15.py b/src/third_party/wiredtiger/test/suite/test_hs15.py index 7d27d166a70..9dded0b2d7d 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs15.py +++ b/src/third_party/wiredtiger/test/suite/test_hs15.py @@ -34,9 +34,6 @@ import time, wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs15.py # Ensure eviction doesn't clear the history store again after checkpoint has done so because of the same update without timestamp. class test_hs15(wttest.WiredTigerTestCase): @@ -44,7 +41,7 @@ class test_hs15(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('string', dict(key_format='S')) + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -71,46 +68,46 @@ class test_hs15(wttest.WiredTigerTestCase): for i in range(2, 1000): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Do a modify and an update with timestamps self.session.begin_transaction() cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('B', 100, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) self.session.begin_transaction() cursor[self.create_key(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Make the modify with timestamp and the update without timestamp obsolete - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) # Do a checkpoint self.session.checkpoint() self.session.begin_transaction() cursor[self.create_key(1)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Insert a bunch of other contents to trigger eviction for i in range(2, 1000): self.session.begin_transaction() cursor[self.create_key(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) expected = list(value1) expected[100] = 'B' expected = str().join(expected) - self.session.begin_transaction('read_timestamp=' + timestamp_str(1)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(1)) self.assertEqual(cursor[self.create_key(1)], expected) self.session.rollback_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) self.assertEqual(cursor[self.create_key(1)], value2) self.session.rollback_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) self.assertEqual(cursor[self.create_key(1)], value3) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs16.py b/src/third_party/wiredtiger/test/suite/test_hs16.py index 11ca79456d0..1a5ed51128c 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs16.py +++ b/src/third_party/wiredtiger/test/suite/test_hs16.py @@ -29,9 +29,6 @@ import time, wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs16.py # Ensure that we don't panic when inserting an update without timestamp to the history store. class test_hs16(wttest.WiredTigerTestCase): @@ -39,7 +36,7 @@ class test_hs16(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = ( ('column', dict(key_format='r')), - ('string', dict(key_format='S')) + ('string-row', dict(key_format='S')) ) scenarios = make_scenarios(key_format_values) @@ -62,7 +59,7 @@ class test_hs16(wttest.WiredTigerTestCase): # Update an update at timestamp 1 self.session.begin_transaction() cursor[self.create_key(1)] = 'b' - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) # Open anther session to make the next update without timestamp non-globally visible session2 = self.setUpSessionOpen(self.conn) @@ -78,7 +75,7 @@ class test_hs16(wttest.WiredTigerTestCase): # Update an update at timestamp 2 self.session.begin_transaction() cursor[self.create_key(1)] = 'd' - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Do a checkpoint, it should not panic self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_hs18.py b/src/third_party/wiredtiger/test/suite/test_hs18.py index 889d2dceccc..bcef53e4d17 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs18.py +++ b/src/third_party/wiredtiger/test/suite/test_hs18.py @@ -29,9 +29,6 @@ import time, wiredtiger, wttest, unittest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs18.py # Test various older reader scenarios class test_hs18(wttest.WiredTigerTestCase): @@ -39,7 +36,7 @@ class test_hs18(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('string', dict(key_format='S')) + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -76,7 +73,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 3 self.session.begin_transaction() cursor[self.create_key(1)] = value0 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Start a long running transaction which could see update 0. session2.begin_transaction() @@ -85,12 +82,12 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Insert another update at timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Insert a bunch of contents to fill the cache for i in range(2000, 10000): @@ -106,7 +103,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 15 self.session.begin_transaction() cursor[self.create_key(1)] = value5 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Check our value is still correct. self.check_value(cursor2, value0) @@ -122,11 +119,6 @@ class test_hs18(wttest.WiredTigerTestCase): # Test that we don't get the wrong value if we read with a timestamp originally. def test_read_timestamp_weirdness(self): - # FIXME-WT-7136: Mixed mode transactions not supported for column store scenario. - # Re-enable once complete. - if self.key_format == 'r': - return - uri = 'table:test_hs18' self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) @@ -144,20 +136,20 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 3 self.session.begin_transaction() cursor[self.create_key(1)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Start a long running transaction which could see update 0. - session2.begin_transaction('read_timestamp=' + timestamp_str(5)) + session2.begin_transaction('read_timestamp=' + self.timestamp_str(5)) # Check our value is still correct. self.check_value(cursor2, value1) # Insert another update at timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Start a long running transaction which could see update 0. - session3.begin_transaction('read_timestamp=' + timestamp_str(5)) + session3.begin_transaction('read_timestamp=' + self.timestamp_str(5)) # Check our value is still correct. self.check_value(cursor3, value1) @@ -175,7 +167,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 15 self.session.begin_transaction() cursor[self.create_key(1)] = value5 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Check our value is still correct. self.check_value(cursor2, value1) @@ -219,12 +211,12 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Insert another update at timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Insert a bunch of other contents to trigger eviction for i in range(2, 10000): @@ -251,11 +243,6 @@ class test_hs18(wttest.WiredTigerTestCase): # Test older readers for each of the updates moved to the history store. def test_multiple_older_readers(self): - # FIXME-WT-7136: Mixed mode transactions not supported for column store scenario. - # Re-enable once complete. - if self.key_format == 'r': - return - uri = 'table:test_multiple_older_readers' self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) @@ -274,7 +261,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 3 self.session.begin_transaction() cursor[self.create_key(1)] = values[0] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Start a transaction that will see update 0. self.start_txn(sessions, cursors, values, 0) @@ -282,7 +269,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = values[1] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Start a transaction that will see update 1. self.start_txn(sessions, cursors, values, 1) @@ -290,7 +277,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert another update at timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = values[2] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Start a transaction that will see update 2. self.start_txn(sessions, cursors, values, 2) @@ -312,7 +299,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 15 self.session.begin_transaction() cursor[self.create_key(1)] = values[4] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Insert a bunch of other contents to trigger eviction for i in range(10001, 20000): @@ -328,11 +315,6 @@ class test_hs18(wttest.WiredTigerTestCase): cursors[i].reset() def test_multiple_older_readers_with_multiple_mixed_mode(self): - # FIXME-WT-7136: Mixed mode transactions not supported for column store scenario. - # Re-enable once complete. - if self.key_format == 'r': - return - uri = 'table:test_multiple_older_readers' self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) @@ -351,7 +333,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 3 self.session.begin_transaction() cursor[self.create_key(1)] = values[0] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Start a transaction that will see update 0. self.start_txn(sessions, cursors, values, 0) @@ -359,7 +341,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = values[1] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Start a transaction that will see update 1. self.start_txn(sessions, cursors, values, 1) @@ -367,7 +349,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert another update at timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = values[2] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Start a transaction that will see update 2. self.start_txn(sessions, cursors, values, 2) @@ -389,7 +371,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = values[4] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Start a transaction that will see update 4. self.start_txn(sessions, cursors, values, 4) @@ -397,7 +379,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = values[5] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Start a transaction that will see update 5. self.start_txn(sessions, cursors, values, 5) @@ -405,7 +387,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 15 self.session.begin_transaction() cursor[self.create_key(1)] = values[6] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Start a transaction that will see update 6. self.start_txn(sessions, cursors, values, 6) @@ -434,7 +416,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = values[8] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Insert a bunch of other contents to trigger eviction for i in range(10001, 20000): @@ -450,17 +432,14 @@ class test_hs18(wttest.WiredTigerTestCase): cursors[i].reset() def test_modifies(self): - # FIXME-WT-7136: Modify not supported for column store scenario. - # Re-enable once complete. - if self.key_format == 'r': - return - uri = 'table:test_modifies' self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) session_ts_reader = self.setUpSessionOpen(self.conn) cursor_ts_reader = session_ts_reader.open_cursor(uri) + self.skipTest('Skip this part of test_hs18 until WT-7931 is resolved') + # The ID of the session corresponds the value it should see. sessions = [] cursors = [] @@ -480,7 +459,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 3 self.session.begin_transaction() cursor[self.create_key(1)] = values[0] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Start a long running transaction which could see update 0. self.start_txn(sessions, cursors, values, 0) @@ -489,7 +468,7 @@ class test_hs18(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(self.create_key(1)) cursor.modify([wiredtiger.Modify('a', 0, 0)]) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) session_ts_reader.begin_transaction('read_timestamp=3') self.check_value(cursor_ts_reader, values[0]) @@ -501,7 +480,7 @@ class test_hs18(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(self.create_key(1)) cursor.modify([wiredtiger.Modify('b', 0, 0)]) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Start a long running transaction which could see modify 1. self.start_txn(sessions, cursors, values, 2) @@ -524,7 +503,7 @@ class test_hs18(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(self.create_key(1)) cursor.modify([wiredtiger.Modify('e', 0, 0)]) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Start a long running transaction which could see modify 2. sessions[4].begin_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs19.py b/src/third_party/wiredtiger/test/suite/test_hs19.py index e0549077275..2e3452ae0ca 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs19.py +++ b/src/third_party/wiredtiger/test/suite/test_hs19.py @@ -29,9 +29,6 @@ import time, wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs19.py # Ensure eviction doesn't clear the history store again after checkpoint has done so because of the same update without timestamp. class test_hs19(wttest.WiredTigerTestCase): @@ -39,7 +36,7 @@ class test_hs19(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('string', dict(key_format='S')) + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -57,7 +54,7 @@ class test_hs19(wttest.WiredTigerTestCase): cursor2 = session2.open_cursor(junk_uri) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) value1 = 'a' * 500 value2 = 'b' * 500 @@ -73,13 +70,13 @@ class test_hs19(wttest.WiredTigerTestCase): cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('B', 100, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) self.session.begin_transaction() cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('C', 101, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Start a transaction to pin back the reconciliation last running value. session2.begin_transaction() @@ -93,14 +90,14 @@ class test_hs19(wttest.WiredTigerTestCase): cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('AAAAAAAAAA', 102, 0)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Insert a modify to get written as the on disk value by checkpoint. self.session.begin_transaction() cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('D', 102, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Checkpoint such that all modifies get written out to the history store and the latest # modify gets written to the on disk value. @@ -112,7 +109,7 @@ class test_hs19(wttest.WiredTigerTestCase): cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('E', 103, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # First deposition the first cursor, so the page can be evicted. cursor.reset() @@ -129,7 +126,7 @@ class test_hs19(wttest.WiredTigerTestCase): expected = str().join(expected) # Retrieve the value at timestamp 1. - self.session.begin_transaction('read_timestamp=' + timestamp_str(1)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(1)) cursor.set_key(self.create_key(1)) cursor.search() @@ -143,7 +140,7 @@ class test_hs19(wttest.WiredTigerTestCase): expected = str().join(expected) # Retrieve the value at timestamp 1. - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) cursor.set_key(self.create_key(1)) cursor.search() @@ -159,7 +156,7 @@ class test_hs19(wttest.WiredTigerTestCase): expected = str().join(expected) # Retrieve the value at timestamp 1. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) cursor.set_key(self.create_key(1)) cursor.search() # Assert that it matches our expected value. diff --git a/src/third_party/wiredtiger/test/suite/test_hs20.py b/src/third_party/wiredtiger/test/suite/test_hs20.py index 53e2f141726..8a8a89ffe4f 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs20.py +++ b/src/third_party/wiredtiger/test/suite/test_hs20.py @@ -27,23 +27,36 @@ # OTHER DEALINGS IN THE SOFTWARE. import time, wiredtiger, wttest +from wtscenario import make_scenarios # test_hs20.py # Ensure we never reconstruct a reverse modify update in the history store based on the onpage overflow value -def timestamp_str(t): - return '%x' % t - class test_hs20(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,eviction=(threads_max=1)' session_config = 'isolation=snapshot' + # Return the k'th (0-based) key. + def make_column_key(k): + return k + 1 + def make_string_key(k): + return str(k) + + key_format_values = [ + ('column', dict(key_format='r', make_key=make_column_key)), + ('string-row', dict(key_format='S', make_key=make_string_key)), + ] + + scenarios = make_scenarios(key_format_values) + def test_hs20(self): uri = 'table:test_hs20' + key_format = 'key_format=' + self.key_format + # Set a very small maximum leaf value to trigger writing overflow values - self.session.create(uri, 'key_format=S,value_format=S,leaf_value_max=10B') + self.session.create(uri, '{},value_format=S,leaf_value_max=10B'.format(key_format)) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) value1 = 'a' * 500 value2 = 'b' * 50 @@ -51,41 +64,41 @@ class test_hs20(wttest.WiredTigerTestCase): # Insert a value that is larger than the maximum leaf value. for i in range(0, 10): self.session.begin_transaction() - cursor[str(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + cursor[self.make_key(i)] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Do 2 modifies. for i in range(0, 10): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.make_key(i)) mods = [wiredtiger.Modify('B', 500, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(0, 10): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.make_key(i)) mods = [wiredtiger.Modify('C', 501, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Insert more data to trigger eviction. for i in range(10, 100000): self.session.begin_transaction() - cursor[str(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + cursor[self.make_key(i)] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Update the overflow values. for i in range(0, 10): self.session.begin_transaction() - cursor[str(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + cursor[self.make_key(i)] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Do a checkpoint to move the overflow values to the history store but keep the current in memory disk image. self.session.checkpoint() # Search the first modifies. for i in range(0, 10): - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) - self.assertEqual(cursor[str(i)], value1 + "B") + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) + self.assertEqual(cursor[self.make_key(i)], value1 + "B") self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs21.py b/src/third_party/wiredtiger/test/suite/test_hs21.py index 920dc41951b..7e8755430ca 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs21.py +++ b/src/third_party/wiredtiger/test/suite/test_hs21.py @@ -30,9 +30,7 @@ import time, re import wiredtiger, wttest from wtdataset import SimpleDataSet from wiredtiger import stat - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_hs21.py # Test we don't lose any data when idle files with an active history are closed/sweeped. @@ -49,6 +47,13 @@ class test_hs21(wttest.WiredTigerTestCase): numfiles = 10 nrows = 1000 + key_format_values = [ + ('column', dict(key_format='r', key1=1, key2=2)), + ('string-row', dict(key_format='S', key1=str(0), key2=str(1))), + ] + + scenarios = make_scenarios(key_format_values) + def large_updates(self, uri, value, ds, nrows, commit_ts): # Update a large number of records, we'll hang if the history store table isn't working. session = self.session @@ -56,13 +61,13 @@ class test_hs21(wttest.WiredTigerTestCase): session.begin_transaction() for i in range(1, nrows + 1): cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, session, check_value, uri, nrows, read_ts=-1): # Validate we read an expected value (optionally at a given read timestamp). if read_ts != -1: - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -70,6 +75,12 @@ class test_hs21(wttest.WiredTigerTestCase): count += 1 if read_ts != -1: session.rollback_transaction() + if count != nrows: + self.prout("Oops") + self.prout("value: " + str(check_value)) + self.prout("count: " + str(count)) + self.prout("nrows: " + str(nrows)) + self.prout("read_ts: " + str(read_ts)) self.assertEqual(count, nrows) cursor.close() @@ -101,7 +112,7 @@ class test_hs21(wttest.WiredTigerTestCase): file_uri = 'file:%s.%d.wt' % (self.file_name, f) # Create a small table. ds = SimpleDataSet( - self, table_uri, 0, key_format='S', value_format='S', config='log=(enabled=false)') + self, table_uri, 0, key_format=self.key_format, value_format='S', config='log=(enabled=false)') ds.populate() # Checkpoint to ensure we write the files metadata checkpoint value. self.session.checkpoint() @@ -110,8 +121,8 @@ class test_hs21(wttest.WiredTigerTestCase): active_files.append((base_write_gen, ds)) # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) # Perform a series of updates over our files at timestamp 2. This being data we can later assert # to ensure the history store is working as intended. @@ -122,7 +133,7 @@ class test_hs21(wttest.WiredTigerTestCase): # We want to create a long running read transaction in a seperate session which we will persist over the closing and # re-opening of handles. We want to ensure the correct data gets read throughout this time period. session_read = self.conn.open_session() - session_read.begin_transaction('read_timestamp=' + timestamp_str(2)) + session_read.begin_transaction('read_timestamp=' + self.timestamp_str(2)) # Check our inital set of updates are seen at the read timestamp. for (_, ds) in active_files: # Check that all updates at timestamp 2 are seen. diff --git a/src/third_party/wiredtiger/test/suite/test_hs22.py b/src/third_party/wiredtiger/test/suite/test_hs22.py index cf30767b8bc..97a65ace10a 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs22.py +++ b/src/third_party/wiredtiger/test/suite/test_hs22.py @@ -27,9 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_hs22.py # Test the case that out of order timestamp @@ -38,43 +36,53 @@ class test_hs22(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', key1=1, key2=2)), + ('string-row', dict(key_format='S', key1=str(0), key2=str(1))), + ] + + scenarios = make_scenarios(key_format_values) + def test_onpage_out_of_order_timestamp_update(self): uri = 'table:test_hs22' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) + + key1 = self.key1 + key2 = self.key2 value1 = 'a' value2 = 'b' # Insert a key. self.session.begin_transaction() - cursor[str(0)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + cursor[key1] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Remove the key. self.session.begin_transaction() - cursor.set_key(str(0)) + cursor.set_key(key1) self.assertEqual(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Do an out of order timestamp # update and write it to the data # store later. self.session.begin_transaction() - cursor[str(0)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + cursor[key1] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Insert another key. self.session.begin_transaction() - cursor[str(1)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + cursor[key2] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Update the key. self.session.begin_transaction() - cursor[str(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + cursor[key2] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) # Do a checkpoint to trigger # history store reconciliation. @@ -83,58 +91,61 @@ class test_hs22(wttest.WiredTigerTestCase): evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") # Search the key to evict it. - self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) - self.assertEqual(evict_cursor[str(0)], value2) + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(15)) + self.assertEqual(evict_cursor[key1], value2) self.assertEqual(evict_cursor.reset(), 0) self.session.rollback_transaction() # Search the key again to verify the data is still as expected. - self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) - self.assertEqual(cursor[str(0)], value2) + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(15)) + self.assertEqual(cursor[key1], value2) self.session.rollback_transaction() def test_out_of_order_timestamp_update_newer_than_tombstone(self): uri = 'table:test_hs22' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) + + key1 = self.key1 + key2 = self.key2 value1 = 'a' value2 = 'b' # Insert a key. self.session.begin_transaction() - cursor[str(0)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + cursor[key1] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Remove a key. self.session.begin_transaction() - cursor.set_key(str(0)) + cursor.set_key(key1) self.assertEqual(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Do an out of order timestamp # update and write it to the # history store later. self.session.begin_transaction() - cursor[str(0)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + cursor[key1] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Add another update. self.session.begin_transaction() - cursor[str(0)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + cursor[key1] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Insert another key. self.session.begin_transaction() - cursor[str(1)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + cursor[key2] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Update the key. self.session.begin_transaction() - cursor[str(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + cursor[key2] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) # Do a checkpoint to trigger # history store reconciliation. @@ -143,12 +154,12 @@ class test_hs22(wttest.WiredTigerTestCase): evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") # Search the key to evict it. - self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) - self.assertEqual(evict_cursor[str(0)], value2) + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(15)) + self.assertEqual(evict_cursor[key1], value2) self.assertEqual(evict_cursor.reset(), 0) self.session.rollback_transaction() # Search the key again to verify the data is still as expected. - self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) - self.assertEqual(cursor[str(0)], value2) + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(15)) + self.assertEqual(cursor[key1], value2) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs23.py b/src/third_party/wiredtiger/test/suite/test_hs23.py index cfc7ad70623..d6bc1ec3d70 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs23.py +++ b/src/third_party/wiredtiger/test/suite/test_hs23.py @@ -27,9 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_hs23.py # Test the case that we have update, out of order timestamp @@ -38,12 +36,21 @@ class test_hs23(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', key=1)), + ('string-row', dict(key_format='S', key=str(0))), + ] + + scenarios = make_scenarios(key_format_values) + def test(self): uri = 'table:test_hs23' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) + + key = self.key value1 = 'a' value2 = 'b' @@ -53,34 +60,34 @@ class test_hs23(wttest.WiredTigerTestCase): # Insert a key. self.session.begin_transaction() - cursor[str(0)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + cursor[key] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Update at 10, update at 20, update at 15 (out of order), and # update at 20 in the same transaction self.session.begin_transaction() - cursor.set_key(str(0)) + cursor.set_key(key) cursor.set_value(value2) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(10)) + 'commit_timestamp=' + self.timestamp_str(10)) self.assertEquals(cursor.update(), 0) - cursor.set_key(str(0)) + cursor.set_key(key) cursor.set_value(value3) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(20)) + 'commit_timestamp=' + self.timestamp_str(20)) self.assertEquals(cursor.update(), 0) - cursor.set_key(str(0)) + cursor.set_key(key) cursor.set_value(value4) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(15)) + 'commit_timestamp=' + self.timestamp_str(15)) self.assertEquals(cursor.update(), 0) - cursor.set_key(str(0)) + cursor.set_key(key) cursor.set_value(value5) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(20)) + 'commit_timestamp=' + self.timestamp_str(20)) self.assertEquals(cursor.update(), 0) self.session.commit_transaction() @@ -92,16 +99,16 @@ class test_hs23(wttest.WiredTigerTestCase): # Search the key to evict it. self.session.begin_transaction() - self.assertEqual(evict_cursor[str(0)], value5) + self.assertEqual(evict_cursor[key], value5) self.assertEqual(evict_cursor.reset(), 0) self.session.rollback_transaction() # Search the latest update - self.session.begin_transaction("read_timestamp=" + timestamp_str(20)) - self.assertEqual(cursor[str(0)], value5) + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(20)) + self.assertEqual(cursor[key], value5) self.session.rollback_transaction() - # Serarch the out of order timestamp update - self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) - self.assertEqual(cursor[str(0)], value4) + # Search the out of order timestamp update + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(15)) + self.assertEqual(cursor[key], value4) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs24.py b/src/third_party/wiredtiger/test/suite/test_hs24.py index 4c4ee4a4b94..08d6f303bc5 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs24.py +++ b/src/third_party/wiredtiger/test/suite/test_hs24.py @@ -28,9 +28,7 @@ import wttest, threading, wiredtiger from helper import simulate_crash_restart - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_hs24.py # Test that out of order timestamp fix racing with checkpointing the history store doesn't create inconsistent checkpoint. @@ -38,24 +36,32 @@ class test_hs24(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,timing_stress_for_test=(history_store_checkpoint_delay)' session_config = 'isolation=snapshot' uri = 'table:test_hs24' + numrows = 2000 + + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) value1 = 'a' * 500 value2 = 'b' * 500 value3 = 'c' * 500 value4 = 'd' * 500 def test_zero_ts(self): - self.session.create(self.uri, 'key_format=i,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.session.create(self.uri, 'key_format={},value_format=S'. format(self.key_format)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): self.session.begin_transaction() cursor[i] = self.value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() cursor[i] = self.value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) cursor.close() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) thread = threading.Thread(target=self.zero_ts_deletes) thread.start() self.session.checkpoint() @@ -64,13 +70,13 @@ class test_hs24(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri) session2 = self.conn.open_session(None) cursor2 = session2.open_cursor(self.uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) - session2.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) + session2.begin_transaction('read_timestamp=' + self.timestamp_str(4)) # Check the data store and the history store content is consistent. # If we have a value in the data store, we should see the older # version in the history store as well. newer_data_visible = False - for i in range(0, 2000): + for i in range(1, self.numrows + 1): cursor.set_key(i) cursor2.set_key(i) ret = cursor.search() @@ -88,7 +94,7 @@ class test_hs24(wttest.WiredTigerTestCase): def zero_ts_deletes(self): session = self.setUpSessionOpen(self.conn) cursor = session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): session.begin_transaction() cursor.set_key(i) cursor.remove() @@ -97,29 +103,29 @@ class test_hs24(wttest.WiredTigerTestCase): session.close() def test_zero_commit(self): - self.session.create(self.uri, 'key_format=i,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): self.session.begin_transaction() cursor[i] = self.value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() cursor[i] = self.value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) cursor.close() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(4)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(4)) thread = threading.Thread(target=self.zero_ts_commits) thread.start() self.session.checkpoint() thread.join() simulate_crash_restart(self, '.', "RESTART") cursor = self.session.open_cursor(self.uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) # Check we can only see the version committed by the zero timestamp # commit thread before the checkpoint starts or value1. newer_data_visible = False - for i in range(0, 2000): + for i in range(1, self.numrows + 1): value = cursor[i] if not newer_data_visible: newer_data_visible = value != self.value3 @@ -132,7 +138,7 @@ class test_hs24(wttest.WiredTigerTestCase): def zero_ts_commits(self): session = self.setUpSessionOpen(self.conn) cursor = session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): session.begin_transaction() cursor[i] = self.value3 session.commit_transaction() @@ -140,21 +146,21 @@ class test_hs24(wttest.WiredTigerTestCase): session.close() def test_out_of_order_ts(self): - self.session.create(self.uri, 'key_format=i,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): self.session.begin_transaction() cursor[i] = self.value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() cursor[i] = self.value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(4)) - for i in range(0, 2000): + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(4)) + for i in range(1, self.numrows + 1): self.session.begin_transaction() cursor[i] = self.value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(6)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(6)) cursor.close() thread = threading.Thread(target=self.out_of_order_ts_commits) thread.start() @@ -162,12 +168,12 @@ class test_hs24(wttest.WiredTigerTestCase): thread.join() simulate_crash_restart(self, '.', "RESTART") cursor = self.session.open_cursor(self.uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) # Check we can only see the version at timestamp 4, it's either # committed by the out of order timestamp commit thread before the # checkpoint starts or value1. newer_data_visible = False - for i in range(0, 2000): + for i in range(1, self.numrows + 1): value = cursor[i] if not newer_data_visible: newer_data_visible = value != self.value4 @@ -180,9 +186,9 @@ class test_hs24(wttest.WiredTigerTestCase): def out_of_order_ts_commits(self): session = self.setUpSessionOpen(self.conn) cursor = session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): session.begin_transaction() cursor[i] = self.value4 - session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) cursor.close() session.close() diff --git a/src/third_party/wiredtiger/test/suite/test_hs25.py b/src/third_party/wiredtiger/test/suite/test_hs25.py index e353ce076fe..a1a18928864 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs25.py +++ b/src/third_party/wiredtiger/test/suite/test_hs25.py @@ -27,9 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_hs25.py # Ensure updates structure is correct when processing each key. @@ -38,31 +36,38 @@ class test_hs25(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' uri = 'table:test_hs25' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_insert_updates_hs(self): - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) - self.session.create(self.uri, 'key_format=i,value_format=S') + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) + self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) s = self.conn.open_session() # Update the first key. cursor1 = self.session.open_cursor(self.uri) self.session.begin_transaction() cursor1[1] = 'a' - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Update the second key. self.session.begin_transaction() cursor1[2] = 'a' - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() cursor1[2] = 'b' - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Prepared update on the first key. self.session.begin_transaction() cursor1[1] = 'b' cursor1[1] = 'c' - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(4)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(4)) # Run eviction cursor. s.begin_transaction('ignore_prepare=true') diff --git a/src/third_party/wiredtiger/test/suite/test_hs26.py b/src/third_party/wiredtiger/test/suite/test_hs26.py new file mode 100644 index 00000000000..9aab2322c43 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hs26.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_hs26.py +# Test that changes overlapping variable-length column store RLE groups don't lose or corrupt data. +# (The concern doesn't exist for row stores, so while this test could be run for row stores there's +# little benefit to doing so; thus, no row-store scenarios are generated.) +# +# This works by writing batches of duplicate values, then overwriting them with new batches, +# all with relatively prime counts so the ranges overlap. It also tests the cases where not +# all the old or new value batches exist. +# +# It checks that the immediately written values can be read back, but the interesting part is +# whether they read back correctly after forcing eviction, which will RLE-encode the duplicates +# and read them back. There are many opportunities for the interaction betwee RLE groups and +# history store accesses to go off the rails. +class test_hs26(wttest.WiredTigerTestCase): + conn_config = '' + session_config = 'isolation=snapshot' + + # We control the duplication of values by appending a number computed from the key. + # Because the keys are 1..N (not 0..N-1), to get aligned RLE groups the suffix is + # computed as (key - 1) // value_modulus. This way, if value_modulus is e.g. 7, we + # get the first 7 keys (1-7) using the same suffix. + # + # It would not be _wrong_ to use key // value modulus, but it makes it much more + # difficult to reason about which RLE groups are overlapping where if the first RLE + # group is shorter than the others by 1. So we don't do that. + # + # The cases where the RLE group is mismatched by exactly 1 are particularly likely to + # have issues, so we want to be sure to exercise those cases. Any pair of relatively + # prime integers will generate these cases as long as there are enough keys (e.g. 7 + # and 13 generate one at 14 and the other at 78) but we also want to encounter these + # cases relative to the number of keys written, both the shorter and longer numbers. + # + # Consequently I've picked 103 for one key count (between multiples of 17 and 13) and + # 209 for the other (between multiples of 13 and 7). The ratio of the two key counts + # doesn't signify much but it's close to 2x on general principles. + # + # Other cases of overlapping the key count are still interesting so we still generate + # the full product of the scenarios. + + nrows_values = [ + ('more', dict(nrows_1=103, nrows_2=211)), + ('same', dict(nrows_1=211, nrows_2=211)), + ('less', dict(nrows_1=211, nrows_2=103)), + ] + value_modulus_1_values = [ + ('7', dict(value_modulus_1=7)), + ('13', dict(value_modulus_1=13)), + ('17', dict(value_modulus_1=17)), + ] + value_modulus_2_values = [ + ('7', dict(value_modulus_2=7)), + ('13', dict(value_modulus_2=13)), + ('17', dict(value_modulus_2=17)), + ] + + scenarios = make_scenarios(nrows_values, value_modulus_1_values, value_modulus_2_values) + + value_1 = 'a' * 500 + value_2 = 'd' * 500 + + timestamp_1 = 2 + timestamp_2 = 100 + + # Generate the value for a key. + def make_value(self, key, base_value, value_modulus): + return base_value + str((key - 1) // value_modulus) + + # Write nrows records, using value as the base value string. + def make_updates(self, uri, ds, value, value_modulus, nrows, commit_ts): + session = self.session + cursor = session.open_cursor(uri) + session.begin_transaction() + for i in range(1, nrows + 1): + cursor[ds.key(i)] = self.make_value(i, value, value_modulus) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + cursor.close() + + # Figure the expected value for a key, based on the read time. + # - If the read time is timestamp_1, we should see value_1. + # - If the read time is timestamp_2, we should see value_2, except if nrows_2 < nrows_1, + # in which case we should see value_1 for keys past nrows_1. + def expected_value(self, key, readtime): + if readtime == self.timestamp_1: + return self.make_value(key, self.value_1, self.value_modulus_1) + elif readtime == self.timestamp_2: + if self.nrows_2 < self.nrows_1 and key > self.nrows_2: + return self.make_value(key, self.value_1, self.value_modulus_1) + else: + return self.make_value(key, self.value_2, self.value_modulus_2) + else: + self.prout("expected_value: Unexpected readtime {}".format(readtime)) + self.assertTrue(False) + return None + + # Return the number of keys we expect, based on the read time. + # - If the read time is timestamp_1, we should see nrows_1. + # - If the read time is timestamp_2, we should see max(nrows_1, nrows_2). + def expected_numvalues(self, readtime): + if readtime == self.timestamp_1: + return self.nrows_1 + elif readtime == self.timestamp_2: + return max(self.nrows_1, self.nrows_2) + else: + self.prout("expected_numvalues: Unexpected readtime {}".format(readtime)) + self.assertTrue(False) + return None + + # Check that we got the values we expected. In particular, also make sure that + # we get the expected number of values back. Expect the values that should be + # there at readtime; if explicit_read_ts is set, open a transaction at that + # timestamp. + def check(self, session, uri, readtime, explicit_read_ts=-1): + if explicit_read_ts != -1: + session.begin_transaction('read_timestamp=' + self.timestamp_str(explicit_read_ts)) + cursor = session.open_cursor(uri) + count = 0 + for k, v in cursor: + # Count is key - 1, so pass count + 1 as key. + self.assertEqual(v, self.expected_value(count + 1, readtime)) + count += 1 + if explicit_read_ts != -1: + session.rollback_transaction() + self.assertEqual(count, self.expected_numvalues(readtime)) + cursor.close() + + def test_hs(self): + + # Create a file that contains active history (content newer than the oldest timestamp). + table_uri = 'table:hs26' + ds = SimpleDataSet( + self, table_uri, 0, key_format='r', value_format='S', config='log=(enabled=false)') + ds.populate() + self.session.checkpoint() + + # Pin oldest and stable to timestamp 1. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) + + # Write the first set of values at timestamp_1. + self.make_updates(ds.uri, ds, self.value_1, self.value_modulus_1, self.nrows_1, self.timestamp_1) + + # Create a long running read transaction in a separate session. + # (Is it necessary for it to be separate? Not sure.) + session_read = self.conn.open_session() + session_read.begin_transaction('read_timestamp=' + self.timestamp_str(self.timestamp_1)) + + # Check that the initial writes (at timestamp_1) are seen. + self.check(session_read, ds.uri, self.timestamp_1) + + # Write different values at a later timestamp. + self.make_updates(ds.uri, ds, self.value_2, self.value_modulus_2, self.nrows_2, self.timestamp_2) + + # Check that the new updates are only seen after the update timestamp. + self.check(self.session, ds.uri, self.timestamp_1, self.timestamp_1) + self.check(self.session, ds.uri, self.timestamp_2, self.timestamp_2) + + self.session.breakpoint() + + # Now forcibly evict, so that all the pages are RLE-encoded and then read back in. + # There doesn't seem to be any way to just forcibly evict an entire table, so what + # I'm going to do is assume that each page can hold at least 41 values, and evict + # every 41st key. If this evicts pages repeatedly it won't really hurt anything, + # just waste some time. + + evict_cursor = self.session.open_cursor(ds.uri, None, "debug=(release_evict)") + self.session.begin_transaction() + for k in range(1, max(self.nrows_1, self.nrows_2) + 1, 41): + # Search the key to evict it. + v = evict_cursor[ds.key(k)] + xv = self.expected_value(k, self.timestamp_2) + self.assertEqual(v, xv) + self.assertEqual(evict_cursor.reset(), 0) + self.session.rollback_transaction() + + # Using the long running read transaction, check that the correct data can still be read. + # It should see all the updates at timestamp_1. + self.check(session_read, ds.uri, self.timestamp_1) + session_read.rollback_transaction() + + # Also check that the most recent transaction has the later data. + self.check(self.session, ds.uri, self.timestamp_2, self.timestamp_2) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_hs27.py b/src/third_party/wiredtiger/test/suite/test_hs27.py new file mode 100644 index 00000000000..e685503b8a2 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hs27.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_hs27.py +# Test that variable-length column store doesn't RLE-compact adjacent data with heterogeneous +# timestamps. +# +# (The concern doesn't exist for row stores, so while this test could be run for row stores there's +# little benefit to doing so; thus, no row-store scenarios are generated.) +# +# This works by writing the same value to adjacent keys at different times, evicting them, and +# making sure they read back correctly. (The eviction is necessary to go through the RLE code.) +class test_hs27(wttest.WiredTigerTestCase): + conn_config = '' + session_config = 'isolation=snapshot' + + nrows = 100 + value_1 = 'a' * 119 + value_2 = 'd' * 119 + + # Configure the number of different timestamps to write at. + ntimes_values = [ + ('two', dict(ntimes=2)), + ('three', dict(ntimes=3)), + ('ten', dict(ntimes=10)), + ] + + # Configure the number of keys to write at each timestamp. + nkeys_values = [ + ('one', dict(nkeys=1)), + ('two', dict(nkeys=2)), + ('three', dict(nkeys=3)), + ] + + # Configure whether the keys should be initialized first. + doinit_values = [ + ('init', dict(doinit=True)), + ('noinit', dict(doinit=False)), + ] + + # Configure whether the timestamp groups of keys are written forwards or backwards. + group_forward_values = [ + ('forward', dict(group_forward=True)), + ('backward', dict(group_forward=False)), + ] + + # Configure whether the keys within a timestamp group are written forwards or backwards. + keys_forward_values = [ + ('forward', dict(keys_forward=True)), + ('backward', dict(keys_forward=False)), + ] + + scenarios = make_scenarios(ntimes_values, nkeys_values, doinit_values, + group_forward_values, keys_forward_values) + + # Get the m'th writer timestamp. + def get_writetime(self, m): + return 40 + 2 * m + + # Get the m'th reader timestamp. + def get_readtime(self, m): + return 41 + 2 * m + + # Get the k'th key for the m'th timestamp. + def get_key(self, k, m): + if self.group_forward: + ts_offset = m * self.nkeys + else: + ts_offset = (self.ntimes - m - 1) * self.nkeys + if self.keys_forward: + key_offset = k + else: + key_offset = self.nkeys - k - 1 + return 71 + ts_offset + key_offset + + # Return the key number k and timestamp number m for a key (inverse of get_key). + def invert_key(self, key): + if key < 71 or key >= 71 + self.ntimes * self.nkeys: + return None + key -= 71 + if self.group_forward: + m = key // self.nkeys + else: + m = self.ntimes - (key // self.nkeys) - 1 + if self.keys_forward: + k = key % self.nkeys + else: + k = self.nkeys - (key % self.nkeys) - 1 + return (k, m) + + # Return the timestamp number k for a timestamp (inverse of get_read/writetime). + # Return -1 or self.ntimes for other timestamps as appropriate, for comparison. + def invert_timestamp(self, ts, beforeval, afterval): + if ts < 40: + return beforeval + if ts >= 40 + self.ntimes * 2: + return afterval + ts -= 40 + m = ts // 2 + return m + + # Figure if we should see value_2 for a given key and read time. + def expect_value_2(self, key, readtime): + # Get the key number and timestamp number for the key. + km = self.invert_key(key) + if km is None: + return False + (k, m) = km + + # Get the timestamp number associated with readtime. + m2 = self.invert_timestamp(readtime, -1, self.ntimes) + + # We should see value_2 if we are at or after the time it was written. + return m2 >= m + + # Check each value explicitly to make sure it's what we meant to see. + # Don't bother checking the background values. + def check1(self, session, uri, ds, readtime, make_own_txn): + if make_own_txn: + session.begin_transaction('read_timestamp=' + self.timestamp_str(readtime)) + cursor = session.open_cursor(uri) + for k in range(0, self.nkeys): + for m in range(0, self.ntimes): + key = self.get_key(k, m) + if self.expect_value_2(key, readtime): + self.assertEqual(cursor[ds.key(key)], self.value_2) + elif self.doinit: + self.assertEqual(cursor[ds.key(key)], self.value_1) + else: + self.assertRaisesException(KeyError, lambda: cursor[ds.key(key)]) + if make_own_txn: + session.rollback_transaction() + cursor.close() + + # Scan through the whole table and make sure it's what we expect. + def check2(self, session, uri, readtime, make_own_txn): + if make_own_txn: + session.begin_transaction('read_timestamp=' + self.timestamp_str(readtime)) + cursor = session.open_cursor(uri) + count = 0 + for (key, valueseen) in cursor: + val = self.value_2 if self.expect_value_2(key, readtime) else self.value_1 + self.assertEqual(valueseen, val) + count += 1 + if self.doinit: + # The table was initialized; should always see all rows. + self.assertEqual(count, self.nrows) + else: + # The table was not initialized; should see only what we wrote by this time. + # (If m is 0 we should see one batch of keys.) + m = self.invert_timestamp(readtime, -1, self.ntimes - 1) + self.assertEqual(count, self.nkeys * (m + 1)) + if make_own_txn: + session.rollback_transaction() + cursor.close() + + # Scan through the whole table backward too, since cursor_prev isn't adequately tested. + def check3(self, session, uri, readtime, make_own_txn): + if make_own_txn: + session.begin_transaction('read_timestamp=' + self.timestamp_str(readtime)) + cursor = session.open_cursor(uri) + count = 0 + while cursor.prev() == 0: + key = cursor.get_key() + valueseen = cursor[key] + val = self.value_2 if self.expect_value_2(key, readtime) else self.value_1 + self.assertEqual(valueseen, val) + count += 1 + if self.doinit: + # The table was initialized; should always see all rows. + self.assertEqual(count, self.nrows) + else: + # The table was not initialized; should see only what we wrote by this time. + # (If m is 0 we should see one batch of keys.) + m = self.invert_timestamp(readtime, -1, self.ntimes - 1) + self.assertEqual(count, self.nkeys * (m + 1)) + if make_own_txn: + session.rollback_transaction() + cursor.close() + + # Do all three checks. + def check(self, session, uri, ds, readtime, make_own_txn=True): + self.check1(session, uri, ds, readtime, make_own_txn) + self.check2(session, uri, readtime, make_own_txn) + self.check3(session, uri, readtime, make_own_txn) + + # Scan through the whole table at each relevant timestamp and make sure it's what we expect. + def checkall(self, session, uri, ds): + self.check(session, uri, ds, 2) + for m in range(0, self.ntimes): + self.check(session, uri, ds, self.get_readtime(m)) + self.check(session, uri, ds, 100) + + # Write self.nrows records at timestamp 1, using self.value_1 as the value. + def initialize(self, uri, ds): + session = self.session + cursor = session.open_cursor(uri) + session.begin_transaction() + for i in range(1, self.nrows + 1): + cursor[ds.key(i)] = self.value_1 + session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) + cursor.close() + + # Do the m'th update. + def update(self, uri, ds, m): + writetime = self.get_writetime(m) + + session = self.session + cursor = session.open_cursor(uri) + session.begin_transaction() + if self.keys_forward: + for k in range(0, self.nkeys): + key = self.get_key(k, m) + cursor[ds.key(key)] = self.value_2 + else: + for k in range(self.nkeys - 1, -1, -1): + key = self.get_key(k, m) + cursor[ds.key(key)] = self.value_2 + session.commit_transaction('commit_timestamp=' + self.timestamp_str(writetime)) + cursor.close() + + # Do all the updates. + def updateall(self, uri, ds): + if self.group_forward: + for m in range(0, self.ntimes): + self.update(uri, ds, m) + else: + for m in range(self.ntimes - 1, -1, -1): + self.update(uri, ds, m) + + def test_hs(self): + + # Create a file that contains active history (content newer than the oldest timestamp). + table_uri = 'table:hs27' + ds = SimpleDataSet( + self, table_uri, 0, key_format='r', value_format='S', config='log=(enabled=false)') + ds.populate() + self.session.checkpoint() + + # Pin oldest and stable to timestamp 1. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) + + # Write the initial values, if requested. + if self.doinit: + self.initialize(ds.uri, ds) + + # Create a long running read transaction in a separate session. + # (Is it necessary for it to be separate? Not sure.) + session_read = self.conn.open_session() + session_read.begin_transaction('read_timestamp=' + self.timestamp_str(2)) + + # Check that the initial writes (at timestamp 1) are seen (at timestamp 2). + self.check(session_read, ds.uri, ds, 2, make_own_txn=False) + + # Write more values at assorted timestamps. + self.updateall(ds.uri, ds) + + # Check that the new updates are appropriately visible. + self.checkall(self.session, ds.uri, ds) + + self.session.breakpoint() + + # Now forcibly evict, so that all the pages are RLE-encoded and then read back in. + # There doesn't seem to be any way to just forcibly evict an entire table, so what + # I'm going to do is assume that what we care about is evicting the updates (the + # initial values are not so interesting) and they are on a maximum of two pages, + # so we can evict the first and last key. If this evicts the same page twice, it + # won't really hurt anything. (This also avoids having to worry about whether we + # wrote initial values or not.) + + evict_cursor = self.session.open_cursor(ds.uri, None, "debug=(release_evict)") + self.session.begin_transaction() + firstkey = self.get_key(0, 0) + lastkey = self.get_key(self.nkeys - 1, self.ntimes - 1) + for k in [firstkey, lastkey]: + # Search the key to evict it. + v = evict_cursor[ds.key(k)] + self.assertEqual(v, self.value_2) + self.assertEqual(evict_cursor.reset(), 0) + self.session.rollback_transaction() + + # Check that the long-running read transaction still reads the correct data. + self.check(session_read, ds.uri, ds, 2, make_own_txn=False) + + # Check that our main session reads the correct data. + self.checkall(self.session, ds.uri, ds) + + # Drop the long running read transaction. + session_read.rollback_transaction() + + # Check that our main session can still read the latest data. + self.check(self.session, ds.uri, ds, 100) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_import01.py b/src/third_party/wiredtiger/test/suite/test_import01.py index 9376b805a23..daaae5f95e3 100644 --- a/src/third_party/wiredtiger/test/suite/test_import01.py +++ b/src/third_party/wiredtiger/test/suite/test_import01.py @@ -103,14 +103,10 @@ class test_import_base(wttest.WiredTigerTestCase): "Tmplog" not in file_name and "Preplog" not in file_name: shutil.copy(src_path, dest_dir) - # Convert a WiredTiger timestamp to a string. - def timestamp_str(self, t): - return '%x' % t - # test_import01 class test_import01(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' original_db_file = 'original_db_file' diff --git a/src/third_party/wiredtiger/test/suite/test_import02.py b/src/third_party/wiredtiger/test/suite/test_import02.py index 57f80b769d0..9dbb7b6f66a 100644 --- a/src/third_party/wiredtiger/test/suite/test_import02.py +++ b/src/third_party/wiredtiger/test/suite/test_import02.py @@ -34,7 +34,7 @@ import wiredtiger, wttest from test_import01 import test_import_base class test_import02(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' original_db_file = 'original_db_file' diff --git a/src/third_party/wiredtiger/test/suite/test_import03.py b/src/third_party/wiredtiger/test/suite/test_import03.py index a62486f1bb9..278daafa98e 100644 --- a/src/third_party/wiredtiger/test/suite/test_import03.py +++ b/src/third_party/wiredtiger/test/suite/test_import03.py @@ -34,7 +34,7 @@ from wtscenario import make_scenarios from test_import01 import test_import_base class test_import03(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' ntables = 10 diff --git a/src/third_party/wiredtiger/test/suite/test_import04.py b/src/third_party/wiredtiger/test/suite/test_import04.py index 230fad578e2..f64aefdee70 100644 --- a/src/third_party/wiredtiger/test/suite/test_import04.py +++ b/src/third_party/wiredtiger/test/suite/test_import04.py @@ -54,7 +54,7 @@ from wtscenario import make_scenarios from test_import01 import test_import_base class test_import04(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' ntables = 10 diff --git a/src/third_party/wiredtiger/test/suite/test_import05.py b/src/third_party/wiredtiger/test/suite/test_import05.py index b78a4b81116..0307c6586cb 100644 --- a/src/third_party/wiredtiger/test/suite/test_import05.py +++ b/src/third_party/wiredtiger/test/suite/test_import05.py @@ -35,7 +35,7 @@ from wtscenario import make_scenarios from test_import01 import test_import_base class test_import05(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' ntables = 10 diff --git a/src/third_party/wiredtiger/test/suite/test_import06.py b/src/third_party/wiredtiger/test/suite/test_import06.py index 083fc860a67..7d1360bacba 100644 --- a/src/third_party/wiredtiger/test/suite/test_import06.py +++ b/src/third_party/wiredtiger/test/suite/test_import06.py @@ -87,7 +87,7 @@ class test_import06(test_import_base): extlist.extension('encryptors', self.encryptor) def conn_config(self): - return 'cache_size=50MB,log=(enabled),statistics=(all),encryption=(name={})'.format( + return 'cache_size=50MB,log=(enabled),encryption=(name={})'.format( self.encryptor + self.encryptor_args) def test_import_repair(self): diff --git a/src/third_party/wiredtiger/test/suite/test_import08.py b/src/third_party/wiredtiger/test/suite/test_import08.py index bac4dc8f431..ed5578d02d2 100644 --- a/src/third_party/wiredtiger/test/suite/test_import08.py +++ b/src/third_party/wiredtiger/test/suite/test_import08.py @@ -34,7 +34,7 @@ from test_import01 import test_import_base from wtscenario import make_scenarios class test_import08(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' original_db_file = 'original_db_file' diff --git a/src/third_party/wiredtiger/test/suite/test_import09.py b/src/third_party/wiredtiger/test/suite/test_import09.py index a41aff32571..cc13688d7a1 100644 --- a/src/third_party/wiredtiger/test/suite/test_import09.py +++ b/src/third_party/wiredtiger/test/suite/test_import09.py @@ -100,7 +100,7 @@ class test_import09(test_import_base): extlist.extension('encryptors', self.encryptor) def conn_config(self): - return 'cache_size=50MB,log=(enabled),statistics=(all),encryption=(name={})'.format( + return 'cache_size=50MB,log=(enabled),encryption=(name={})'.format( self.encryptor + self.encryptor_args) def test_import_table_repair(self): diff --git a/src/third_party/wiredtiger/test/suite/test_prepare02.py b/src/third_party/wiredtiger/test/suite/test_prepare02.py index 60e61376c14..9f796d6a50e 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare02.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare02.py @@ -33,9 +33,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_prepare02(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' @@ -91,9 +88,6 @@ class test_prepare02(wttest.WiredTigerTestCase, suite_subprocess): self.assertTimestampsEqual(self.session.query_timestamp('get=prepare'), '2a') self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda:self.session.checkpoint(), msg) - # WT_SESSION.transaction_pinned_range permitted, not supported in the Python API. - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda:self.session.transaction_sync(), msg) self.session.breakpoint() # Commit the transaction. Test that no "not permitted in a prepared transaction" error has diff --git a/src/third_party/wiredtiger/test/suite/test_prepare05.py b/src/third_party/wiredtiger/test/suite/test_prepare05.py index c5971da32fa..c92a079bf6c 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare05.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare05.py @@ -32,33 +32,38 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_prepare05' uri = 'table:' + tablename session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_timestamp_api(self): - self.session.create(self.uri, 'key_format=i,value_format=i') + self.session.create(self.uri, 'key_format={},value_format=i'.format(self.key_format)) c = self.session.open_cursor(self.uri) # It is illegal to set a prepare timestamp older than oldest timestamp. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(1)), + 'prepare_timestamp=' + self.timestamp_str(1)), "/older than the oldest timestamp/") self.session.rollback_transaction() # Check setting the prepare timestamp same as oldest timestamp is valid. self.session.begin_transaction() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(2)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(3)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(3)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(2)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(3)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(3)) self.session.commit_transaction() # In a single transaction it is illegal to set a commit timestamp @@ -67,10 +72,10 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): # prepare itself is illegal. self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(3)) + 'commit_timestamp=' + self.timestamp_str(3)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(2)), + 'prepare_timestamp=' + self.timestamp_str(2)), "/should not have been set before/") self.session.rollback_transaction() @@ -79,11 +84,11 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): # Start a new reader to have an active read timestamp. if wiredtiger.diagnostic_build(): s_reader = self.conn.open_session() - s_reader.begin_transaction('read_timestamp=' + timestamp_str(4)) + s_reader.begin_transaction('read_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(4)), + 'prepare_timestamp=' + self.timestamp_str(4)), "/must be greater than the latest active read timestamp/") self.session.rollback_transaction() @@ -92,7 +97,7 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[1] = 1 self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(5)) + 'prepare_timestamp=' + self.timestamp_str(5)) # Resolve the reader transaction started earlier. s_reader.rollback_transaction() self.session.rollback_transaction() @@ -103,10 +108,10 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): # It is illegal to set a commit timestamp older than prepare timestamp of a transaction. self.session.begin_transaction() c[1] = 1 - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(5)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(5)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(4)), + 'commit_timestamp=' + self.timestamp_str(4)), "/less than the prepare timestamp/") ''' @@ -115,9 +120,9 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[1] = 1 self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(5)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(5)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(5)) + 'prepare_timestamp=' + self.timestamp_str(5)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(5)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(5)) self.session.commit_transaction() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_prepare06.py b/src/third_party/wiredtiger/test/suite/test_prepare06.py index 58361bb39a2..b4e65f183b0 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare06.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare06.py @@ -32,36 +32,41 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_prepare06' uri = 'table:' + tablename session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_timestamp_api(self): - self.session.create(self.uri, 'key_format=i,value_format=i') + self.session.create(self.uri, 'key_format={},value_format=i'.format(self.key_format)) c = self.session.open_cursor(self.uri) # It is illegal to set the prepare timestamp older than the oldest # timestamp. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(20)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(10)), + 'prepare_timestamp=' + self.timestamp_str(10)), "/older than the oldest timestamp/") self.session.rollback_transaction() # Check setting a prepared transaction timestamps earlier than the # oldest timestamp is valid with roundup_timestamps settings. self.session.begin_transaction('roundup_timestamps=(prepared=true)') - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(15)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(35)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(15)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(35)) self.session.commit_transaction() ''' @@ -71,11 +76,11 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): # oldest timestamp is invalid, if durable timestamp is less than the # stable timestamp. self.session.begin_transaction('roundup_timestamps=(prepared=true)') - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(15)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(25)), + 'durable_timestamp=' + self.timestamp_str(25)), "/is less than the stable timestamp/") self.session.rollback_transaction() ''' @@ -83,7 +88,7 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): # Check the cases with an active reader. # Start a new reader to have an active read timestamp. s_reader = self.conn.open_session() - s_reader.begin_transaction('read_timestamp=' + timestamp_str(40)) + s_reader.begin_transaction('read_timestamp=' + self.timestamp_str(40)) # It is illegal to set the prepare timestamp as earlier than an active # read timestamp even with roundup_timestamps settings. This is only @@ -92,7 +97,7 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction('roundup_timestamps=(prepared=true)') self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(10)), + 'prepare_timestamp=' + self.timestamp_str(10)), "/must be greater than the latest active read timestamp/") self.session.rollback_transaction() @@ -101,7 +106,7 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction('roundup_timestamps=(prepared=true)') self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(40)), + 'prepare_timestamp=' + self.timestamp_str(40)), "/must be greater than the latest active read timestamp/") self.session.rollback_transaction() @@ -112,10 +117,10 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): # timestamp of a transaction. self.session.begin_transaction() c[1] = 1 - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(45)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(45)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(30)), + 'commit_timestamp=' + self.timestamp_str(30)), "/less than the prepare timestamp/") ''' @@ -127,12 +132,12 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction('roundup_timestamps=(prepared=true)') c[1] = 1 self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(45)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(30)) - #self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(30)) + 'prepare_timestamp=' + self.timestamp_str(45)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) + #self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(30)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(30)), + 'durable_timestamp=' + self.timestamp_str(30)), "/is less than the commit timestamp/") self.session.rollback_transaction() ''' diff --git a/src/third_party/wiredtiger/test/suite/test_prepare07.py b/src/third_party/wiredtiger/test/suite/test_prepare07.py index c53f6aaa262..49d3544cf40 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare07.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare07.py @@ -30,9 +30,7 @@ import fnmatch, os, shutil, time from helper import copy_wiredtiger_home import wiredtiger, wttest from wtdataset import SimpleDataSet - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare07.py # Test to ensure prepared updates older than oldest timestamp are not visible. @@ -43,10 +41,17 @@ class test_prepare07(wttest.WiredTigerTestCase): # Force a small cache. conn_config = 'cache_size=50MB' + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(key_format_values) + def older_prepare_updates(self, uri, ds, nrows, value_a): # Commit some updates along with a prepared update, which is not resolved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100)) # Commit some updates. value_b = b"bbbbb" * 100 @@ -55,12 +60,12 @@ class test_prepare07(wttest.WiredTigerTestCase): cursor.set_key(ds.key(nrows + 1)) cursor.set_value(value_b) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(110)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(110)) self.session.begin_transaction('isolation=snapshot') cursor.set_key(ds.key(nrows + 2)) cursor.set_value(value_b) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(120)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(120)) # Prepare a transaction and keep it open. session_p = self.conn.open_session() @@ -69,36 +74,36 @@ class test_prepare07(wttest.WiredTigerTestCase): cursor_p.set_key(ds.key(nrows + 3)) cursor_p.set_value(value_b) self.assertEquals(cursor_p.update(), 0) - session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(130)) + session_p.prepare_transaction('prepare_timestamp=' + self.timestamp_str(130)) # Commit some more updates. self.session.begin_transaction('isolation=snapshot') cursor.set_key(ds.key(nrows + 4)) cursor.set_value(value_b) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(140)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(140)) self.session.begin_transaction('isolation=snapshot') cursor.set_key(ds.key(nrows + 5)) cursor.set_value(value_b) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(150)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(150)) # Move the oldest and the stable timestamp to the latest. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(155)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(155)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(155)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(155)) # Commit an update newer than the stable timestamp. self.session.begin_transaction('isolation=snapshot') cursor.set_key(ds.key(nrows + 6)) cursor.set_value(value_b) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(160)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(160)) # Take a checkpoint here, so that prepared transaction will not be durable.. self.session.checkpoint() # Commit the prepared transaction. - session_p.commit_transaction('commit_timestamp=' + timestamp_str(140) + ',durable_timestamp=' + timestamp_str(160)) + session_p.commit_transaction('commit_timestamp=' + self.timestamp_str(140) + ',durable_timestamp=' + self.timestamp_str(160)) # Take a backup and check the values. self.backup_dir = os.path.join(self.home, "WT_BACKUP") @@ -143,7 +148,7 @@ class test_prepare07(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test" nrows = 100 - ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u') + ds = SimpleDataSet(self, uri, nrows, key_format=self.key_format, value_format='u') ds.populate() value_a = b"aaaaa" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_prepare08.py b/src/third_party/wiredtiger/test/suite/test_prepare08.py index 76a831b0a05..05ad8ab287e 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare08.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare08.py @@ -30,9 +30,7 @@ import fnmatch, os, shutil, time from helper import copy_wiredtiger_home import wiredtiger, wttest from wtdataset import SimpleDataSet - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare08.py # Test to ensure prepared tombstones are properly aborted/committed even when they are written @@ -41,6 +39,13 @@ class test_prepare08(wttest.WiredTigerTestCase): # Force a small cache. conn_config = 'cache_size=10MB,eviction_dirty_trigger=80,eviction_updates_trigger=80' + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(key_format_values) + def updates(self, ds, uri, nrows, value, ts): cursor = self.session.open_cursor(uri) self.session.begin_transaction('isolation=snapshot') @@ -48,7 +53,7 @@ class test_prepare08(wttest.WiredTigerTestCase): cursor.set_key(ds.key(i)) cursor.set_value(value) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def removes(self, ds, uri, nrows, ts): @@ -57,7 +62,7 @@ class test_prepare08(wttest.WiredTigerTestCase): for i in range(1, nrows): cursor.set_key(ds.key(i)) self.assertEquals(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def check(self, ds, uri, nrows, value, ts, release_evict): @@ -65,7 +70,7 @@ class test_prepare08(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") else: cursor = self.session.open_cursor(uri) - self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + timestamp_str(ts)) + self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts)) for i in range(1, nrows): cursor.set_key(ds.key(i)) if value == None: @@ -82,11 +87,11 @@ class test_prepare08(wttest.WiredTigerTestCase): # Create a small table. uri_1 = "table:test_prepare08_1" - ds_1 = SimpleDataSet(self, uri_1, 0, key_format="S", value_format='u') + ds_1 = SimpleDataSet(self, uri_1, 0, key_format=self.key_format, value_format='u') ds_1.populate() uri_2 = "table:test_prepare08_2" - ds_2 = SimpleDataSet(self, uri_2, 0, key_format="S", value_format='u') + ds_2 = SimpleDataSet(self, uri_2, 0, key_format=self.key_format, value_format='u') ds_2.populate() value_a = b"aaaaa" * 100 @@ -96,8 +101,8 @@ class test_prepare08(wttest.WiredTigerTestCase): value_e = b"eeeee" * 100 # Commit some updates along with a prepared update, which is not resolved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Initially load huge data self.updates(ds_1, uri_1, nrows, value_a, 20) @@ -124,7 +129,7 @@ class test_prepare08(wttest.WiredTigerTestCase): for i in range(1, nrows): cursor_p.set_key(ds_1.key(i)) self.assertEquals(cursor_p.remove(), 0) - session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) + session_p.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) # Adding more updates to other table should trigger eviction on uri_1 self.updates(ds_2, uri_2, nrows, value_c, 40) @@ -150,12 +155,12 @@ class test_prepare08(wttest.WiredTigerTestCase): # Create a small table. uri_1 = "table:test_prepare10_1" - ds_1 = SimpleDataSet(self, uri_1, 0, key_format="S", value_format='u') + ds_1 = SimpleDataSet(self, uri_1, 0, key_format=self.key_format, value_format='u') ds_1.populate() # Create another small table. uri_2 = "table:test_prepare10_2" - ds_2 = SimpleDataSet(self, uri_2, 0, key_format="S", value_format='u') + ds_2 = SimpleDataSet(self, uri_2, 0, key_format=self.key_format, value_format='u') ds_2.populate() value_a = b"aaaaa" * 100 @@ -165,8 +170,8 @@ class test_prepare08(wttest.WiredTigerTestCase): value_e = b"eeeee" * 100 # Commit some updates along with a prepared update, which is not resolved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Initially load huge data self.updates(ds_1, uri_1, nrows, value_a, 20) @@ -196,7 +201,7 @@ class test_prepare08(wttest.WiredTigerTestCase): self.assertEquals(cursor_p.update(), 0) cursor_p.set_key(ds_1.key(i)) self.assertEquals(cursor_p.remove(), 0) - session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) + session_p.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) # Adding more updates to other table should trigger eviction on uri_1 self.updates(ds_2, uri_2, nrows, value_c, 40) @@ -208,7 +213,7 @@ class test_prepare08(wttest.WiredTigerTestCase): self.check(ds_1, uri_1, nrows, value_b, 50, True) # Commit the prepared session - session_p.commit_transaction('commit_timestamp=' + timestamp_str(50) + ',durable_timestamp=' + timestamp_str(60)) + session_p.commit_transaction('commit_timestamp=' + self.timestamp_str(50) + ',durable_timestamp=' + self.timestamp_str(60)) self.check(ds_1, uri_1, nrows, value_a, 20, False) self.check(ds_1, uri_1, nrows, value_b, 30, False) @@ -224,12 +229,12 @@ class test_prepare08(wttest.WiredTigerTestCase): # Create a small table. uri_1 = "table:test_prepare10_1" - ds_1 = SimpleDataSet(self, uri_1, 0, key_format="S", value_format='u') + ds_1 = SimpleDataSet(self, uri_1, 0, key_format=self.key_format, value_format='u') ds_1.populate() # Create another small table. uri_2 = "table:test_prepare10_2" - ds_2 = SimpleDataSet(self, uri_2, 0, key_format="S", value_format='u') + ds_2 = SimpleDataSet(self, uri_2, 0, key_format=self.key_format, value_format='u') ds_2.populate() value_a = b"aaaaa" * 100 @@ -239,8 +244,8 @@ class test_prepare08(wttest.WiredTigerTestCase): value_e = b"eeeee" * 100 # Commit some updates along with a prepared update, which is not resolved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Initially load huge data self.updates(ds_1, uri_1, nrows, value_a, 20) @@ -266,7 +271,7 @@ class test_prepare08(wttest.WiredTigerTestCase): self.assertEquals(cursor_p.update(), 0) cursor_p.set_key(ds_1.key(i)) self.assertEquals(cursor_p.remove(), 0) - session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) + session_p.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) # Adding more updates to other table should trigger eviction on uri_1 self.updates(ds_2, uri_2, nrows, value_c, 40) @@ -278,7 +283,7 @@ class test_prepare08(wttest.WiredTigerTestCase): self.check(ds_1, uri_1, 0, None, 50, True) # Commit the prepared session - session_p.commit_transaction('commit_timestamp=' + timestamp_str(50) + ',durable_timestamp=' + timestamp_str(60)) + session_p.commit_transaction('commit_timestamp=' + self.timestamp_str(50) + ',durable_timestamp=' + self.timestamp_str(60)) self.check(ds_1, uri_1, nrows, value_a, 20, False) self.check(ds_1, uri_1, 0, None, 30, False) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare09.py b/src/third_party/wiredtiger/test/suite/test_prepare09.py index 3c0aa546d08..8aaacf1c3fb 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare09.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare09.py @@ -31,24 +31,30 @@ # [END_TAGS] import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare09.py # Validate scenarios involving inserting tombstones when rolling back prepares class test_prepare09(wttest.WiredTigerTestCase): - conn_config = 'cache_size=2MB,statistics=(all)' + conn_config = 'cache_size=2MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_prepared_update_is_aborted_correctly_with_on_disk_value(self): uri = "table:test_prepare09" - create_params = 'value_format=S,key_format=i' + create_params = 'value_format=S,key_format={}'.format(self.key_format) value1 = 'a' * 10000 value2 = 'b' * 10000 value3 = 'c' * 10000 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) self.session.create(uri, create_params) cursor = self.session.open_cursor(uri) @@ -58,7 +64,7 @@ class test_prepare09(wttest.WiredTigerTestCase): # Insert a new value. self.session.begin_transaction() cursor[1] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Get the previous update onto disk. for i in range(2, 10000): @@ -68,7 +74,7 @@ class test_prepare09(wttest.WiredTigerTestCase): # Prepare a full value and roll it back. self.session.begin_transaction() cursor[1] = value3 - self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(3)), 0) + self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(3)), 0) self.session.rollback_transaction() # Get the previous update onto disk. @@ -89,8 +95,8 @@ class test_prepare09(wttest.WiredTigerTestCase): value2 = 'b' * 10000 value3 = 'e' * 10000 value4 = 'd' * 10000 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) self.session.create(uri, create_params) cursor = self.session.open_cursor(uri) @@ -102,7 +108,7 @@ class test_prepare09(wttest.WiredTigerTestCase): cursor[1] = value1 cursor[2] = value2 cursor[3] = value3 - self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(3)), 0) + self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(3)), 0) # Get the prepare onto disk. for i in range(4, 10000): diff --git a/src/third_party/wiredtiger/test/suite/test_prepare10.py b/src/third_party/wiredtiger/test/suite/test_prepare10.py index a9ae73848bd..a4eec9bfcc4 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare10.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare10.py @@ -30,9 +30,7 @@ import fnmatch, os, shutil, time from helper import copy_wiredtiger_home import wiredtiger, wttest from wtdataset import SimpleDataSet - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare10.py # Test to ensure prepared tombstones are properly aborted even when they are written @@ -42,6 +40,13 @@ class test_prepare10(wttest.WiredTigerTestCase): conn_config = 'cache_size=10MB,eviction_dirty_trigger=80,eviction_updates_trigger=80' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(key_format_values) + def updates(self, ds, uri, nrows, value, ts): cursor = self.session.open_cursor(uri) self.session.begin_transaction() @@ -49,7 +54,7 @@ class test_prepare10(wttest.WiredTigerTestCase): cursor.set_key(ds.key(i)) cursor.set_value(value) self.assertEquals(cursor.insert(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def removes(self, ds, uri, nrows, ts): @@ -58,12 +63,12 @@ class test_prepare10(wttest.WiredTigerTestCase): for i in range(1, nrows): cursor.set_key(ds.key(i)) self.assertEquals(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def check(self, ds, uri, nrows, value, ts): cursor = self.session.open_cursor(uri) - self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + timestamp_str(ts)) + self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts)) for i in range(1, nrows): cursor.set_key(ds.key(i)) self.assertEquals(cursor.search(), 0) @@ -73,7 +78,7 @@ class test_prepare10(wttest.WiredTigerTestCase): def check_not_found(self, ds, uri, nrows, ts): cursor = self.session.open_cursor(uri) - self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + timestamp_str(ts)) + self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts)) for i in range(1, nrows): cursor.set_key(ds.key(i)) self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) @@ -84,7 +89,7 @@ class test_prepare10(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test_prepare10" nrows = 1000 - ds = SimpleDataSet(self, uri, 0, key_format="S", value_format='u') + ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format='u') ds.populate() value_a = b"aaaaa" * 100 @@ -92,8 +97,8 @@ class test_prepare10(wttest.WiredTigerTestCase): value_c = b"ccccc" * 100 # Commit some updates along with a prepared update, which is not resolved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Initially load huge data self.updates(ds, uri, nrows, value_a, 20) @@ -141,7 +146,7 @@ class test_prepare10(wttest.WiredTigerTestCase): cursor_p.set_key(ds.key(i)) cursor_p.set_value(value_c) self.assertEquals(cursor_p.insert(), 0) - session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(50)) + session_p.prepare_transaction('prepare_timestamp=' + self.timestamp_str(50)) self.check(ds, uri, nrows, value_a, 20) self.check(ds, uri, nrows, value_b, 35) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare11.py b/src/third_party/wiredtiger/test/suite/test_prepare11.py index 4b35a173576..5770d9ad2f2 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare11.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare11.py @@ -29,38 +29,40 @@ import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_prepare11.py # Test prepare rollback with a reserved update between updates. class test_prepare11(wttest.WiredTigerTestCase): - conn_config = 'cache_size=2MB,statistics=(all)' + conn_config = 'cache_size=2MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', key1=17)), + ('string-row', dict(key_format='S', key1='key1')), + ] + commit_values = [ ('commit', dict(commit=True)), ('rollback', dict(commit=False)), ] - scenarios = make_scenarios(commit_values) + scenarios = make_scenarios(key_format_values, commit_values) def test_prepare_update_rollback(self): uri = "table:test_prepare11" - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) self.session.begin_transaction("isolation=snapshot") # In the scenario where we have a reserved update in between two updates, the key repeated # flag won't get set and we'll call resolve prepared op on both prepared updates. c = self.session.open_cursor(uri, None) - c['key1'] = 'xxxx' - c.set_key('key1') + c[self.key1] = 'xxxx' + c.set_key(self.key1) c.reserve() - c['key1'] = 'yyyy' + c[self.key1] = 'yyyy' self.session.prepare_transaction('prepare_timestamp=10') if self.commit: - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(20)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(30)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(20)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(30)) self.session.commit_transaction() else: self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare12.py b/src/third_party/wiredtiger/test/suite/test_prepare12.py index 55d6c69c2d7..f559bde9c1b 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare12.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare12.py @@ -31,42 +31,48 @@ # [END_TAGS] import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare12.py # Test update restore of a page with prepared update. class test_prepare12(wttest.WiredTigerTestCase): - conn_config = 'cache_size=2MB,statistics=(all)' + conn_config = 'cache_size=2MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_prepare_update_restore(self): uri = "table:test_prepare12" - self.session.create(uri, 'key_format=i,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) # Prepare a transaction cursor = self.session.open_cursor(uri, None) self.session.begin_transaction() - cursor[0] = 'a' - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(1)) + cursor[1] = 'a' + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(1)) # Insert an uncommitted key session2 = self.conn.open_session(None) cursor2 = session2.open_cursor(uri, None) session2.begin_transaction() - cursor2[1] = 'b' + cursor2[2] = 'b' # Insert a bunch of other content to fill the database to trigger eviction. session3 = self.conn.open_session(None) cursor3 = session3.open_cursor(uri, None) - for i in range(2, 100): + for i in range(3, 101): session3.begin_transaction() cursor3[i] = 'a' * 500 session3.commit_transaction() # Commit the prepared update - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1) + ',durable_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1) + ',durable_timestamp=' + self.timestamp_str(2)) # Read the prepared update - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) - self.assertEqual(cursor[0], 'a') + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) + self.assertEqual(cursor[1], 'a') diff --git a/src/third_party/wiredtiger/test/suite/test_prepare13.py b/src/third_party/wiredtiger/test/suite/test_prepare13.py index 8542a182328..fbb735ce4f1 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare13.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare13.py @@ -34,22 +34,29 @@ # Fast-truncate fails when a page contains prepared updates. import wiredtiger, wttest from wtdataset import simple_key, simple_value +from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t class test_prepare13(wttest.WiredTigerTestCase): # Force a small cache. - conn_config = 'cache_size=10MB,statistics=(all),statistics_log=(json,on_close,wait=1)' + conn_config = 'cache_size=10MB' + + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(key_format_values) def test_prepare(self): nrows = 20000 # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) # Create a large table with lots of pages. uri = "table:test_prepare13" - config = 'allocation_size=512,leaf_page_max=512,key_format=S,value_format=S' + key_format_str = "key_format=" + self.key_format + config = 'allocation_size=512,leaf_page_max=512,{},value_format=S'.format(key_format_str) self.session.create(uri, config) cursor = self.session.open_cursor(uri) for i in range(1, nrows): @@ -61,12 +68,12 @@ class test_prepare13(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri) cursor[simple_key(cursor, 1000)] = "replacement_value" cursor.close() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)) try: # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) # Open a separate session and cursor and perform updates to let prepared update to evict. s = self.conn.open_session() @@ -74,7 +81,7 @@ class test_prepare13(wttest.WiredTigerTestCase): for i in range(2000, nrows): s.begin_transaction() cursor[simple_key(cursor, i)] = simple_value(cursor, i) - s.commit_transaction('commit_timestamp=' + timestamp_str(20)) + s.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) cursor.close() # Truncate the middle chunk and expect a conflict. @@ -90,8 +97,8 @@ class test_prepare13(wttest.WiredTigerTestCase): s.rollback_transaction() finally: - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(50)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(50)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(50)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(50)) self.session.commit_transaction() s.close() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare14.py b/src/third_party/wiredtiger/test/suite/test_prepare14.py index fb32aefc713..71737907526 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare14.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare14.py @@ -28,11 +28,9 @@ import wttest from wiredtiger import WT_NOTFOUND +from wtdataset import simple_key from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_prepare14.py # Test that the transaction visibility of an on-disk update # that has both the start and the stop time points from the @@ -61,37 +59,34 @@ class test_prepare14(wttest.WiredTigerTestCase): return config def test_prepare14(self): - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:prepare14" - create_config = 'allocation_size=512,key_format=S,value_format=S' + create_config = 'allocation_size=512,key_format={},value_format=S'.format(self.key_format) self.session.create(uri, create_config) # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value = 'a' - # Perform several updates and removes. + # Perform an update and a remove. s = self.conn.open_session() cursor = s.open_cursor(uri) s.begin_transaction() - cursor[str(0)] = value - cursor.set_key(str(0)) + key = simple_key(cursor, 1) + cursor[key] = value + cursor.set_key(key) cursor.remove() cursor.close() - s.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") # Search for the key so we position our cursor on the page that we want to evict. self.session.begin_transaction("ignore_prepare = true") - evict_cursor.set_key(str(0)) + evict_cursor.set_key(key) self.assertEquals(evict_cursor.search(), WT_NOTFOUND) evict_cursor.reset() evict_cursor.close() @@ -99,6 +94,6 @@ class test_prepare14(wttest.WiredTigerTestCase): self.session.begin_transaction("ignore_prepare = true") cursor2 = self.session.open_cursor(uri) - cursor2.set_key(str(0)) + cursor2.set_key(key) self.assertEquals(cursor2.search(), WT_NOTFOUND) self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare15.py b/src/third_party/wiredtiger/test/suite/test_prepare15.py index 4c4ba49a182..b4db822caf6 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare15.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare15.py @@ -30,9 +30,6 @@ import wttest from wiredtiger import WT_NOTFOUND from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_prepare15.py # Test that the prepare transaction rollback removes the on-disk key # or replace it with history store and commit retains the changes when @@ -64,18 +61,14 @@ class test_prepare15(wttest.WiredTigerTestCase): return config def test_prepare_restore_hs_update(self): - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:prepare15" create_config = 'allocation_size=512,key_format=S,value_format=S' self.session.create(uri, create_config) # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = 'a' valueb = 'a' @@ -84,12 +77,12 @@ class test_prepare15(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri) self.session.begin_transaction() cursor[str(0)] = valuea - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) self.session.begin_transaction() cursor.set_key(str(0)) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) cursor.close() # Perform an update and remove. @@ -100,7 +93,7 @@ class test_prepare15(wttest.WiredTigerTestCase): cursor.set_key(str(0)) cursor.remove() cursor.close() - s.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) + s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") @@ -115,8 +108,8 @@ class test_prepare15(wttest.WiredTigerTestCase): if self.commit: # Commit the prepared transaction - s.timestamp_transaction('commit_timestamp=' + timestamp_str(50)) - s.timestamp_transaction('durable_timestamp=' + timestamp_str(60)) + s.timestamp_transaction('commit_timestamp=' + self.timestamp_str(50)) + s.timestamp_transaction('durable_timestamp=' + self.timestamp_str(60)) s.commit_transaction() else: # Rollback the prepared transaction @@ -133,7 +126,7 @@ class test_prepare15(wttest.WiredTigerTestCase): evict_cursor.close() self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(20)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) cursor2 = self.session.open_cursor(uri) cursor2.set_key(str(0)) self.assertEquals(cursor2.search(), 0) @@ -141,18 +134,14 @@ class test_prepare15(wttest.WiredTigerTestCase): self.session.commit_transaction() def test_prepare_not_found(self): - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:prepare15" create_config = 'allocation_size=512,key_format=S,value_format=S' self.session.create(uri, create_config) # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value = 'a' @@ -164,7 +153,7 @@ class test_prepare15(wttest.WiredTigerTestCase): cursor.set_key(str(0)) cursor.remove() cursor.close() - s.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") @@ -179,8 +168,8 @@ class test_prepare15(wttest.WiredTigerTestCase): if self.commit: # Commit the prepared transaction - s.timestamp_transaction('commit_timestamp=' + timestamp_str(30)) - s.timestamp_transaction('durable_timestamp=' + timestamp_str(40)) + s.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) + s.timestamp_transaction('durable_timestamp=' + self.timestamp_str(40)) s.commit_transaction() else: # Rollback the prepared transaction diff --git a/src/third_party/wiredtiger/test/suite/test_prepare16.py b/src/third_party/wiredtiger/test/suite/test_prepare16.py index 23eb85bf444..f6e05aadfa9 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare16.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare16.py @@ -30,9 +30,6 @@ import wttest from wiredtiger import WT_NOTFOUND from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_prepare16.py # Test that the prepare transaction rollback/commit multiple keys # and each key can occupy a leaf page. @@ -65,18 +62,14 @@ class test_prepare16(wttest.WiredTigerTestCase): def test_prepare(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:prepare16" create_config = 'allocation_size=512,key_format=S,value_format=S,leaf_page_max=512,leaf_value_max=64MB' self.session.create(uri, create_config) # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = 'a' * 400 @@ -87,7 +80,7 @@ class test_prepare16(wttest.WiredTigerTestCase): cursor.reset() cursor.close() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)) s = self.conn.open_session() s.begin_transaction('ignore_prepare = true') @@ -100,17 +93,17 @@ class test_prepare16(wttest.WiredTigerTestCase): evict_cursor.reset() if self.commit: - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(20)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(30)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(20)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(30)) self.session.commit_transaction() else: self.session.rollback_transaction() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) if not self.in_memory: self.session.checkpoint() - self.session.begin_transaction('read_timestamp=' + timestamp_str(20)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) cursor = self.session.open_cursor(uri) for i in range(1, nrows + 1): cursor.set_key(str(i)) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py b/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py index 3f298e63b9b..d62ee17b2a2 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py @@ -32,15 +32,21 @@ import wiredtiger, wttest from wtdataset import simple_key, simple_value - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_prepare_conflict(wttest.WiredTigerTestCase): + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_prepare(self): # Create a large table with lots of pages. uri = "table:test_prepare_conflict" - config = 'allocation_size=512,leaf_page_max=512,key_format=S,value_format=S' + key_format = 'key_format=' + self.key_format + config = 'allocation_size=512,leaf_page_max=512,{},value_format=S'.format(key_format) self.session.create(uri, config) cursor = self.session.open_cursor(uri) for i in range(1, 80000): @@ -68,9 +74,9 @@ class test_prepare_conflict(wttest.WiredTigerTestCase): cursor.close() # Prepare and commit the transaction. - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(20)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(20)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(20)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(20)) self.session.commit_transaction() # WT-6325 reports WT_PREPARE_CONFLICT while iterating the cursor. diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py b/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py index aa2e45bec32..579927bd38e 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py @@ -35,9 +35,6 @@ from wtdataset import SimpleDataSet, SimpleIndexDataSet from wtdataset import SimpleLSMDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' %t - # test_prepare_cursor01.py # WT_CURSOR navigation (next/prev) tests with prepared transactions class test_prepare_cursor01(wttest.WiredTigerTestCase): @@ -103,18 +100,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_cursor.set_key(ds.key(51)) prep_cursor.set_value(ds.value(51)) prep_cursor.insert() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(100)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(100)) # Point all cursors to key 50. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(50)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(50)) before_ts_c.set_key(ds.key(50)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(150)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(150)) between_ts_c.set_key(ds.key(50)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(250)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(250)) after_ts_c.set_key(ds.key(50)) self.assertEquals(after_ts_c.search(), 0) @@ -138,8 +135,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.next()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(200)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(200)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(200)) prep_session.commit_transaction() before_ts_s.commit_transaction() @@ -160,18 +157,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_cursor.set_key(ds.key(1)) prep_cursor.set_value(ds.value(1)) prep_cursor.insert() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(100)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(100)) # Point all cursors to key 2. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(50)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(50)) before_ts_c.set_key(ds.key(2)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(150)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(150)) between_ts_c.set_key(ds.key(2)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(250)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(250)) after_ts_c.set_key(ds.key(2)) self.assertEquals(after_ts_c.search(), 0) @@ -196,8 +193,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.prev()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(200)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(200)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(200)) prep_session.commit_transaction() # As read is between(i.e before commit), prev is not found. @@ -223,18 +220,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_cursor.set_key(ds.key(51)) prep_cursor.set_value(ds.value(151)) prep_cursor.update() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(300)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(300)) # Point all cursors to key 51. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(250)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(250)) before_ts_c.set_key(ds.key(50)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(350)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(350)) between_ts_c.set_key(ds.key(50)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(450)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(450)) after_ts_c.set_key(ds.key(50)) self.assertEquals(after_ts_c.search(), 0) @@ -256,8 +253,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.next()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(400)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(400)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(400)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(400)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. @@ -287,18 +284,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_cursor.set_key(ds.key(1)) prep_cursor.set_value(ds.value(111)) prep_cursor.update() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(300)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(300)) # Point all cursors to key 2. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(250)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(250)) before_ts_c.set_key(ds.key(2)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(350)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(350)) between_ts_c.set_key(ds.key(2)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(450)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(450)) after_ts_c.set_key(ds.key(2)) self.assertEquals(after_ts_c.search(), 0) @@ -320,8 +317,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.prev()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(400)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(400)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(400)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(400)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. @@ -355,18 +352,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_session.begin_transaction() prep_cursor.set_key(ds.key(51)) prep_cursor.remove() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(500)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(500)) # Point all cursors to key 51. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(450)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(450)) before_ts_c.set_key(ds.key(50)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(550)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(550)) between_ts_c.set_key(ds.key(50)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(650)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(650)) after_ts_c.set_key(ds.key(50)) self.assertEquals(after_ts_c.search(), 0) @@ -387,8 +384,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.next()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(600)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(600)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(600)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(600)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. @@ -411,18 +408,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_session.begin_transaction() prep_cursor.set_key(ds.key(1)) prep_cursor.remove() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(500)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(500)) # Point all cursors to key 2. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(450)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(450)) before_ts_c.set_key(ds.key(2)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(550)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(550)) between_ts_c.set_key(ds.key(2)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(650)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(650)) after_ts_c.set_key(ds.key(2)) self.assertEquals(after_ts_c.search(), 0) @@ -443,8 +440,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.prev()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(600)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(600)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(600)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(600)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. @@ -474,18 +471,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_session.begin_transaction() prep_cursor.set_key(ds.key(49)) prep_cursor.remove() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(700)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(700)) # Point all cursors to key 48. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(650)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(650)) before_ts_c.set_key(ds.key(48)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(750)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(750)) between_ts_c.set_key(ds.key(48)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(850)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(850)) after_ts_c.set_key(ds.key(48)) self.assertEquals(after_ts_c.search(), 0) @@ -506,8 +503,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.next()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(800)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(800)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(800)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(800)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. @@ -532,18 +529,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_session.begin_transaction() prep_cursor.set_key(ds.key(3)) prep_cursor.remove() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(700)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(700)) # Point all cursors to key 4. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(650)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(650)) before_ts_c.set_key(ds.key(4)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(750)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(750)) between_ts_c.set_key(ds.key(4)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(850)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(850)) after_ts_c.set_key(ds.key(4)) self.assertEquals(after_ts_c.search(), 0) @@ -564,8 +561,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.prev()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(800)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(800)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(800)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(800)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_cursor02.py b/src/third_party/wiredtiger/test/suite/test_prepare_cursor02.py index 65df05fd239..4353ffabf70 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_cursor02.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_cursor02.py @@ -31,9 +31,6 @@ from wtdataset import SimpleDataSet, SimpleIndexDataSet from wtdataset import SimpleLSMDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' %t - # test_prepare_cursor02.py # WT_CURSOR navigation (next/prev) tests with prepared transactions class test_prepare_cursor02(wttest.WiredTigerTestCase): @@ -69,7 +66,7 @@ class test_prepare_cursor02(wttest.WiredTigerTestCase): cursor.set_key(ds.key(1)) cursor.set_value(ds.value(1)) cursor.insert() - session.prepare_transaction('prepare_timestamp=' + timestamp_str(100)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(100)) prep_session = self.conn.open_session(self.session_config) prep_cursor = prep_session.open_cursor(uri, None) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py index c7810e0f719..909f534982e 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py @@ -29,9 +29,7 @@ from helper import copy_wiredtiger_home import wiredtiger, wttest from wtdataset import SimpleDataSet - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare_hs01.py # test to ensure history store eviction is working for prepared transactions. @@ -39,9 +37,16 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): # Force a small cache. conn_config = 'cache_size=50MB,eviction_updates_trigger=95,eviction_updates_target=80' + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(key_format_values) + def check(self, uri, ds, nrows, nsessions, nkeys, read_ts, expected_value, not_expected_value): cursor = self.session.open_cursor(uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) for i in range(1, nsessions * nkeys): cursor.set_key(ds.key(nrows + i)) self.assertEquals(cursor.search(), 0) @@ -67,7 +72,7 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): # memory. Hence testing if we can read prepared updates from the history store. # Start with setting a stable timestamp to pin history in cache - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) # Commit some updates to get eviction and history store fired up bigvalue1 = b"bbbbb" * 100 @@ -77,7 +82,7 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): cursor.set_key(ds.key(nrows + i)) cursor.set_value(bigvalue1) self.assertEquals(cursor.insert(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) # Have prepared updates in multiple sessions. This should ensure writing # prepared updates to the history store @@ -95,7 +100,7 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): cursors[j].set_key(ds.key(nrows + i)) cursors[j].set_value(bigvalue2) self.assertEquals(cursors[j].insert(), 0) - sessions[j].prepare_transaction('prepare_timestamp=' + timestamp_str(2)) + sessions[j].prepare_transaction('prepare_timestamp=' + self.timestamp_str(2)) # Re-read the original versions of all the data. This ensures reading # original versions from the history store @@ -116,7 +121,7 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test_prepare_hs01" nrows = 100 - ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u') + ds = SimpleDataSet(self, uri, nrows, key_format=self.key_format, value_format='u') ds.populate() bigvalue = b"aaaaa" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs02.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs02.py index 5ba68e6a614..6fbd416e94b 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs02.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs02.py @@ -36,9 +36,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_prepare_cursor' uri = 'table:' + tablename @@ -72,16 +69,16 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction(self.txn_config) c[1] = 1 # update the value with in this transaction - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(100)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(100)) if self.txn_commit == True: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(101) + ',durable_timestamp=' + timestamp_str(101)) + 'commit_timestamp=' + self.timestamp_str(101) + ',durable_timestamp=' + self.timestamp_str(101)) else: self.session.rollback_transaction() # Trigger a checkpoint, which could trigger reconciliation - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(150)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(150)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(150)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(150)) self.session.checkpoint() # Scenario: 2 @@ -94,16 +91,16 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): # update a uncommitted value, insert and update a key. c[2] = 1 c[2] = 2 - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(200)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(200)) if self.txn_commit == True: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(201) + ',durable_timestamp=' + timestamp_str(201)) + 'commit_timestamp=' + self.timestamp_str(201) + ',durable_timestamp=' + self.timestamp_str(201)) else: self.session.rollback_transaction() # Trigger a checkpoint, which could trigger reconciliation - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(250)) self.session.checkpoint() # Scenario: 3 @@ -121,10 +118,10 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): c[3] = 2 c.set_key(3) c.remove() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(300)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(300)) if self.txn_commit == True: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(301) + ',durable_timestamp=' + timestamp_str(301)) + 'commit_timestamp=' + self.timestamp_str(301) + ',durable_timestamp=' + self.timestamp_str(301)) else: self.session.rollback_transaction() @@ -133,11 +130,11 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): c[1] = 1 c[2] = 1 c[3] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(302)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(302)) # Trigger a checkpoint, which could trigger reconciliation - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(350)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(350)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(350)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(350)) self.session.checkpoint() #Scenario: 4 @@ -145,8 +142,8 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): # creating the modify update_chain for key instead of insert update # chain. self.reopen_conn() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(350)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(350)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(350)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(350)) self.session.create(self.uri, self.s_config) cur = self.session.open_cursor(self.uri) @@ -159,16 +156,16 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): # Remove a updated key cur.set_key(3) cur.remove() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(400)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(400)) if self.txn_commit == True: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(401) + ',durable_timestamp=' + timestamp_str(401)) + 'commit_timestamp=' + self.timestamp_str(401) + ',durable_timestamp=' + self.timestamp_str(401)) else: self.session.rollback_transaction() # Trigger a checkpoint, which could trigger reconciliation - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(450)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(450)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(450)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(450)) self.session.checkpoint() cur.close() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs03.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs03.py index 502a7ba9c8e..c2d6ca66d4e 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs03.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs03.py @@ -38,9 +38,6 @@ import os, shutil from wtscenario import make_scenarios from wiredtiger import stat -def timestamp_str(t): - return '%x' % t - # test_prepare_hs03.py # test to ensure salvage, verify & simulating crash are working for prepared transactions. class test_prepare_hs03(wttest.WiredTigerTestCase): @@ -51,10 +48,17 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test_prepare_hs03" - scenarios = make_scenarios([ + corrupt_values = [ ('corrupt_table', dict(corrupt=True)), ('dont_corrupt_table', dict(corrupt=False)) - ]) + ] + + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(corrupt_values, key_format_values) def corrupt_table(self): tablename="test_prepare_hs03.wt" @@ -96,12 +100,12 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): cursor.set_key(ds.key(nrows + i)) cursor.set_value(commit_value) self.assertEquals(cursor.insert(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) cursor.close() # Set the stable/oldest timstamps. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) # Corrupt the table, call salvage to recover data from the corrupted table and call verify self.corrupt_salvage_verify() @@ -124,7 +128,7 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): cursors[j].set_key(ds.key(nrows + i)) cursors[j].set_value(prepare_value) self.assertEquals(cursors[j].insert(), 0) - sessions[j].prepare_transaction('prepare_timestamp=' + timestamp_str(4)) + sessions[j].prepare_transaction('prepare_timestamp=' + self.timestamp_str(4)) hs_writes = self.get_stat(stat.conn.cache_write_hs) - hs_writes_start @@ -133,7 +137,7 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): # Test if we can read prepared updates from the history store. cursor = self.session.open_cursor(self.uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) for i in range(1, nsessions * nkeys): cursor.set_key(ds.key(nrows + i)) # The search should pass. @@ -156,7 +160,7 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): # Finally, search for the keys inserted with commit timestamp cursor = self.session.open_cursor(self.uri) self.pr('Read Keys') - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) for i in range(1, nkeys): cursor.set_key(ds.key(nrows + i)) # The search should pass @@ -177,7 +181,7 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri) # Search the keys inserted with commit timestamp after crash - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) for i in range(1, nkeys): cursor.set_key(ds.key(nrows + i)) # The search should pass @@ -195,7 +199,7 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): def test_prepare_hs(self): nrows = 100 - ds = SimpleDataSet(self, self.uri, nrows, key_format="S", value_format='u') + ds = SimpleDataSet(self, self.uri, nrows, key_format=self.key_format, value_format='u') ds.populate() bigvalue = b"aaaaa" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py index 40a80ed0b0e..fa74584ab0c 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py @@ -32,9 +32,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from wiredtiger import stat -def timestamp_str(t): - return '%x' % t - # test_prepare_hs04.py # Read prepared updates from on-disk with ignore_prepare. # Committing or aborting a prepared update when there exists a tombstone for that key already. @@ -50,10 +47,18 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): nkeys = 40 nrows = 100 - scenarios = make_scenarios([ + commit_values = [ ('commit_transaction', dict(commit=True)), ('rollback_transaction', dict(commit=False)) - ]) + ] + + key_format_values = [ + # Note: commit_key must exceed nrows to give behavior comparable to the row case. + ('column', dict(key_format='r', commit_key=1000)), + ('string-row', dict(key_format='S', commit_key='C')), + ] + + scenarios = make_scenarios(commit_values, key_format_values) def get_stat(self, stat): stat_cursor = self.session.open_cursor('statistics:') @@ -64,7 +69,7 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): def search_keys_timestamp_and_ignore(self, ds, txn_config, expected_value, conflict=False): cursor = self.session.open_cursor(self.uri) - commit_key = "C" + commit_key = self.commit_key self.session.begin_transaction(txn_config) for i in range(1, self.nsessions * self.nkeys): key = commit_key + ds.key(self.nrows + i) @@ -82,12 +87,12 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): def prepare_updates(self, ds): # Set oldest and stable timestamp for the database. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) # Commit some updates to get eviction and history store fired up. # Insert a key at timestamp 1. - commit_key = "C" + commit_key = self.commit_key commit_value = b"bbbbb" * 100 cursor = self.session.open_cursor(self.uri) for i in range(1, self.nsessions * self.nkeys): @@ -96,7 +101,7 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): cursor.set_key(key) cursor.set_value(commit_value) self.assertEquals(cursor.insert(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) cursor.close() # Call checkpoint. @@ -108,11 +113,11 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): key = commit_key + ds.key(self.nrows + i) cursor.set_key(key) self.assertEquals(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) cursor.close() # Move the stable timestamp to match the timestamp for the last update. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) hs_writes_start = self.get_stat(stat.conn.cache_write_hs) # Have prepared updates in multiple sessions. This should ensure writing prepared updates to @@ -131,22 +136,22 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): cursors[j].set_key(commit_key + ds.key(self.nrows + i)) cursors[j].set_value(prepare_value) self.assertEquals(cursors[j].insert(), 0) - sessions[j].prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + sessions[j].prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) hs_writes = self.get_stat(stat.conn.cache_write_hs) - hs_writes_start # Assert if not writing anything to the history store. self.assertGreaterEqual(hs_writes, 0) - txn_config = 'read_timestamp=' + timestamp_str(5) + ',ignore_prepare=true' + txn_config = 'read_timestamp=' + self.timestamp_str(5) + ',ignore_prepare=true' # Search keys with timestamp 5, ignore_prepare=true and expect the cursor search to return 0 (key found) self.search_keys_timestamp_and_ignore(ds, txn_config, commit_value) - txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=true' + txn_config = 'read_timestamp=' + self.timestamp_str(20) + ',ignore_prepare=true' # Search keys with timestamp 20, ignore_prepare=true, expect the cursor to return wiredtiger.WT_NOTFOUND self.search_keys_timestamp_and_ignore(ds, txn_config, None) prepare_conflict_msg = '/conflict with a prepared update/' - txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=false' + txn_config = 'read_timestamp=' + self.timestamp_str(20) + ',ignore_prepare=false' # Search keys with timestamp 20, ignore_prepare=false and expect the cursor the cursor search to return prepare conflict message self.search_keys_timestamp_and_ignore(ds, txn_config, prepare_conflict_msg, True) @@ -156,9 +161,9 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): # Commit the prepared_transactions with timestamp 30. for j in range (0, self.nsessions): sessions[j].commit_transaction( - 'commit_timestamp=' + timestamp_str(30) + ',durable_timestamp=' + timestamp_str(30)) + 'commit_timestamp=' + self.timestamp_str(30) + ',durable_timestamp=' + self.timestamp_str(30)) # Move the stable timestamp to match the durable timestamp for prepared updates. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) self.session.checkpoint() @@ -174,44 +179,44 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): # Search keys with timestamp 5, ignore_prepare=true and expect the cursor search to return # value committed before prepared update. - txn_config = 'read_timestamp=' + timestamp_str(5) + ',ignore_prepare=false' + txn_config = 'read_timestamp=' + self.timestamp_str(5) + ',ignore_prepare=false' self.search_keys_timestamp_and_ignore(ds, txn_config, commit_value) # Search keys with timestamp 20, ignore_prepare=true and expect the cursor search to return # WT_NOTFOUND. - txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=true' + txn_config = 'read_timestamp=' + self.timestamp_str(20) + ',ignore_prepare=true' self.search_keys_timestamp_and_ignore(ds, txn_config, None) # Search keys with timestamp 20, ignore_prepare=false and expect the cursor search to return WT_NOTFOUND. - txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=false' + txn_config = 'read_timestamp=' + self.timestamp_str(20) + ',ignore_prepare=false' self.search_keys_timestamp_and_ignore(ds, txn_config, None) # If commit is true then the commit_tramsactions was called and we will expect prepare_value. if self.commit == True: - txn_config = 'read_timestamp=' + timestamp_str(30) + ',ignore_prepare=true' + txn_config = 'read_timestamp=' + self.timestamp_str(30) + ',ignore_prepare=true' # Search keys with timestamp 30, ignore_prepare=true and expect the cursor value to be prepare_value. self.search_keys_timestamp_and_ignore(ds, txn_config, prepare_value) else: # Commit is false and we simulated a crash/restart which would have rolled-back the transactions, therefore we expect the # cursor search to return WT_NOTFOUND. - txn_config = 'read_timestamp=' + timestamp_str(30) + ',ignore_prepare=true' + txn_config = 'read_timestamp=' + self.timestamp_str(30) + ',ignore_prepare=true' # Search keys with timestamp 30, ignore_prepare=true and expect the cursor value to return WT_NOTFOUND. self.search_keys_timestamp_and_ignore(ds, txn_config, None) if self.commit == True: - txn_config = 'read_timestamp=' + timestamp_str(30) + ',ignore_prepare=false' + txn_config = 'read_timestamp=' + self.timestamp_str(30) + ',ignore_prepare=false' # Search keys with timestamp 30, ignore_prepare=false and expect the cursor value to be prepare_value. self.search_keys_timestamp_and_ignore(ds, txn_config, prepare_value) else: # Commit is false and we simulated a crash/restart which would have rolled-back the transactions, therefore we expect the # cursor search to return WT_NOTFOUND. - txn_config = 'read_timestamp=' + timestamp_str(30) + ',ignore_prepare=false' + txn_config = 'read_timestamp=' + self.timestamp_str(30) + ',ignore_prepare=false' # Search keys with timestamp 30, ignore_prepare=false and expect the cursor value to return WT_NOTFOUND. self.search_keys_timestamp_and_ignore(ds, txn_config, None) def test_prepare_hs(self): - ds = SimpleDataSet(self, self.uri, self.nrows, key_format="S", value_format='u') + ds = SimpleDataSet(self, self.uri, self.nrows, key_format=self.key_format, value_format='u') ds.populate() bigvalue = b"aaaaa" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py index ae79bd247f5..cc6f4a2f05d 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py @@ -30,52 +30,56 @@ import wiredtiger, wttest from wtscenario import make_scenarios from wiredtiger import stat, WT_NOTFOUND -def timestamp_str(t): - return '%x' % t - # test_prepare_hs05.py # Test that after aborting prepare transaction, correct update from the history store is restored. class test_prepare_hs05(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', key=1)), + ('string-row', dict(key_format='S', key=str(1))), + ] + + scenarios = make_scenarios(key_format_values) + def test_check_prepare_abort_hs_restore(self): uri = 'table:test_prepare_hs05' - create_params = 'key_format=S,value_format=S' + create_params = 'key_format={},value_format=S'.format(self.key_format) self.session.create(uri, create_params) value1 = 'a' * 5 value2 = 'b' * 5 value3 = 'c' * 5 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - key = 1 + key = self.key self.session.begin_transaction() - cursor[str(key)] = value1 - cursor.set_key(str(key)) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + cursor[key] = value1 + cursor.set_key(key) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Commit update and remove operation in the same transaction. self.session.begin_transaction() - cursor[str(key)] = value2 - cursor.set_key(str(key)) + cursor[key] = value2 + cursor.set_key(key) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Add a prepared update for the key. self.session.begin_transaction() - cursor[str(key)] = value3 - self.session.prepare_transaction('prepare_timestamp='+ timestamp_str(4)) + cursor[key] = value3 + self.session.prepare_transaction('prepare_timestamp='+ self.timestamp_str(4)) # Try to evict the page with prepared update. This will ensure that prepared update is # written as the on-disk version and the older versions are moved to the history store. session2 = self.conn.open_session() session2.begin_transaction('ignore_prepare=true') cursor2 = session2.open_cursor(uri, None, "debug=(release_evict=true)") - cursor2.set_key(str(key)) + cursor2.set_key(key) self.assertEquals(cursor2.search(), WT_NOTFOUND) cursor2.reset() @@ -85,14 +89,14 @@ class test_prepare_hs05(wttest.WiredTigerTestCase): self.session.checkpoint() # We should be able to read the older version of the key from the history store. - self.session.begin_transaction('read_timestamp='+timestamp_str(2)) - cursor.set_key(str(key)) + self.session.begin_transaction('read_timestamp='+self.timestamp_str(2)) + cursor.set_key(key) self.assertEqual(cursor.search(), 0) self.assertEqual(cursor.get_value(), value1) self.session.rollback_transaction() # The latest version should be marked deleted. self.session.begin_transaction() - cursor.set_key(str(key)) + cursor.set_key(key) self.assertEqual(cursor.search(), WT_NOTFOUND) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_readonly02.py b/src/third_party/wiredtiger/test/suite/test_readonly02.py index 4adaf476a0b..62270a52302 100644 --- a/src/third_party/wiredtiger/test/suite/test_readonly02.py +++ b/src/third_party/wiredtiger/test/suite/test_readonly02.py @@ -42,15 +42,15 @@ class test_readonly02(wttest.WiredTigerTestCase, suite_subprocess): entries = 10 conn_params = \ - 'create,statistics=(fast),' + \ + 'create,' + \ 'log=(enabled,file_max=100K,zero_fill=true),' + \ 'operation_tracking=(enabled=false),' conn_params_rd = \ - 'create,readonly=true,statistics=(fast),' + \ + 'create,readonly=true,' + \ 'log=(enabled,zero_fill=false),' + \ 'operation_tracking=(enabled=false),' conn_params_rdcfg = \ - 'create,readonly=true,statistics=(fast),log=(enabled),' + \ + 'create,readonly=true,log=(enabled),' + \ 'operation_tracking=(enabled=false),' # diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py index 7613e169fb1..e85de366970 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py @@ -34,9 +34,6 @@ from wiredtiger import stat, wiredtiger_strerror, WiredTigerError, WT_ROLLBACK from wtscenario import make_scenarios from time import sleep -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable01.py # Shared base class used by rollback to stable tests. class test_rollback_to_stable_base(wttest.WiredTigerTestCase): @@ -77,12 +74,12 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): if commit_ts == 0: session.commit_transaction() elif prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() except WiredTigerError as e: rollback_str = wiredtiger_strerror(WT_ROLLBACK) @@ -104,12 +101,12 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): if commit_ts == 0: session.commit_transaction() elif prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() except WiredTigerError as e: rollback_str = wiredtiger_strerror(WT_ROLLBACK) @@ -129,12 +126,12 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): if commit_ts == 0: session.commit_transaction() elif prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() except WiredTigerError as e: rollback_str = wiredtiger_strerror(WT_ROLLBACK) @@ -147,7 +144,7 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): if read_ts == 0: session.begin_transaction() else: - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -189,10 +186,6 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 10000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable01" ds = SimpleDataSet( @@ -200,8 +193,8 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) valuea = "aaaaa" * 100 self.large_updates(uri, valuea, ds, nrows, self.prepare, 10) @@ -215,9 +208,9 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): # Pin stable to timestamp 20 if prepare otherwise 10. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Checkpoint to ensure that all the updates are flushed to disk. if not self.in_memory: self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py index 20b5d7d6a41..8c27f5395e7 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py @@ -34,9 +34,6 @@ from wiredtiger import stat from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable02.py # Test that rollback to stable brings back the history value to replace on-disk value. class test_rollback_to_stable02(test_rollback_to_stable_base): @@ -70,10 +67,6 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 10000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable02" ds = SimpleDataSet( @@ -81,8 +74,8 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) valuea = "aaaaa" * 100 valueb = "bbbbb" * 100 @@ -106,9 +99,9 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): # Pin stable to timestamp 30 if prepare otherwise 20. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) # Checkpoint to ensure that all the data is flushed. if not self.in_memory: self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py index a0e2d4337b2..c7260db1166 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py @@ -34,9 +34,6 @@ from wiredtiger import stat from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable03.py # Test that rollback to stable clears the history store updates from reconciled pages. class test_rollback_to_stable01(test_rollback_to_stable_base): @@ -70,10 +67,6 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable03" ds = SimpleDataSet( @@ -81,8 +74,8 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) valuea = "aaaaa" * 100 valueb = "bbbbb" * 100 @@ -101,9 +94,9 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): # Pin stable to timestamp 30 if prepare otherwise 20. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) # Checkpoint to ensure that all the updates are flushed to disk. if not self.in_memory: self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py index f865a3975f7..2342163a327 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py @@ -31,9 +31,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - def mod_val(value, char, location, nbytes=1): return value[0:location] + char + value[location+nbytes:] @@ -71,10 +68,6 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable04" ds = SimpleDataSet( @@ -82,8 +75,8 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -131,9 +124,9 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): # Pin stable to timestamp 40 if prepare otherwise 30. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) # Checkpoint to ensure the data is flushed, then rollback to the stable timestamp. if not self.in_memory: diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py index 5604196241c..4a32b589b18 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py @@ -34,9 +34,6 @@ from wiredtiger import stat from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable05.py # Test that rollback to stable cleans history store for non-timestamp tables. class test_rollback_to_stable05(test_rollback_to_stable_base): @@ -70,10 +67,6 @@ class test_rollback_to_stable05(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create two tables without logging. uri_1 = "table:rollback_to_stable05_1" ds_1 = SimpleDataSet( diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py index cd2c35783ce..7b702ca7b5e 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py @@ -31,9 +31,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable06.py # Test that rollback to stable removes all keys when the stable timestamp is earlier than # all commit timestamps. @@ -68,10 +65,6 @@ class test_rollback_to_stable06(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable06" ds = SimpleDataSet( @@ -79,8 +72,8 @@ class test_rollback_to_stable06(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py index 9c9e39d16b9..f745d0a3592 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py @@ -33,9 +33,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable07.py # Test the rollback to stable operation performs as expected following a server crash and # recovery. Verify that @@ -63,10 +60,6 @@ class test_rollback_to_stable07(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable07" ds = SimpleDataSet( @@ -74,8 +67,8 @@ class test_rollback_to_stable07(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -96,9 +89,9 @@ class test_rollback_to_stable07(test_rollback_to_stable_base): # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) # Perform additional updates. self.large_updates(uri, value_b, ds, nrows, self.prepare, 60) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py index 05153034362..e5c3c5bc9ae 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py @@ -31,9 +31,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable08.py # Test that rollback to stable does not abort updates when the stable timestamp is # set to the latest commit. @@ -68,10 +65,6 @@ class test_rollback_to_stable08(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 10000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable08" ds = SimpleDataSet( @@ -79,8 +72,8 @@ class test_rollback_to_stable08(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -101,9 +94,9 @@ class test_rollback_to_stable08(test_rollback_to_stable_base): # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(60)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(60)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Checkpoint to ensure the data is flushed, then rollback to the stable timestamp. if not self.in_memory: diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py index d0a45f0522d..70664a3d776 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py @@ -32,15 +32,17 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable09.py # Test that rollback to stable does not abort schema operations that are done # as they don't have transaction support class test_rollback_to_stable09(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + colstore_values = [ + ('column', dict(use_columns=True)), + ('row', dict(use_columns=False)), + ] + in_memory_values = [ ('no_inmem', dict(in_memory=False)), ('inmem', dict(in_memory=True)) @@ -55,10 +57,10 @@ class test_rollback_to_stable09(test_rollback_to_stable_base): uri = "table:" + tablename index_uri = "index:test_rollback_stable09:country" - scenarios = make_scenarios(in_memory_values, prepare_values) + scenarios = make_scenarios(colstore_values, in_memory_values, prepare_values) def conn_config(self): - config = 'cache_size=250MB,statistics=(all)' + config = 'cache_size=250MB' if self.in_memory: config += ',in_memory=true' else: @@ -69,16 +71,20 @@ class test_rollback_to_stable09(test_rollback_to_stable_base): self.pr('create table') session = self.session session.begin_transaction() - session.create(self.uri, 'key_format=5s,value_format=HQ,columns=(country,year,population)') + if self.use_columns: + config = 'key_format=r,value_format=5sHQ,columns=(id,country,year,population)' + else: + config = 'key_format=5s,value_format=HQ,columns=(country,year,population)' + session.create(self.uri, config) if commit_ts == 0: session.commit_transaction() elif self.prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) def drop_table(self, commit_ts): self.pr('drop table') @@ -88,12 +94,12 @@ class test_rollback_to_stable09(test_rollback_to_stable_base): if commit_ts == 0: session.commit_transaction() elif self.prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) def create_index(self, commit_ts): session = self.session @@ -102,17 +108,17 @@ class test_rollback_to_stable09(test_rollback_to_stable_base): if commit_ts == 0: session.commit_transaction() elif self.prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) def test_rollback_to_stable(self): # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) # Create table and index at a later timestamp self.create_table(20) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py index cfd0c2f3c46..25aaae7ff67 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py @@ -34,9 +34,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from wtthread import checkpoint_thread, op_thread -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable10.py # Test the rollback to stable operation performs sweeping history store. class test_rollback_to_stable10(test_rollback_to_stable_base): @@ -61,10 +58,6 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. self.pr("create/populate tables") uri_1 = "table:rollback_to_stable10_1" @@ -79,8 +72,8 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): ds_2.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -114,9 +107,9 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(60)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(60)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Create a checkpoint thread done = threading.Event() @@ -180,10 +173,6 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): def test_rollback_to_stable_prepare(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. self.pr("create/populate tables") uri_1 = "table:rollback_to_stable10_1" @@ -198,8 +187,8 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): ds_2.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -235,9 +224,9 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(60)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(60)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Here's the update operations we'll perform, encapsulated so we can easily retry # it if we get a rollback. Rollbacks may occur when checkpoint is running. @@ -265,7 +254,7 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): self.retry_rollback('update ds1', session_p1, lambda: prepare_range_updates( session_p1, cursor_p1, ds_1, value_e, nrows, - 'prepare_timestamp=' + timestamp_str(69))) + 'prepare_timestamp=' + self.timestamp_str(69))) # Perform several updates in parallel with checkpoint. session_p2 = self.conn.open_session() @@ -274,7 +263,7 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): self.retry_rollback('update ds2', session_p2, lambda: prepare_range_updates( session_p2, cursor_p2, ds_2, value_e, nrows, - 'prepare_timestamp=' + timestamp_str(69))) + 'prepare_timestamp=' + self.timestamp_str(69))) finally: done.set() ckpt.join() @@ -290,8 +279,8 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): copy_wiredtiger_home(self, ".", "RESTART") # Commit the prepared transaction. - session_p1.commit_transaction('commit_timestamp=' + timestamp_str(70) + ',durable_timestamp=' + timestamp_str(71)) - session_p2.commit_transaction('commit_timestamp=' + timestamp_str(70) + ',durable_timestamp=' + timestamp_str(71)) + session_p1.commit_transaction('commit_timestamp=' + self.timestamp_str(70) + ',durable_timestamp=' + self.timestamp_str(71)) + session_p2.commit_transaction('commit_timestamp=' + self.timestamp_str(70) + ',durable_timestamp=' + self.timestamp_str(71)) session_p1.close() session_p2.close() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py index 59014d5aac9..f88d1ec0082 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py @@ -33,9 +33,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable11.py # Test the rollback to stable is retrieving the proper history store update. class test_rollback_to_stable11(test_rollback_to_stable_base): @@ -60,10 +57,6 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable11" ds = SimpleDataSet( @@ -71,8 +64,8 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -90,9 +83,9 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): # Pin stable to timestamp 30 if prepare otherwise 20. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) # Checkpoint to ensure that all the updates are flushed to disk. self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py index cd0211f2d12..0ab66b1d991 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py @@ -33,9 +33,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable12.py # Test the rollback to stable operation skipping subtrees in during tree walk. class test_rollback_to_stable12(test_rollback_to_stable_base): @@ -60,10 +57,6 @@ class test_rollback_to_stable12(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable12" ds = SimpleDataSet( @@ -71,8 +64,8 @@ class test_rollback_to_stable12(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -85,9 +78,9 @@ class test_rollback_to_stable12(test_rollback_to_stable_base): # Pin stable to timestamp 30 if prepare otherwise 20. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) # Load a single row modification to be removed. commit_ts = 30 @@ -95,12 +88,12 @@ class test_rollback_to_stable12(test_rollback_to_stable_base): self.session.begin_transaction() cursor[ds.key(1)] = value_b if self.prepare: - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) self.session.commit_transaction() else: - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py index 73f23b1f615..dddbd02c260 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py @@ -31,9 +31,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable13.py # Test the rollback to stable should retain/restore the tombstone from # the update list or from the history store for on-disk database. @@ -59,10 +56,6 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable13" ds = SimpleDataSet( @@ -70,8 +63,8 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -92,9 +85,9 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) self.session.checkpoint() # Simulate a server crash and restart. @@ -113,10 +106,6 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): def test_rollback_to_stable_with_aborted_updates(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable13" ds = SimpleDataSet( @@ -124,8 +113,8 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -164,9 +153,9 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) self.session.checkpoint() # Simulate a server crash and restart. @@ -185,10 +174,6 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): def test_rollback_to_stable_with_history_tombstone(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable13" ds = SimpleDataSet( @@ -196,8 +181,8 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -216,14 +201,14 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): cursor[ds.key(i)] = value_b cursor.set_key(ds.key(i)) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(40)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) cursor.close() # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) self.session.checkpoint() @@ -251,17 +236,14 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): def test_rollback_to_stable_with_stable_remove(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') # Create a table without logging. uri = "table:rollback_to_stable13" ds = SimpleDataSet( self, uri, 0, key_format=self.key_format, value_format="S", config='split_pct=50,log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 value_c = "ccccc" * 100 @@ -273,9 +255,9 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): self.large_removes(uri, ds, nrows, self.prepare, 40) # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) # Perform several updates and checkpoint. self.large_updates(uri, value_c, ds, nrows, self.prepare, 60) self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py index e7a5229555d..25b625e1cfa 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py @@ -34,9 +34,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from wtthread import checkpoint_thread, op_thread -def timestamp_str(t): - return '%x' % t - def mod_val(value, char, location, nbytes=1): return value[0:location] + char + value[location+nbytes:] @@ -67,10 +64,6 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1500 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. self.pr("create/populate table") uri = "table:rollback_to_stable14" @@ -79,8 +72,8 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 @@ -106,9 +99,9 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(60)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(60)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Create a checkpoint thread done = threading.Event() @@ -172,10 +165,6 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): def test_rollback_to_stable_same_ts(self): nrows = 1500 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. self.pr("create/populate table") uri = "table:rollback_to_stable14" @@ -184,8 +173,8 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 @@ -213,7 +202,7 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.check(value_modQ, uri, nrows, 30) self.check(value_modT, uri, nrows, 60) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Create a checkpoint thread done = threading.Event() @@ -275,10 +264,6 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): def test_rollback_to_stable_same_ts_append(self): nrows = 1500 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. self.pr("create/populate table") uri = "table:rollback_to_stable14" @@ -287,8 +272,8 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 @@ -316,7 +301,7 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.check(value_modQ, uri, nrows, 30) self.check(value_modT, uri, nrows, 60) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Create a checkpoint thread done = threading.Event() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable15.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable15.py index b91691b99cd..080fcddfe9a 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable15.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable15.py @@ -32,9 +32,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable15.py # Test that roll back to stable handles updates present in the # update-list for both fixed length and variable length column store. @@ -43,7 +40,7 @@ class test_rollback_to_stable15(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), + ('integer-row', dict(key_format='i')), ] value_format_values = [ # Fixed length @@ -70,7 +67,7 @@ class test_rollback_to_stable15(wttest.WiredTigerTestCase): if read_ts == 0: session.begin_transaction() else: - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -89,8 +86,8 @@ class test_rollback_to_stable15(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri) # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) value20 = 0x20 value30 = 0x30 @@ -101,17 +98,17 @@ class test_rollback_to_stable15(wttest.WiredTigerTestCase): for i in range(1, nrows): self.session.begin_transaction() cursor[i] = value20 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) #First Update to value 30 at timestamp 5 for i in range(1, nrows): self.session.begin_transaction() cursor[i] = value30 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) cursor.close() #Set stable timestamp to 2 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(2)) self.conn.rollback_to_stable() # Check that only value20 is available self.check(value20, uri, nrows - 1, 2) @@ -121,17 +118,17 @@ class test_rollback_to_stable15(wttest.WiredTigerTestCase): for i in range(1, nrows): self.session.begin_transaction() cursor[i] = value30 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(7)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(7)) #Third Update to value40 at timestamp 9 for i in range(1, nrows): self.session.begin_transaction() cursor[i] = value40 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(9)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(9)) cursor.close() #Set stable timestamp to 7 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(7)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(7)) self.conn.rollback_to_stable() #Check that only value30 is available self.check(value30, uri, nrows - 1, 7) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py index 5ca08c18baf..4c52e52f50c 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py @@ -37,9 +37,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable16.py # Test that rollback to stable removes updates present on disk for column store. class test_rollback_to_stable16(wttest.WiredTigerTestCase): @@ -81,7 +78,7 @@ class test_rollback_to_stable16(wttest.WiredTigerTestCase): cursor[i] = value + str(i) else: cursor[i] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(timestamp)) cursor.close() def check(self, check_value, uri, nrows, start_row, read_ts): @@ -89,7 +86,7 @@ class test_rollback_to_stable16(wttest.WiredTigerTestCase): if read_ts == 0: session.begin_transaction() else: - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 @@ -128,14 +125,14 @@ class test_rollback_to_stable16(wttest.WiredTigerTestCase): self.session.create(uri, create_params) # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) for i in range(len(values)): self.insert_update_data(uri, values[i], start_row, nrows, ts[i]) start_row += nrows - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) if not self.in_memory: # Checkpoint to ensure that all the updates are flushed to disk. diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py index 829e5d13680..1697cd50c8b 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py @@ -33,9 +33,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable17.py # Test that rollback to stable handles updates present on history store and data store for variable # length column store. @@ -67,12 +64,12 @@ class test_rollback_to_stable17(wttest.WiredTigerTestCase): for i in range(start_row, end_row): self.session.begin_transaction() cursor[i] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(timestamp)) cursor.close() def check(self, check_value, uri, nrows, read_ts): session = self.session - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 @@ -96,15 +93,15 @@ class test_rollback_to_stable17(wttest.WiredTigerTestCase): self.session.create(uri, create_params) # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) # Make a series of updates for the same keys with different values at different timestamps. for i in range(len(values)): self.insert_update_data(uri, values[i], start_row, nrows, ts[i]) # Set the stable timestamp to 5. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) if not self.in_memory: # Checkpoint to ensure that all the updates are flushed to disk. diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py index a9725841b8b..d34e4ffb08b 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py @@ -38,9 +38,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable18.py # Test the rollback to stable shouldn't skip any pages that don't have aggregated time window. class test_rollback_to_stable18(test_rollback_to_stable_base): @@ -65,10 +62,6 @@ class test_rollback_to_stable18(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 10000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable18" ds = SimpleDataSet( @@ -76,8 +69,8 @@ class test_rollback_to_stable18(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 @@ -102,11 +95,11 @@ class test_rollback_to_stable18(test_rollback_to_stable_base): # Pin stable and oldest to timestamp 30 if prepare otherwise 20. if self.prepare: - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(30) + - ',stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(30) + + ',stable_timestamp=' + self.timestamp_str(30)) else: - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(20) + - ',stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(20) + + ',stable_timestamp=' + self.timestamp_str(20)) # Perform rollback to stable. self.conn.rollback_to_stable() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py index dd98263ae41..3f10b186b47 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py @@ -33,9 +33,6 @@ from wiredtiger import stat, WT_NOTFOUND from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable19.py # Test that rollback to stable aborts both insert and remove updates from a single prepared transaction class test_rollback_to_stable19(test_rollback_to_stable_base): @@ -52,8 +49,8 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): ] restart_options = [ - ('shutdown', dict(crash='false')), - ('crash', dict(crash='true')), + ('shutdown', dict(crash=False)), + ('crash', dict(crash=True)), ] scenarios = make_scenarios(in_memory_values, key_format_values, restart_options) @@ -69,10 +66,6 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): def test_rollback_to_stable_no_history(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable19" ds = SimpleDataSet( @@ -80,8 +73,8 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = "aaaaa" * 100 @@ -94,7 +87,7 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): cursor.set_key(i) cursor.remove() cursor.close() - s.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") @@ -116,7 +109,7 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): cursor2.close() # Pin stable timestamp to 20. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) if not self.in_memory: self.session.checkpoint() @@ -137,17 +130,22 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): stat_cursor = self.session.open_cursor('statistics:', None, None) upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] - self.assertGreater(upd_aborted, 0) - self.assertGreater(keys_removed, 0) + + # After restart (not crash) the stats for the aborted updates will be 0, as the updates + # will be aborted during shutdown, and on startup there will be no updates to be aborted. + # This is similar case with keys removed. + if not self.in_memory and not self.crash: + self.assertEqual(upd_aborted, 0) + self.assertEqual(keys_removed, 0) + else: + self.assertGreater(upd_aborted, 0) + self.assertGreater(keys_removed, 0) + stat_cursor.close() def test_rollback_to_stable_with_history(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable19" ds = SimpleDataSet( @@ -155,8 +153,8 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = "aaaaa" * 100 valueb = "bbbbb" * 100 @@ -176,7 +174,7 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): cursor.set_key(i) cursor.remove() cursor.close() - s.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) + s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") @@ -198,7 +196,7 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): cursor2.close() # Pin stable timestamp to 40. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) if not self.in_memory: self.session.checkpoint() @@ -220,6 +218,15 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): stat_cursor = self.session.open_cursor('statistics:', None, None) upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] - self.assertGreater(upd_aborted, 0) + + # After restart (not crash) the stats for the aborted updates and history store removed will be 0, + # as the updates aborted and history store removed will occur during shutdown, and on startup there + # will be no updates to be removed. if not self.in_memory: - self.assertGreater(hs_removed, 0) + if self.crash: + self.assertGreater(hs_removed, 0) + else: + self.assertEqual(hs_removed, 0) + self.assertEqual(upd_aborted, 0) + else: + self.assertGreater(upd_aborted, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable20.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable20.py index a2c301ec362..8a2f44c5a36 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable20.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable20.py @@ -30,17 +30,22 @@ import time from helper import copy_wiredtiger_home import wiredtiger, wttest from wtdataset import SimpleDataSet +from wtscenario import make_scenarios from wiredtiger import stat from helper import simulate_crash_restart from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # Test that rollback to stable does not open any dhandles that don't have unstable updates. class test_rollback_to_stable20(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def conn_config(self): config = 'cache_size=50MB,statistics=(all)' return config @@ -51,12 +56,12 @@ class test_rollback_to_stable20(test_rollback_to_stable_base): create_params = 'key_format=i,value_format=S' uri = "table:rollback_to_stable20" ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds.populate() # Pin oldest and stable timestamp to 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) valuea = "aaaaa" * 100 @@ -65,7 +70,7 @@ class test_rollback_to_stable20(test_rollback_to_stable_base): self.session.create(uri, create_params) self.large_updates(uri, valuea, ds, nrows, 0, 10) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py index 4e44b9c2d97..3a348d00dc0 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py @@ -37,9 +37,6 @@ from helper import simulate_crash_restart from wtdataset import SimpleDataSet from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable21.py # Test rollback to stable when an out of order prepared transaction is written to disk class test_rollback_to_stable21(test_rollback_to_stable_base): @@ -57,10 +54,6 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable21" ds = SimpleDataSet( @@ -68,8 +61,8 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = 'a' * 400 valueb = 'b' * 400 @@ -79,7 +72,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): for i in range(1, nrows + 1): cursor[i] = valuea - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) self.session.begin_transaction() for i in range(1, nrows + 1): @@ -87,7 +80,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): cursor.reset() cursor.close() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) s = self.conn.open_session() s.begin_transaction('ignore_prepare = true') @@ -101,7 +94,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): evict_cursor.reset() s.rollback_transaction() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) s.checkpoint() # Rollback the prepared transaction @@ -123,10 +116,6 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): def test_rollback_to_stable_with_different_tombstone(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable21" ds = SimpleDataSet( @@ -134,8 +123,8 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = 'a' * 400 valueb = 'b' * 400 @@ -144,13 +133,13 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): self.session.begin_transaction() for i in range(1, nrows + 1): cursor[i] = valuea - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) self.session.begin_transaction() for i in range(1, nrows + 1): cursor.set_key(i) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(40)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) self.session.begin_transaction() for i in range(1, nrows + 1): @@ -158,10 +147,10 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): cursor.reset() cursor.close() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) s = self.conn.open_session() - s.begin_transaction('ignore_prepare = true, read_timestamp = ' + timestamp_str(30)) + s.begin_transaction('ignore_prepare = true, read_timestamp = ' + self.timestamp_str(30)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = s.open_cursor(uri, None, "debug=(release_evict)") @@ -172,7 +161,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): evict_cursor.reset() s.rollback_transaction() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) s.checkpoint() # Rollback the prepared transaction @@ -197,10 +186,6 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): def test_rollback_to_stable_with_same_tombstone(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable21" ds = SimpleDataSet( @@ -208,8 +193,8 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = 'a' * 400 valueb = 'b' * 400 @@ -221,7 +206,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): cursor.set_key(i) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) self.session.begin_transaction() for i in range(1, nrows + 1): @@ -229,7 +214,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): cursor.reset() cursor.close() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) s = self.conn.open_session() s.begin_transaction('ignore_prepare = true') @@ -242,7 +227,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): evict_cursor.reset() s.rollback_transaction() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) s.checkpoint() # Rollback the prepared transaction diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py index ac56c6c6b5f..f6ef52cc388 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py @@ -29,16 +29,13 @@ from test_rollback_to_stable01 import test_rollback_to_stable_base from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable22 # Test history store operations conflicting with rollback to stable. We're trying to trigger a # history store eviction concurrently to a rollback to stable call. We'll do this by limiting # the cache size to 100MB and performing 100MB worth of inserts while periodically calling rollback # to stable. class test_rollback_to_stable22(test_rollback_to_stable_base): - conn_config = 'cache_size=100MB,statistics=(fast),statistics_log=(wait=1,json)' + conn_config = 'cache_size=100MB' session_config = 'isolation=snapshot' prepare = False @@ -46,6 +43,8 @@ class test_rollback_to_stable22(test_rollback_to_stable_base): nrows = 1000 nds = 10 + self.skipTest('Skip it until the fix is provided to handle concurrent internal transactions running in parallel.') + # Create a few tables and populate them with some initial data. # # Our way of preventing history store operations from interfering with rollback to stable's @@ -80,5 +79,5 @@ class test_rollback_to_stable22(test_rollback_to_stable_base): if i % 100 == 0: # Put the timestamp backwards so we can rollback the updates we just did. stable_ts = (i - 1) * 10 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(stable_ts)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(stable_ts)) self.conn.rollback_to_stable() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable23.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable23.py new file mode 100644 index 00000000000..e4ff9e4dfb4 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable23.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from helper import simulate_crash_restart +from test_rollback_to_stable01 import test_rollback_to_stable_base +from wiredtiger import stat +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +def mod_val(value, char, location, nbytes=1): + return value[0:location] + char + value[location+nbytes:] + +# test_rollback_to_stable23.py +# Test to verify that search operation uses proper base update while returning modifies from +# the history store after the on-disk update is removed by the rollback to stable. +class test_rollback_to_stable23(test_rollback_to_stable_base): + session_config = 'isolation=snapshot' + + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + prepare_values = [ + ('no_prepare', dict(prepare=False)), + ('prepare', dict(prepare=True)) + ] + + scenarios = make_scenarios(key_format_values, prepare_values) + + def conn_config(self): + config = 'cache_size=50MB,statistics=(all),statistics_log=(json,on_close,wait=1)' + return config + + def check_with_set_key(self, ds, check_value, uri, nrows, read_ts): + cursor = self.session.open_cursor(uri) + self.session.begin_transaction("read_timestamp = " + self.timestamp_str(read_ts)) + for i in range(1, nrows + 1): + cursor.set_key(ds.key(i)) + self.assertEquals(cursor.search(), 0) + self.assertEquals(cursor.get_value(), check_value) + cursor.close() + self.session.commit_transaction() + + def test_rollback_to_stable(self): + nrows = 1000 + + # Create a table without logging. + uri = "table:rollback_to_stable23" + ds = SimpleDataSet( + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + ds.populate() + + # Pin oldest and stable to timestamp 10. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) + + value_a = "aaaaa" * 100 + + value_modQ = mod_val(value_a, 'Q', 0) + value_modR = mod_val(value_modQ, 'R', 1) + value_modS = mod_val(value_modR, 'S', 2) + value_modT = mod_val(value_modS, 'T', 3) + + # Perform a combination of modifies and updates. + self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) + self.large_modifies(uri, 'Q', ds, 0, 1, nrows, self.prepare, 30) + self.large_modifies(uri, 'R', ds, 1, 1, nrows, self.prepare, 40) + self.large_modifies(uri, 'S', ds, 2, 1, nrows, self.prepare, 50) + self.large_modifies(uri, 'T', ds, 3, 1, nrows, self.prepare, 60) + + # Verify data is visible and correct. + self.check(value_a, uri, nrows, 20) + self.check(value_modQ, uri, nrows, 30) + self.check(value_modR, uri, nrows, 40) + self.check(value_modS, uri, nrows, 50) + self.check(value_modT, uri, nrows, 60) + + # Pin stable to timestamp 60 if prepare otherwise 50. + if self.prepare: + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(60)) + else: + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) + + # Checkpoint the database. + self.session.checkpoint() + + # Simulate a server crash and restart. + simulate_crash_restart(self, ".", "RESTART") + + # Check that the correct data is seen at and after the stable timestamp. + self.check_with_set_key(ds, value_a, uri, nrows, 20) + self.check_with_set_key(ds, value_modQ, uri, nrows, 30) + self.check_with_set_key(ds, value_modR, uri, nrows, 40) + self.check_with_set_key(ds, value_modS, uri, nrows, 50) + + stat_cursor = self.session.open_cursor('statistics:', None, None) + hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] + hs_restore_updates = stat_cursor[stat.conn.txn_rts_hs_restore_updates][2] + upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] + stat_cursor.close() + + self.assertEqual(hs_restore_updates, nrows) + if self.prepare: + self.assertGreaterEqual(upd_aborted, 0) + else: + self.assertEqual(upd_aborted, 0) + self.assertGreaterEqual(hs_removed, nrows) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_search_near01.py b/src/third_party/wiredtiger/test/suite/test_search_near01.py index 80f50d2f97a..412cbd894f0 100755 --- a/src/third_party/wiredtiger/test/suite/test_search_near01.py +++ b/src/third_party/wiredtiger/test/suite/test_search_near01.py @@ -30,9 +30,6 @@ import time, wiredtiger, wttest, unittest from wiredtiger import stat -def timestamp_str(t): - return '%x' % t - # test_search_near01.py # Test various prefix search near scenarios. class test_search_near01(wttest.WiredTigerTestCase): diff --git a/src/third_party/wiredtiger/test/suite/test_stat09.py b/src/third_party/wiredtiger/test/suite/test_stat09.py index 9f775f7f949..5345449581e 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat09.py +++ b/src/third_party/wiredtiger/test/suite/test_stat09.py @@ -31,10 +31,6 @@ import wiredtiger, wttest # test_stat09.py # Check oldest active read timestamp statistic - -def timestamp_str(t): - return '%x' % t - class test_stat09(wttest.WiredTigerTestCase): tablename = 'test_stat09' uri = 'table:' + tablename @@ -88,7 +84,7 @@ class test_stat09(wttest.WiredTigerTestCase): for k in keys: self.session.begin_transaction() c[k] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k)) # Create a cursor on statistics that we can use repeatedly allstat_cursor = self.session.open_cursor('statistics:', None, None) @@ -98,15 +94,15 @@ class test_stat09(wttest.WiredTigerTestCase): # Introduce multiple transactions with varying read_timestamp s1 = self.conn.open_session() - s1.begin_transaction('read_timestamp=' + timestamp_str(10)) + s1.begin_transaction('read_timestamp=' + self.timestamp_str(10)) s2 = self.conn.open_session() - s2.begin_transaction('read_timestamp=' + timestamp_str(20)) + s2.begin_transaction('read_timestamp=' + self.timestamp_str(20)) s3 = self.conn.open_session() - s3.begin_transaction('read_timestamp=' + timestamp_str(30)) + s3.begin_transaction('read_timestamp=' + self.timestamp_str(30)) s4 = self.conn.open_session() - s4.begin_transaction('read_timestamp=' + timestamp_str(40)) + s4.begin_transaction('read_timestamp=' + self.timestamp_str(40)) s5 = self.conn.open_session() - s5.begin_transaction('read_timestamp=' + timestamp_str(50)) + s5.begin_transaction('read_timestamp=' + self.timestamp_str(50)) # Check oldest reader self.check_stat_oldest_read(allstat_cursor, 10, commit_range) @@ -117,18 +113,18 @@ class test_stat09(wttest.WiredTigerTestCase): # Set and advance the oldest timestamp, it should be ignored for # determining the oldest active read. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5)) self.check_stat_oldest_read(allstat_cursor, 20, commit_range) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(30)) self.check_stat_oldest_read(allstat_cursor, 20, commit_range) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(150)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(150)) self.check_stat_oldest_read(allstat_cursor, 20, commit_range) # Move the commit timestamp and check again commit_range = 200 - s2.commit_transaction('commit_timestamp=' + timestamp_str(commit_range)) + s2.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_range)) self.check_stat_oldest_read(allstat_cursor, 30, commit_range) # Close all the readers and check the oldest reader, it should be back to 0 diff --git a/src/third_party/wiredtiger/test/suite/test_tiered02.py b/src/third_party/wiredtiger/test/suite/test_tiered02.py index 6e50ec595d6..bed8e57ef54 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered02.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered02.py @@ -48,9 +48,11 @@ class test_tiered02(wttest.WiredTigerTestCase): 'bucket_prefix=%s,' % self.bucket_prefix + \ 'name=%s),tiered_manager=(wait=0)' % self.extension_name - # Load the local store extension, but skip the test if it is missing. + # Load the local store extension. def conn_extensions(self, extlist): - extlist.skip_if_missing = True + # Windows doesn't support dynamically loaded extension libraries. + if os.name == 'nt': + extlist.skip_if_missing = True extlist.extension('storage_sources', self.extension_name) def progress(self, s): diff --git a/src/third_party/wiredtiger/test/suite/test_tiered03.py b/src/third_party/wiredtiger/test/suite/test_tiered03.py index 536bc00dd70..f605342fa5a 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered03.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered03.py @@ -56,15 +56,16 @@ class test_tiered03(wttest.WiredTigerTestCase): if not os.path.exists(self.bucket): os.mkdir(self.bucket) return \ - 'statistics=(all),' + \ 'tiered_storage=(auth_token=%s,' % self.auth_token + \ 'bucket=%s,' % self.bucket + \ 'bucket_prefix=%s,' % self.bucket_prefix + \ 'name=%s)' % self.extension_name - # Load the local store extension, but skip the test if it is missing. + # Load the local store extension. def conn_extensions(self, extlist): - extlist.skip_if_missing = True + # Windows doesn't support dynamically loaded extension libraries. + if os.name == 'nt': + extlist.skip_if_missing = True extlist.extension('storage_sources', self.extension_name) # Test sharing data between a primary and a secondary diff --git a/src/third_party/wiredtiger/test/suite/test_tiered04.py b/src/third_party/wiredtiger/test/suite/test_tiered04.py index bca7f43fc4d..efb5630eda6 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered04.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered04.py @@ -67,9 +67,11 @@ class test_tiered04(wttest.WiredTigerTestCase): 'name=%s,' % self.extension_name + \ 'object_target_size=%s)' % self.object_sys - # Load the local store extension, but skip the test if it is missing. + # Load the local store extension. def conn_extensions(self, extlist): - extlist.skip_if_missing = True + # Windows doesn't support dynamically loaded extension libraries. + if os.name == 'nt': + extlist.skip_if_missing = True extlist.extension('storage_sources', self.extension_name) # Check for a specific string as part of the uri's metadata. @@ -183,10 +185,15 @@ class test_tiered04(wttest.WiredTigerTestCase): retain = self.get_stat(stat.conn.tiered_retention, None) self.assertEqual(retain, new) self.pr("reconfigure flush_tier") - self.session.flush_tier(None) + # Call flush_tier with its various configuration arguments. It is difficult + # to force a timeout or lock contention with a unit test. So just test the + # call for now. + self.session.flush_tier('timeout=10') + self.session.flush_tier('lock_wait=false') + self.session.flush_tier('sync=off') self.pr("reconfigure get stat") calls = self.get_stat(stat.conn.flush_tier, None) - self.assertEqual(calls, 5) + self.assertEqual(calls, 7) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered05.py b/src/third_party/wiredtiger/test/suite/test_tiered05.py index 5ac7293cb46..1d199bad74b 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered05.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered05.py @@ -42,13 +42,14 @@ class test_tiered05(wttest.WiredTigerTestCase): wait = 2 def conn_extensions(self, extlist): - extlist.skip_if_missing = True + # Windows doesn't support dynamically loaded extension libraries. + if os.name == 'nt': + extlist.skip_if_missing = True extlist.extension('storage_sources', self.extension_name) def conn_config(self): os.mkdir(self.bucket) return \ - 'statistics=(fast),' + \ 'tiered_manager=(wait=%d),' % self.wait + \ 'tiered_storage=(auth_token=%s,' % self.auth_token + \ 'bucket=%s,' % self.bucket + \ diff --git a/src/third_party/wiredtiger/test/suite/test_tiered06.py b/src/third_party/wiredtiger/test/suite/test_tiered06.py index d1eb9feae6f..5593c75f3ff 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered06.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered06.py @@ -32,9 +32,11 @@ FileSystem = wiredtiger.FileSystem # easy access to constants # test_tiered06.py # Test the local storage source. class test_tiered06(wttest.WiredTigerTestCase): - # Load the local store extension, but skip the test if it is missing. + # Load the local store extension. def conn_extensions(self, extlist): - extlist.skip_if_missing = True + # Windows doesn't support dynamically loaded extension libraries. + if os.name == 'nt': + extlist.skip_if_missing = True #extlist.extension('storage_sources', # 'local_store=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")') extlist.extension('storage_sources', 'local_store') diff --git a/src/third_party/wiredtiger/test/suite/test_tiered07.py b/src/third_party/wiredtiger/test/suite/test_tiered07.py index 61293325e56..1e123f8cd1c 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered07.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered07.py @@ -41,7 +41,9 @@ class test_tiered07(wttest.WiredTigerTestCase): extension_name = "local_store" def conn_extensions(self, extlist): - extlist.skip_if_missing = True + # Windows doesn't support dynamically loaded extension libraries. + if os.name == 'nt': + extlist.skip_if_missing = True extlist.extension('storage_sources', self.extension_name) def conn_config(self): diff --git a/src/third_party/wiredtiger/test/suite/test_tiered08.py b/src/third_party/wiredtiger/test/suite/test_tiered08.py index af5e7af32bc..b06e84b5f12 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered08.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered08.py @@ -63,9 +63,11 @@ class test_tiered08(wttest.WiredTigerTestCase): 'bucket_prefix=%s,' % self.bucket_prefix + \ 'name=%s),tiered_manager=(wait=0)' % self.extension_name - # Load the local store extension, but skip the test if it is missing. + # Load the local store extension. def conn_extensions(self, extlist): - extlist.skip_if_missing = True + # Windows doesn't support dynamically loaded extension libraries. + if os.name == 'nt': + extlist.skip_if_missing = True extlist.extension('storage_sources', self.extension_name) def get_stat(self, stat): @@ -110,7 +112,7 @@ class test_tiered08(wttest.WiredTigerTestCase): # FIXME-WT-7833 # This test can trigger races in file handle access during flush_tier. # We will re-enable it when that is fixed. - return + self.skipTest('Concurrent flush_tier and insert operations not supported yet.') cfg = self.conn_config() self.pr('Config is: ' + cfg) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp01.py b/src/third_party/wiredtiger/test/suite/test_timestamp01.py index 433347bcec4..f8c768eb2f8 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp01.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp01.py @@ -33,9 +33,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_timestamp01(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' @@ -43,28 +40,28 @@ class test_timestamp01(wttest.WiredTigerTestCase, suite_subprocess): # Cannot set a timestamp on a non-running transaction self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(1 << 5000)), + 'commit_timestamp=' + self.timestamp_str(1 << 5000)), '/only permitted in a running/') # Zero is not permitted self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(0)), + 'commit_timestamp=' + self.timestamp_str(0)), '/zero not permitted/') # Too big is also not permitted self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(1 << 5000)), + 'commit_timestamp=' + self.timestamp_str(1 << 5000)), '/too long/') # Anything other than lower case hexadecimal characters is not permitted self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(-1)), + 'commit_timestamp=' + self.timestamp_str(-1)), '/Failed to parse commit timestamp/') self.session.begin_transaction() @@ -88,13 +85,13 @@ class test_timestamp01(wttest.WiredTigerTestCase, suite_subprocess): # One is okay, as is upper case hex and 2**64 - 1 self.session.begin_transaction() self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(1)) + 'commit_timestamp=' + self.timestamp_str(1)) self.session.begin_transaction() self.session.commit_transaction( 'commit_timestamp=0A78F') self.session.begin_transaction() self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(1 << 64 - 1)) + 'commit_timestamp=' + self.timestamp_str(1 << 64 - 1)) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp02.py b/src/third_party/wiredtiger/test/suite/test_timestamp02.py index b77f409a182..c0f45f82dc5 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp02.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp02.py @@ -35,9 +35,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp02(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp02' uri = 'table:' + tablename @@ -79,58 +76,58 @@ class test_timestamp02(wttest.WiredTigerTestCase, suite_subprocess): for k in keys: self.session.begin_transaction() c[k] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k)) # Don't set a stable timestamp yet. Make sure we can read with # a timestamp before the stable timestamp has been set. # Now check that we see the expected state when reading at each # timestamp for i, t in enumerate(orig_keys): - self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t), dict((k, 1) for k in orig_keys[:i+1])) # Everything up to and including timestamp 100 has been committed. - self.assertTimestampsEqual(self.conn.query_timestamp(), timestamp_str(100)) + self.assertTimestampsEqual(self.conn.query_timestamp(), self.timestamp_str(100)) # Bump the oldest timestamp, we're not going back... - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100)) # Update them and retry. random.shuffle(keys) for k in keys: self.session.begin_transaction() c[k] = 2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k + 100)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k + 100)) # Everything up to and including timestamp 200 has been committed. - self.assertTimestampsEqual(self.conn.query_timestamp(), timestamp_str(200)) + self.assertTimestampsEqual(self.conn.query_timestamp(), self.timestamp_str(200)) # Test that we can manually move the commit timestamp back - self.conn.set_timestamp('commit_timestamp=' + timestamp_str(150)) - self.assertTimestampsEqual(self.conn.query_timestamp(), timestamp_str(150)) - self.conn.set_timestamp('commit_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('commit_timestamp=' + self.timestamp_str(150)) + self.assertTimestampsEqual(self.conn.query_timestamp(), self.timestamp_str(150)) + self.conn.set_timestamp('commit_timestamp=' + self.timestamp_str(200)) # Now the stable timestamp before we read. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(200)) for i, t in enumerate(orig_keys): - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 100), dict((k, (2 if j <= i else 1)) for j, k in enumerate(orig_keys))) # Bump the oldest timestamp, we're not going back... - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(200)) # Remove them and retry random.shuffle(keys) for k in keys: self.session.begin_transaction() del c[k] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k + 200)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k + 200)) # We have to continue to advance the stable timestamp before reading. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(300)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(300)) for i, t in enumerate(orig_keys): - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 200), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 200), dict((k, 2) for k in orig_keys[i+1:])) def test_read_your_writes(self): diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp03.py b/src/third_party/wiredtiger/test/suite/test_timestamp03.py index 862ed241b17..a18d3fd86b0 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp03.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp03.py @@ -36,9 +36,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): table_ts_log = 'ts03_ts_logged' table_ts_nolog = 'ts03_ts_nologged' @@ -46,11 +43,13 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): table_nots_nolog = 'ts03_nots_nologged' types = [ - ('file', dict(uri='file:', use_cg=False, use_index=False)), - ('lsm', dict(uri='lsm:', use_cg=False, use_index=False)), - ('table-cg', dict(uri='table:', use_cg=True, use_index=False)), - ('table-index', dict(uri='table:', use_cg=False, use_index=True)), - ('table-simple', dict(uri='table:', use_cg=False, use_index=False)), + ('file-row', dict(uri='file:', key_format='i', use_cg=False, use_index=False)), + ('file-col', dict(uri='file:', key_format='r', use_cg=False, use_index=False)), + ('lsm', dict(uri='lsm:', key_format='i', use_cg=False, use_index=False)), + ('table-row', dict(uri='table:', key_format='i', use_cg=False, use_index=False)), + ('table-row-index', dict(uri='table:', key_format='i', use_cg=False, use_index=True)), + ('table-col', dict(uri='table:', key_format='r', use_cg=False, use_index=False)), + ('table-col-cg', dict(uri='table:', key_format='r', use_cg=True, use_index=False)), ] ckpt = [ @@ -166,13 +165,14 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): # 3. Table is logged and does not use timestamps. # 4. Table is not logged and does not use timestamps. # - self.session.create(uri_ts_log, 'key_format=i,value_format=S') + format = 'key_format={},value_format=S'.format(self.key_format) + self.session.create(uri_ts_log, format) cur_ts_log = self.session.open_cursor(uri_ts_log) - self.session.create(uri_ts_nolog, 'key_format=i,value_format=S,log=(enabled=false)') + self.session.create(uri_ts_nolog, format + ',log=(enabled=false)') cur_ts_nolog = self.session.open_cursor(uri_ts_nolog) - self.session.create(uri_nots_log, 'key_format=i,value_format=S') + self.session.create(uri_nots_log, format) cur_nots_log = self.session.open_cursor(uri_nots_log) - self.session.create(uri_nots_nolog, 'key_format=i,value_format=S, log=(enabled=false)') + self.session.create(uri_nots_nolog, format + ',log=(enabled=false)') cur_nots_nolog = self.session.open_cursor(uri_nots_nolog) # Insert keys 1..100 each with timestamp=key, in some order @@ -187,7 +187,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() cur_ts_log[k] = self.value cur_ts_nolog[k] = self.value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k)) # Scenario: 1 # Check that we see all the inserted values as per transaction @@ -207,19 +207,19 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): for i, t in enumerate(orig_keys): # Tables using the timestamps should see the values as per the # given read_timestamp - self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t), self.table_ts_log, dict((k, self.value) for k in orig_keys[:i+1])) - self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t), self.table_ts_nolog, dict((k, self.value) for k in orig_keys[:i+1])) # Tables not using the timestamps should see all the values. - self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t), self.table_nots_log, dict((k, self.value) for k in orig_keys)) - self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t), self.table_nots_nolog, dict((k, self.value) for k in orig_keys)) # Bump the oldest_timestamp, we're not going back... - self.assertTimestampsEqual(self.conn.query_timestamp(), timestamp_str(100)) - old_ts = timestamp_str(100) + self.assertTimestampsEqual(self.conn.query_timestamp(), self.timestamp_str(100)) + old_ts = self.timestamp_str(100) self.conn.set_timestamp('oldest_timestamp=' + old_ts) self.conn.set_timestamp('stable_timestamp=' + old_ts) @@ -248,7 +248,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() cur_ts_log[k] = self.value2 cur_ts_nolog[k] = self.value2 - ts = timestamp_str(k + 100) + ts = self.timestamp_str(k + 100) self.session.commit_transaction('commit_timestamp=' + ts) count += 1 @@ -270,7 +270,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): # This scenario is same as earlier one with read_timestamp earlier than # oldest_timestamp and using the option of rounding read_timestamp to # the oldest_timestamp - earlier_ts = timestamp_str(90) + earlier_ts = self.timestamp_str(90) self.check(self.session, 'read_timestamp=' + earlier_ts +',roundup_timestamps=(read=true)', self.table_ts_log, dict((k, self.value) for k in orig_keys)) @@ -294,15 +294,15 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): expected_dict[i+1] = self.value2 # Tables using the timestamps should see the updated values as per # the given read_timestamp - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 100), self.table_ts_log, expected_dict) - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 100), self.table_ts_nolog, expected_dict) # Tables not using the timestamps should see all the data values as # updated values (i.e. value2). - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 100), self.table_nots_log, dict((k, self.value2) for k in orig_keys)) - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 100), self.table_nots_nolog, dict((k, self.value2) for k in orig_keys)) # Take a checkpoint using the given configuration. Then verify @@ -359,7 +359,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): # Update the stable_timestamp to the latest, but not the # oldest_timestamp and make sure we can see the data. Once the # stable_timestamp is moved we should see all keys with value2. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100+nkeys)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100+nkeys)) self.ckpt_backup(self.value2, nkeys, nkeys, nkeys, nkeys) # If we're not using the log we're done. @@ -378,7 +378,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() cur_ts_log[k] = self.value3 cur_ts_nolog[k] = self.value3 - ts = timestamp_str(k + 200) + ts = self.timestamp_str(k + 200) self.session.commit_transaction('commit_timestamp=' + ts) count += 1 diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp04.py b/src/third_party/wiredtiger/test/suite/test_timestamp04.py index 9aff0351e2d..e2aa7609fa7 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp04.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp04.py @@ -35,9 +35,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): table_ts_log = 'table:ts04_ts_logged' table_ts_nolog = 'table:ts04_ts_nologged' @@ -56,12 +53,13 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Minimum cache_size requirement of lsm is 31MB. types = [ - # The commented columnar tests needs to be enabled once rollback to stable for columnar is fixed in (WT-5548). - # ('col_fix', dict(empty=1, cacheSize='cache_size=20MB', extra_config=',key_format=r,value_format=8t')), - # ('col_var', dict(empty=0, cacheSize='cache_size=20MB', extra_config=',key_format=r')), + # FLCS does not yet work in a timestamp world. + #('col_fix', dict(empty=1, \ + # cacheSize='cache_size=20MB', extra_config=',key_format=r,value_format=8t')), ('lsm', dict(empty=0, cacheSize='cache_size=31MB', extra_config=',type=lsm')), ('row', dict(empty=0, cacheSize='cache_size=20MB', extra_config='',)), ('row-smallcache', dict(empty=0, cacheSize='cache_size=2MB', extra_config='',)), + ('var', dict(empty=0, cacheSize='cache_size=20MB', extra_config=',key_format=r')), ] scenarios = make_scenarios(conncfg, types) @@ -146,10 +144,10 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() cur_ts_log[k] = 1 cur_ts_nolog[k] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k)) # Setup an oldest timestamp to ensure state remains in cache. if k == 1: - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cur_ts_log.close() cur_ts_nolog.close() cur_nots_log.close() @@ -157,7 +155,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Scenario: 1 # Check that we see all the inserted values(i.e 1) in all tables - latest_ts = timestamp_str(key_range) + latest_ts = self.timestamp_str(key_range) self.check(self.session, 'read_timestamp=' + latest_ts, self.table_nots_log, dict((k, 1) for k in keys[:])) self.check(self.session, 'read_timestamp=' + latest_ts, @@ -169,7 +167,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Scenario: 2 # Roll back half timestamps. - stable_ts = timestamp_str(key_range // 2) + stable_ts = self.timestamp_str(key_range // 2) self.conn.set_timestamp('stable_timestamp=' + stable_ts) # We're about to test rollback-to-stable which requires a checkpoint to which we can roll back. @@ -231,7 +229,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() cur_ts_log[k] = 2 cur_ts_nolog[k] = 2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k + key_range)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k + key_range)) cur_ts_log.close() cur_ts_nolog.close() cur_nots_log.close() @@ -239,7 +237,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Scenario: 3 # Check that we see all values updated (i.e 2) in all tables. - latest_ts = timestamp_str(2 * key_range) + latest_ts = self.timestamp_str(2 * key_range) self.check(self.session, 'read_timestamp=' + latest_ts, self.table_nots_log, dict((k, 2) for k in keys[:])) self.check(self.session, 'read_timestamp=' + latest_ts, @@ -253,7 +251,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Advance the stable_timestamp by a quarter range and rollback. # Three-fourths of the later timestamps will be rolled back. rolled_range = key_range + key_range // 4 - stable_ts = timestamp_str(rolled_range) + stable_ts = self.timestamp_str(rolled_range) self.conn.set_timestamp('stable_timestamp=' + stable_ts) self.conn.rollback_to_stable() stat_cursor = self.session.open_cursor('statistics:', None, None) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp05.py b/src/third_party/wiredtiger/test/suite/test_timestamp05.py index e7be33de000..852823231c7 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp05.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp05.py @@ -35,13 +35,16 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp05(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:ts05' session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def test_create(self): s = self.session conn = self.conn @@ -51,8 +54,8 @@ class test_timestamp05(wttest.WiredTigerTestCase, suite_subprocess): # Commit at 100 s.begin_transaction() - s.create(self.uri, 'key_format=i,value_format=S') - s.commit_transaction('commit_timestamp=' + timestamp_str(100)) + s.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) + s.commit_transaction('commit_timestamp=' + self.timestamp_str(100)) # Make sure the tree is dirty c = s.open_cursor(self.uri) @@ -65,7 +68,7 @@ class test_timestamp05(wttest.WiredTigerTestCase, suite_subprocess): s = self.session conn = self.conn - s.create(self.uri, 'key_format=i,value_format=S') + s.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) c = s.open_cursor(self.uri, None, 'bulk') # Insert keys 1..100 each with timestamp=key, in some order @@ -79,7 +82,7 @@ class test_timestamp05(wttest.WiredTigerTestCase, suite_subprocess): # Commit at 100 s.begin_transaction() c.close() - s.commit_transaction('commit_timestamp=' + timestamp_str(100)) + s.commit_transaction('commit_timestamp=' + self.timestamp_str(100)) # Make sure the tree is dirty c = s.open_cursor(self.uri) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp06.py b/src/third_party/wiredtiger/test/suite/test_timestamp06.py index d958a3ba3ed..27e32309fdc 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp06.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp06.py @@ -36,15 +36,13 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp06(wttest.WiredTigerTestCase, suite_subprocess): table_ts_log = 'table:ts06_ts_logged' table_ts_nolog = 'table:ts06_ts_nologged' types = [ - ('col_fix', dict(empty=1, extra_config=',key_format=r,value_format=8t')), + # FLCS does not yet work in a timestamp world. + #('col_fix', dict(empty=1, extra_config=',key_format=r,value_format=8t')), ('col_var', dict(empty=0, extra_config=',key_format=r')), ('lsm', dict(empty=0, extra_config=',type=lsm')), ('row', dict(empty=0, extra_config='',)), @@ -125,9 +123,9 @@ class test_timestamp06(wttest.WiredTigerTestCase, suite_subprocess): # Open two timestamp tables: # 1. Table is logged and uses timestamps. # 2. Table is not logged and uses timestamps. - self.session.create(self.table_ts_log, 'key_format=i,value_format=i') + self.session.create(self.table_ts_log, 'key_format=i,value_format=i' + self.extra_config) cur_ts_log = self.session.open_cursor(self.table_ts_log) - self.session.create(self.table_ts_nolog, 'key_format=i,value_format=i,log=(enabled=false)') + self.session.create(self.table_ts_nolog, 'key_format=i,value_format=i,log=(enabled=false)' + self.extra_config) cur_ts_nolog = self.session.open_cursor(self.table_ts_nolog) # Insert keys 1..100 @@ -138,22 +136,22 @@ class test_timestamp06(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() # Make three updates with different timestamps. - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(1)) for k in keys: cur_ts_log[k] = 1 cur_ts_nolog[k] = 1 - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(101)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(101)) for k in keys: cur_ts_log[k] = 2 cur_ts_nolog[k] = 2 - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(201)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(201)) for k in keys: cur_ts_log[k] = 3 cur_ts_nolog[k] = 3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(301)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(301)) cur_ts_log.close() cur_ts_nolog.close() @@ -170,9 +168,9 @@ class test_timestamp06(wttest.WiredTigerTestCase, suite_subprocess): # Check that we see the values till correctly from checkpointed data # files in case of multistep transactions. # Set oldest and stable timestamps - old_ts = timestamp_str(100) + old_ts = self.timestamp_str(100) # Set the stable timestamp such that last update is beyond it. - stable_ts = timestamp_str(200) + stable_ts = self.timestamp_str(200) self.conn.set_timestamp('oldest_timestamp=' + old_ts) self.conn.set_timestamp('stable_timestamp=' + stable_ts) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp07.py b/src/third_party/wiredtiger/test/suite/test_timestamp07.py index e84f9ea5f34..5e25afe1675 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp07.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp07.py @@ -36,14 +36,16 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'ts07_ts_nologged' tablename2 = 'ts07_nots_logged' tablename3 = 'ts07_ts_logged' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + types = [ ('file', dict(uri='file:', use_cg=False, use_index=False)), ('table-cg', dict(uri='table:', use_cg=True, use_index=False)), @@ -61,7 +63,7 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): ('1000keys', dict(nkeys=1000)), ] - scenarios = make_scenarios(types, conncfg, nkeys) + scenarios = make_scenarios(key_format_values, types, conncfg, nkeys) # Binary values. value = u'\u0001\u0002abcd\u0007\u0004' @@ -183,11 +185,11 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): # 2. Table is logged and does not use timestamps. # 3. Table is logged and uses timestamps. # - self.session.create(uri, 'key_format=i,value_format=S,log=(enabled=false)') + self.session.create(uri, 'key_format={},value_format=S,log=(enabled=false)'.format(self.key_format)) c = self.session.open_cursor(uri) - self.session.create(uri2, 'key_format=i,value_format=S') + self.session.create(uri2, 'key_format={},value_format=S'.format(self.key_format)) c2 = self.session.open_cursor(uri2) - self.session.create(uri3, 'key_format=i,value_format=S') + self.session.create(uri3, 'key_format={},value_format=S'.format(self.key_format)) c3 = self.session.open_cursor(uri3) # print "tables created" @@ -201,23 +203,23 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[k] = self.value c3[k] = self.value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k)) # print "value inserted in all tables, reading..." # Now check that we see the expected state when reading at each # timestamp. for k in orig_keys: - self.check(self.session, 'read_timestamp=' + timestamp_str(k), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(k), k, self.value) - self.check(self.session, 'read_timestamp=' + timestamp_str(k), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(k), k + 1, None) # print "all values read, updating timestamps" # Bump the oldest timestamp, we're not going back... - self.assertTimestampsEqual(self.conn.query_timestamp(), timestamp_str(self.nkeys)) - self.oldts = self.stablets = timestamp_str(self.nkeys) + self.assertTimestampsEqual(self.conn.query_timestamp(), self.timestamp_str(self.nkeys)) + self.oldts = self.stablets = self.timestamp_str(self.nkeys) self.conn.set_timestamp('oldest_timestamp=' + self.oldts) self.conn.set_timestamp('stable_timestamp=' + self.stablets) # print "Oldest " + self.oldts @@ -237,7 +239,7 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[k] = self.value2 c3[k] = self.value2 - ts = timestamp_str(k + self.nkeys) + ts = self.timestamp_str(k + self.nkeys) self.session.commit_transaction('commit_timestamp=' + ts) # print "Commit key " + str(k) + " ts " + ts count += 1 @@ -252,7 +254,7 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): # Update the stable timestamp to the latest, but not the oldest # timestamp and make sure we can see the data. Once the stable # timestamp is moved we should see all keys with value2. - self.stablets = timestamp_str(self.nkeys*2) + self.stablets = self.timestamp_str(self.nkeys*2) self.conn.set_timestamp('stable_timestamp=' + self.stablets) # print "check_stable 2" self.check_stable(self.value2, self.nkeys, self.nkeys, self.nkeys) @@ -274,7 +276,7 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[k] = self.value3 c3[k] = self.value3 - ts = timestamp_str(k + self.nkeys*2) + ts = self.timestamp_str(k + self.nkeys*2) self.session.commit_transaction('commit_timestamp=' + ts) # print "Commit key " + str(k) + " ts " + ts count += 1 diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp09.py b/src/third_party/wiredtiger/test/suite/test_timestamp09.py index 91c6435e7d2..2188b919c43 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp09.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp09.py @@ -33,9 +33,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp09' uri = 'table:' + tablename @@ -48,7 +45,7 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): # Begin by adding some data. self.session.begin_transaction() self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(1)) + 'commit_timestamp=' + self.timestamp_str(1)) c[1] = 1 # In a single transaction it is illegal to set a commit timestamp @@ -56,21 +53,21 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): # Check both timestamp_transaction and commit_transaction APIs. self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(3)) + 'commit_timestamp=' + self.timestamp_str(3)) c[3] = 3 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)), + 'commit_timestamp=' + self.timestamp_str(2)), '/older than the first commit timestamp/') self.session.rollback_transaction() self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(4)) + 'commit_timestamp=' + self.timestamp_str(4)) c[4] = 4 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(3)), + 'commit_timestamp=' + self.timestamp_str(3)), '/older than the first commit timestamp/') # Commit timestamp >= Oldest timestamp @@ -78,13 +75,13 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[3] = 3 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(3)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(3)) + 'commit_timestamp=' + self.timestamp_str(3)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(3)) self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)), + 'commit_timestamp=' + self.timestamp_str(2)), '/less than the oldest timestamp/') self.session.rollback_transaction() @@ -92,48 +89,48 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): c[2] = 2 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(2)), + 'commit_timestamp=' + self.timestamp_str(2)), '/less than the oldest timestamp/') self.session.begin_transaction() c[4] = 4 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(4)) + 'commit_timestamp=' + self.timestamp_str(4)) # Oldest timestamp <= Stable timestamp and both oldest and stable # timestamp should proceed forward. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.conn.set_timestamp('oldest_timestamp=' + - timestamp_str(3) + ',stable_timestamp=' + timestamp_str(1)), + self.timestamp_str(3) + ',stable_timestamp=' + self.timestamp_str(1)), '/oldest timestamp \(0, 3\) must not be later than stable timestamp \(0, 1\)/') # Oldest timestamp is 3 at the moment, trying to set it to an earlier # timestamp is a no-op. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) - self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), timestamp_str(3)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), self.timestamp_str(3)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(3) + - ',stable_timestamp=' + timestamp_str(3)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(3) + + ',stable_timestamp=' + self.timestamp_str(3)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) # Stable timestamp is 5 at the moment, trying to set it to an earlier # timestamp is a no-op. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(4)) - self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(4)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), self.timestamp_str(5)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.conn.set_timestamp('oldest_timestamp=' + - timestamp_str(6)), + self.timestamp_str(6)), '/oldest timestamp \(0, 6\) must not be later than stable timestamp \(0, 5\)/') # Commit timestamp >= Stable timestamp. # Check both timestamp_transaction and commit_transaction APIs. # Oldest and stable timestamp are set to 5 at the moment. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(6)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(6)) self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(5)), + 'commit_timestamp=' + self.timestamp_str(5)), '/less than the stable timestamp/') self.session.rollback_transaction() @@ -141,7 +138,7 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): c[5] = 5 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(5)), + 'commit_timestamp=' + self.timestamp_str(5)), '/less than the stable timestamp/') # When explicitly set, commit timestamp for a transaction can be earlier @@ -149,48 +146,48 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[6] = 6 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(6)) + 'commit_timestamp=' + self.timestamp_str(6)) self.session.begin_transaction() c[8] = 8 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(8)) + 'commit_timestamp=' + self.timestamp_str(8)) self.session.begin_transaction() c[7] = 7 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(7)) + 'commit_timestamp=' + self.timestamp_str(7)) # Read timestamp >= Oldest timestamp - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(7) + - ',stable_timestamp=' + timestamp_str(7)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(7) + + ',stable_timestamp=' + self.timestamp_str(7)) with self.expectedStdoutPattern('less than the oldest timestamp'): self.assertRaisesException(wiredtiger.WiredTigerError, - lambda: self.session.begin_transaction('read_timestamp=' + timestamp_str(6))) + lambda: self.session.begin_transaction('read_timestamp=' + self.timestamp_str(6))) # c[8] is not visible at read_timestamp < 8 - self.session.begin_transaction('read_timestamp=' + timestamp_str(7)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(7)) self.assertEqual(c[6], 6) self.assertEqual(c[7], 7) c.set_key(8) self.assertEqual(c.search(), wiredtiger.WT_NOTFOUND) self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(8)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(8)) self.assertEqual(c[6], 6) self.assertEqual(c[7], 7) self.assertEqual(c[8], 8) self.assertTimestampsEqual( - self.conn.query_timestamp('get=oldest_reader'), timestamp_str(8)) + self.conn.query_timestamp('get=oldest_reader'), self.timestamp_str(8)) self.session.commit_transaction() # We can move the oldest timestamp backwards with "force" self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(5) + ',force') + 'oldest_timestamp=' + self.timestamp_str(5) + ',force') with self.expectedStdoutPattern('less than the oldest timestamp'): self.assertRaisesException(wiredtiger.WiredTigerError, - lambda: self.session.begin_transaction('read_timestamp=' + timestamp_str(4))) - self.session.begin_transaction('read_timestamp=' + timestamp_str(6)) + lambda: self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4))) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(6)) self.assertTimestampsEqual( - self.conn.query_timestamp('get=oldest_reader'), timestamp_str(6)) + self.conn.query_timestamp('get=oldest_reader'), self.timestamp_str(6)) self.session.commit_transaction() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp10.py b/src/third_party/wiredtiger/test/suite/test_timestamp10.py index 207370755ab..4af2004c162 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp10.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp10.py @@ -34,9 +34,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): conn_config = 'config_base=false,create,log=(enabled)' session_config = 'isolation=snapshot' @@ -90,16 +87,16 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): curs[i] = i self.pr("i: " + str(i)) self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(i)) + 'commit_timestamp=' + self.timestamp_str(i)) # Set the oldest and stable timestamp a bit earlier than the data # we inserted. Take a checkpoint to the stable timestamp. self.pr("stable ts: " + str(ts)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(ts) + - ',stable_timestamp=' + timestamp_str(ts)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(ts) + + ',stable_timestamp=' + self.timestamp_str(ts)) # This forces a different checkpoint timestamp for each table. self.session.checkpoint() q = self.conn.query_timestamp('get=last_checkpoint') - self.assertTimestampsEqual(q, timestamp_str(ts)) + self.assertTimestampsEqual(q, self.timestamp_str(ts)) return ts def close_and_recover(self, expected_rec_ts): @@ -124,7 +121,7 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): self.open_conn() q = self.conn.query_timestamp('get=recovery') self.pr("query recovery ts: " + q) - self.assertTimestampsEqual(q, timestamp_str(expected_rec_ts)) + self.assertTimestampsEqual(q, self.timestamp_str(expected_rec_ts)) def test_timestamp_recovery(self): # Add some data and checkpoint at a stable timestamp. diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp11.py b/src/third_party/wiredtiger/test/suite/test_timestamp11.py index de5d51ec45e..c4b4210df43 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp11.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp11.py @@ -33,9 +33,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' @@ -51,7 +48,7 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)) + 'commit_timestamp=' + self.timestamp_str(2)) c['key'] = 'value2' c['key2'] = 'value2' self.session.commit_transaction() @@ -64,7 +61,7 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(5)) + 'commit_timestamp=' + self.timestamp_str(5)) c['key'] = 'value5' self.session.commit_transaction() c.close() @@ -81,7 +78,7 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # remain at the non-timestamped value. Also the non-timestamped value # stays regardless of rollbacks or reading at a timestamp. # - stable_ts = timestamp_str(2) + stable_ts = self.timestamp_str(2) self.conn.set_timestamp('stable_timestamp=' + stable_ts) self.session.checkpoint() self.conn.rollback_to_stable() @@ -106,7 +103,7 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(5)) + 'commit_timestamp=' + self.timestamp_str(5)) c['key2'] = 'value5' self.session.commit_transaction() c.close() @@ -140,7 +137,7 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # one at that timestamp and inserted without a timestamp. For the second # we inserted at timestamp 5 after the non-timestamped insert. c = self.session.open_cursor(uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) self.assertEquals(c['key'], 'valueNOTS') self.assertEquals(c['key2'], 'value5') self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp12.py b/src/third_party/wiredtiger/test/suite/test_timestamp12.py index e21ba16158f..e54ed923474 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp12.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp12.py @@ -33,9 +33,6 @@ import shutil, os, wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp12(wttest.WiredTigerTestCase): conn_config = 'config_base=false,create,log=(enabled)' session_config = 'isolation=snapshot' @@ -85,10 +82,10 @@ class test_timestamp12(wttest.WiredTigerTestCase): c_op[i] = 1 c_coll[i] = 1 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(i)) + 'commit_timestamp=' + self.timestamp_str(i)) # Set the oldest and stable timestamp to the end. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(nentries-1) + - ',stable_timestamp=' + timestamp_str(nentries-1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(nentries-1) + + ',stable_timestamp=' + self.timestamp_str(nentries-1)) # Add more data but don't advance the stable timestamp. for i in second_range: @@ -97,7 +94,7 @@ class test_timestamp12(wttest.WiredTigerTestCase): c_coll[i] = 1 self.pr("i: " + str(i)) self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(i)) + 'commit_timestamp=' + self.timestamp_str(i)) # Close and reopen the connection. We cannot use reopen_conn because # we want to test the specific close configuration string. diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp14.py b/src/third_party/wiredtiger/test/suite/test_timestamp14.py index c7a0738fa74..8d6e753bf4d 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp14.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp14.py @@ -35,9 +35,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp14' uri = 'table:' + tablename diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp16.py b/src/third_party/wiredtiger/test/suite/test_timestamp16.py index 5a378eae312..f74557ad2c6 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp16.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp16.py @@ -36,9 +36,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp16(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp16' uri = 'table:' + tablename diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp17.py b/src/third_party/wiredtiger/test/suite/test_timestamp17.py index b282ee49b0a..3c2453b0aa2 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp17.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp17.py @@ -38,9 +38,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp17(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp17' uri = 'table:' + tablename diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp18.py b/src/third_party/wiredtiger/test/suite/test_timestamp18.py index 746551b1dec..c7f20dbd9a4 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp18.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp18.py @@ -38,9 +38,6 @@ import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp18(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' @@ -53,7 +50,7 @@ class test_timestamp18(wttest.WiredTigerTestCase): def test_ts_writes_with_non_ts_write(self): uri = 'table:test_timestamp18' self.session.create(uri, 'key_format=S,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) value1 = 'a' * 500 @@ -65,17 +62,17 @@ class test_timestamp18(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Add a non-timestamped delete. # Let's do every second key to ensure that we get the truncation right and don't @@ -91,7 +88,7 @@ class test_timestamp18(wttest.WiredTigerTestCase): self.session.checkpoint() for ts in range(2, 4): - self.session.begin_transaction('read_timestamp=' + timestamp_str(ts)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts)) for i in range(1, 10000): # The non-timestamped delete should cover all the previous writes and make them effectively # invisible. diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp19.py b/src/third_party/wiredtiger/test/suite/test_timestamp19.py index fa83d53827a..9f5b71df843 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp19.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp19.py @@ -31,11 +31,8 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - class test_timestamp19(wttest.WiredTigerTestCase): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' def updates(self, uri, value, ds, nrows, commit_ts): @@ -44,7 +41,7 @@ class test_timestamp19(wttest.WiredTigerTestCase): for i in range(0, nrows): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def test_timestamp(self): @@ -62,8 +59,8 @@ class test_timestamp19(wttest.WiredTigerTestCase): value_z = 'z' * 1000 # Set the oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ', stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ', stable_timestamp=' + self.timestamp_str(10)) # Insert values with varying timestamps. self.updates(uri, value_x, ds, nrows, 20) @@ -74,8 +71,8 @@ class test_timestamp19(wttest.WiredTigerTestCase): self.session.checkpoint('use_timestamp=true') # Move the oldest and stable timestamps to 40. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(40) + - ', stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(40) + + ', stable_timestamp=' + self.timestamp_str(40)) # Update values. self.updates(uri, value_z, ds, nrows, 50) @@ -91,20 +88,20 @@ class test_timestamp19(wttest.WiredTigerTestCase): self.session = self.setUpSessionOpen(self.conn) # The oldest timestamp on recovery is 40. Trying to set it earlier is a no-op. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) - self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), timestamp_str(40)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), self.timestamp_str(40)) # Trying to set an earlier stable timestamp is an error. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)), + lambda: self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)), '/oldest timestamp \(0, 40\) must not be later than stable timestamp \(0, 10\)/') - self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), timestamp_str(40)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), self.timestamp_str(40)) # Move the oldest and stable timestamps to 70. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(70) + - ', stable_timestamp=' + timestamp_str(70)) - self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), timestamp_str(70)) - self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), timestamp_str(70)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(70) + + ', stable_timestamp=' + self.timestamp_str(70)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), self.timestamp_str(70)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), self.timestamp_str(70)) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp20.py b/src/third_party/wiredtiger/test/suite/test_timestamp20.py index c2ff132cd0b..c4edcbfe592 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp20.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp20.py @@ -28,9 +28,6 @@ import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - # test_timestamp20.py # Exercise fixing up of out-of-order updates in the history store. class test_timestamp20(wttest.WiredTigerTestCase): @@ -40,7 +37,7 @@ class test_timestamp20(wttest.WiredTigerTestCase): def test_timestamp20_standard(self): uri = 'table:test_timestamp20' self.session.create(uri, 'key_format=S,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) value1 = 'a' * 500 @@ -52,33 +49,33 @@ class test_timestamp20(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) old_reader_session = self.conn.open_session() old_reader_cursor = old_reader_session.open_cursor(uri) - old_reader_session.begin_transaction('read_timestamp=' + timestamp_str(20)) + old_reader_session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) # Now put two updates out of order. 5 will go to the history store and will trigger a # correction to the existing contents. for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value4 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(25)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(25)) self.session.begin_transaction() cursor[str(i)] = value5 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(40)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) - self.session.begin_transaction('read_timestamp=' + timestamp_str(30)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(30)) for i in range(1, 10000): self.assertEqual(cursor[str(i)], value4) self.session.rollback_transaction() @@ -94,7 +91,7 @@ class test_timestamp20(wttest.WiredTigerTestCase): def test_timestamp20_modify(self): uri = 'table:test_timestamp20' self.session.create(uri, 'key_format=S,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) value1 = 'a' * 500 @@ -105,20 +102,20 @@ class test_timestamp20(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Now apply a series of modifies. for i in range(1, 10000): self.session.begin_transaction() cursor.set_key(str(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) for i in range(1, 10000): self.session.begin_transaction() cursor.set_key(str(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) # Open an old reader at this point. # @@ -126,7 +123,7 @@ class test_timestamp20(wttest.WiredTigerTestCase): # has been squashed into a full update. old_reader_session = self.conn.open_session() old_reader_cursor = old_reader_session.open_cursor(uri) - old_reader_session.begin_transaction('read_timestamp=' + timestamp_str(20)) + old_reader_session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) # Now apply the last modify. # This will be the end of the chain of modifies. @@ -134,21 +131,21 @@ class test_timestamp20(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(str(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(40)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) # Now put two updates out of order. 5 will go to the history store and will trigger a # correction to the existing contents. for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(25)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(25)) self.session.begin_transaction() cursor[str(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(50)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) # Open up a new transaction and read at 30. # We shouldn't be able to see past 5 due to txnid visibility. - self.session.begin_transaction('read_timestamp=' + timestamp_str(30)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(30)) for i in range(1, 10000): self.assertEqual(cursor[str(i)], value2) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp21.py b/src/third_party/wiredtiger/test/suite/test_timestamp21.py index a3b9ca03ac1..6d553a36246 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp21.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp21.py @@ -28,9 +28,6 @@ import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - # test_timestamp21.py # Test read timestamp configuration that allows read timestamp to be older than oldest. class test_timestamp21(wttest.WiredTigerTestCase): @@ -40,58 +37,58 @@ class test_timestamp21(wttest.WiredTigerTestCase): uri = 'table:test_timestamp21' self.session.create(uri, 'key_format=i,value_format=i') session2 = self.setUpSessionOpen(self.conn) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) cursor2 = session2.open_cursor(uri) # Insert first value at timestamp 10. self.session.begin_transaction() cursor[1] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Begin a read transaction at timestamp 5. - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) # Move the oldest timestamp beyond the currently open transactions read timestamp. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(8)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(8)) # Begin a transaction with a read timestamp of 6 and read_before_oldest specified. self.assertEqual(session2.begin_transaction( - 'read_timestamp=' + timestamp_str(6) + ',read_before_oldest=true'), 0) + 'read_timestamp=' + self.timestamp_str(6) + ',read_before_oldest=true'), 0) session2.rollback_transaction() # Begin a transaction with a read timestamp of 6 and no additional config. with self.expectedStdoutPattern('less than the oldest timestamp'): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: session2.begin_transaction( - 'read_timestamp=' + timestamp_str(6))) + 'read_timestamp=' + self.timestamp_str(6))) # Begin a transaction with the config specified but no read timestamp. session2.begin_transaction('read_before_oldest=true') # Set a read timestamp behind the oldest timestamp. - self.assertEqual(session2.timestamp_transaction('read_timestamp=' + timestamp_str(5)), 0) + self.assertEqual(session2.timestamp_transaction('read_timestamp=' + self.timestamp_str(5)), 0) session2.rollback_transaction() # Begin a transaction with a read timestamp of 5 and read_before_oldest specified. self.assertEqual(session2.begin_transaction( - 'read_timestamp=' + timestamp_str(5) + ',read_before_oldest=true'), 0) + 'read_timestamp=' + self.timestamp_str(5) + ',read_before_oldest=true'), 0) session2.rollback_transaction() # Begin a transaction with a read timestamp of 4 and read_before_oldest specified. We get a # different std out message in this scenario. with self.expectedStdoutPattern('less than the pinned timestamp'): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: session2.begin_transaction( - 'read_timestamp=' + timestamp_str(4) + ',read_before_oldest=true')) + 'read_timestamp=' + self.timestamp_str(4) + ',read_before_oldest=true')) # Begin a transaction with a read timestamp of 6 and read_before_oldest off, this will have # the same behaviour as not specifying it. with self.expectedStdoutPattern('less than the oldest timestamp'): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: session2.begin_transaction( - 'read_timestamp=' + timestamp_str(6) + ',read_before_oldest=false')) + 'read_timestamp=' + self.timestamp_str(6) + ',read_before_oldest=false')) # Expect an error when we use roundup timestamps alongside allow read timestamp before # oldest. self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: session2.begin_transaction( - 'read_timestamp=' + timestamp_str(6) + + 'read_timestamp=' + self.timestamp_str(6) + ',read_before_oldest=true,roundup_timestamps=(read)'), '/cannot specify roundup_timestamps.read and read_before_oldest/') diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp22.py b/src/third_party/wiredtiger/test/suite/test_timestamp22.py index d03e18d8d72..e02dbf99e09 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp22.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp22.py @@ -32,9 +32,6 @@ import wiredtiger, wttest, re, suite_random from wtdataset import SimpleDataSet from contextlib import contextmanager -def timestamp_str(t): - return '%x' % t - class test_timestamp22(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' @@ -124,7 +121,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): this_commit_ts = -1 if self.do_illegal(): # setting durable timestamp must be after prepare call - config += ',durable_timestamp=' + timestamp_str(self.gen_ts(commit_ts)) + config += ',durable_timestamp=' + self.timestamp_str(self.gen_ts(commit_ts)) ok = False # ODDITY: if we set the durable timestamp (which is illegal at this point), and set a @@ -143,7 +140,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): else: # It's possible this will succeed, we'll check below. this_commit_ts = self.gen_ts(commit_ts) - config += ',commit_timestamp=' + timestamp_str(this_commit_ts) + config += ',commit_timestamp=' + self.timestamp_str(this_commit_ts) if this_commit_ts >= 0: if this_commit_ts < running_commit_ts: @@ -161,7 +158,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): session = self.session needs_rollback = False prepare_config = None - commit_config = 'commit_timestamp=' + timestamp_str(commit_ts) + commit_config = 'commit_timestamp=' + self.timestamp_str(commit_ts) tstxn1_config = '' tstxn2_config = '' @@ -173,11 +170,11 @@ class test_timestamp22(wttest.WiredTigerTestCase): # Occasionally put a durable timestamp on a commit without a prepare, # that will be an error. if do_prepare or not ok_commit: - commit_config += ',durable_timestamp=' + timestamp_str(durable_ts) + commit_config += ',durable_timestamp=' + self.timestamp_str(durable_ts) cursor = session.open_cursor(self.uri) prepare_ts = self.gen_ts(commit_ts) - prepare_config = 'prepare_timestamp=' + timestamp_str(prepare_ts) - begin_config = '' if read_ts < 0 else 'read_timestamp=' + timestamp_str(read_ts) + prepare_config = 'prepare_timestamp=' + self.timestamp_str(prepare_ts) + begin_config = '' if read_ts < 0 else 'read_timestamp=' + self.timestamp_str(read_ts) # We might do timestamp_transaction calls either before/after inserting # values, or both. @@ -302,7 +299,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): for ts_name in ['oldest', 'stable', 'commit', 'durable']: val = eval(ts_name) if val >= 0: - configs.append(ts_name + '_timestamp=' + timestamp_str(val)) + configs.append(ts_name + '_timestamp=' + self.timestamp_str(val)) return ','.join(configs) # Determine whether we expect the set_timestamp to succeed. @@ -357,8 +354,8 @@ class test_timestamp22(wttest.WiredTigerTestCase): self.pr('updating stable: ' + str(stable)) # Make sure the state of global timestamps is what we think. - expect_query_oldest = timestamp_str(self.oldest_ts) - expect_query_stable = timestamp_str(self.stable_ts) + expect_query_oldest = self.timestamp_str(self.oldest_ts) + expect_query_stable = self.timestamp_str(self.stable_ts) query_oldest = self.conn.query_timestamp('get=oldest') query_stable = self.conn.query_timestamp('get=stable') diff --git a/src/third_party/wiredtiger/test/suite/test_truncate05.py b/src/third_party/wiredtiger/test/suite/test_truncate05.py index eb9e1ec922f..73f9f5444ed 100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate05.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate05.py @@ -28,9 +28,6 @@ import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - # test_truncate05.py # Test various fast truncate visibility scenarios class test_truncate05(wttest.WiredTigerTestCase): @@ -49,7 +46,7 @@ class test_truncate05(wttest.WiredTigerTestCase): for i in range(1, 1000): self.session.begin_transaction() cursor[i] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Reopen the connection to force all content to disk. self.reopen_conn() @@ -59,17 +56,17 @@ class test_truncate05(wttest.WiredTigerTestCase): # Insert a single update at a later timestamp. self.session.begin_transaction() cursor[500] = value2 - self.assertEqual(self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)), 0) + self.assertEqual(self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)), 0) # Insert a bunch of other content to fill the database and evict the committed update. for i in range(1000, 20000): self.session.begin_transaction() cursor[i] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Start a transaction with an earlier read timestamp than the commit timestamp of the # previous update. - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) # Truncate from key 1 to 1000. start = self.session.open_cursor(uri, None) diff --git a/src/third_party/wiredtiger/test/suite/test_txn03.py b/src/third_party/wiredtiger/test/suite/test_txn03.py index fd365240301..151b98ca8e2 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn03.py +++ b/src/third_party/wiredtiger/test/suite/test_txn03.py @@ -37,13 +37,18 @@ class test_txn03(wttest.WiredTigerTestCase): tablename = 'test_txn03' uri1 = 'table:' + tablename + "_1" uri2 = 'table:' + tablename + "_2" - key_str = "TEST_KEY1" - data_str1 = "VAL" - data_str2 = "TEST_VAL1" + key = "TEST_KEY1" + data1 = "VAL" + data2 = "TEST_VAL1" nentries = 1000 scenarios = make_scenarios([ - ('var', dict(create_params = "key_format=S,value_format=S")), + ('row', dict(create_params = "key_format=S,value_format=S", + key = "TEST_KEY1", data1 = "VAL", data2 = "TEST_VAL1")), + ('var', dict(create_params = "key_format=r,value_format=S", + key = 123, data1 = "VAL", data2 = "TEST_VAL1")), + ('fix', dict(create_params = "key_format=r,value_format=8t", + key = 123, data1 = 0x17, data2 = 0xaa)), ]) def test_ops(self): @@ -52,17 +57,17 @@ class test_txn03(wttest.WiredTigerTestCase): # Set up the table with entries for 1 and 10 # We use the overwrite config so insert can update as needed. c = self.session.open_cursor(self.uri1, None, 'overwrite') - c[self.key_str] = self.data_str1 + c[self.key] = self.data1 c.close() c = self.session.open_cursor(self.uri2, None, 'overwrite') - c[self.key_str] = self.data_str1 + c[self.key] = self.data1 c.close() # Update the first table - this update should be visible in the # new session. self.session.begin_transaction() c = self.session.open_cursor(self.uri1, None, 'overwrite') - c[self.key_str] = self.data_str2 + c[self.key] = self.data2 self.session.commit_transaction() c.close() @@ -75,16 +80,16 @@ class test_txn03(wttest.WiredTigerTestCase): # Make an update in the first session. self.session.begin_transaction() c = self.session.open_cursor(self.uri2, None, 'overwrite') - c[self.key_str] = self.data_str2 + c[self.key] = self.data2 self.session.commit_transaction() c.close() - t1c.set_key(self.key_str) + t1c.set_key(self.key) t1c.search() - t2c.set_key(self.key_str) + t2c.set_key(self.key) t2c.search() - self.assertEqual(t1c.get_value(), self.data_str2) - self.assertEqual(t2c.get_value(), self.data_str1) + self.assertEqual(t1c.get_value(), self.data2) + self.assertEqual(t2c.get_value(), self.data1) # Clean up t1c.close() diff --git a/src/third_party/wiredtiger/test/suite/test_txn06.py b/src/third_party/wiredtiger/test/suite/test_txn06.py index 86b0913a17b..1bcdac1d4b0 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn06.py +++ b/src/third_party/wiredtiger/test/suite/test_txn06.py @@ -32,6 +32,7 @@ from suite_subprocess import suite_subprocess from wtdataset import SimpleDataSet import wiredtiger, wttest +from wtscenario import make_scenarios class test_txn06(wttest.WiredTigerTestCase, suite_subprocess): conn_config = 'verbose=[transaction]' @@ -40,13 +41,23 @@ class test_txn06(wttest.WiredTigerTestCase, suite_subprocess): source_uri = 'table:' + tablename + "_src" nrows = 100000 + format_values = [ + ('row', dict(key_format = 'S', value_format='S')), + ('var', dict(key_format = 'r', value_format='S')), + ('fix', dict(key_format = 'r', value_format='8t')), + ] + scenarios = make_scenarios(format_values) + def test_long_running(self): # Populate a table - SimpleDataSet(self, self.source_uri, self.nrows).populate() + ds = SimpleDataSet(self, self.source_uri, self.nrows, + key_format=self.key_format, value_format=self.value_format) + ds.populate() # Now scan the table and copy the rows into a new table. The cursor will keep the snapshot # in self.session pinned while the inserts cause new IDs to be allocated. - c_src = self.session.create(self.uri, "key_format=S,value_format=S") + format = "key_format={},value_format={}".format(self.key_format, self.value_format) + c_src = self.session.create(self.uri, format) c_src = self.session.open_cursor(self.source_uri) insert_session = self.conn.open_session() c = insert_session.open_cursor(self.uri) diff --git a/src/third_party/wiredtiger/test/suite/test_txn07.py b/src/third_party/wiredtiger/test/suite/test_txn07.py index 44624f82af9..b25b9edf6cc 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn07.py +++ b/src/third_party/wiredtiger/test/suite/test_txn07.py @@ -49,13 +49,9 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): ] types = [ - ('row', dict(tabletype='row', - create_params = 'key_format=i,value_format=S')), - # The commented columnar tests needs to be enabled once rollback to stable for columnar is fixed in (WT-5548). - # ('var', dict(tabletype='var', - # create_params = 'key_format=r,value_format=S')), - # ('fix', dict(tabletype='fix', - # create_params = 'key_format=r,value_format=8t')), + ('row', dict(tabletype='row', create_params = 'key_format=i,value_format=S')), + ('var', dict(tabletype='var', create_params = 'key_format=r,value_format=S')), + ('fix', dict(tabletype='fix', create_params = 'key_format=r,value_format=8t')), ] op1s = [ ('trunc-all', dict(op1=('all', 0))), diff --git a/src/third_party/wiredtiger/test/suite/test_txn08.py b/src/third_party/wiredtiger/test/suite/test_txn08.py index d8f4188a5d8..9f1032272cd 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn08.py +++ b/src/third_party/wiredtiger/test/suite/test_txn08.py @@ -34,20 +34,27 @@ import fnmatch, os, shutil, run, time from suite_subprocess import suite_subprocess from wiredtiger import stat import wttest +from wtscenario import make_scenarios class test_txn08(wttest.WiredTigerTestCase, suite_subprocess): logmax = "100K" tablename = 'test_txn08' uri = 'table:' + tablename + key_format_values = [ + ('col', dict(key_format='r')), + ('row', dict(key_format='i')) + ] + scenarios = make_scenarios(key_format_values) + # Turn on logging for this test. def conn_config(self): return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ 'transaction_sync="(method=dsync,enabled)"' def test_printlog_unicode(self): - # print "Creating %s with config '%s'" % (self.uri, self.create_params) - create_params = 'key_format=i,value_format=S' + create_params = 'key_format={},value_format=S'.format(self.key_format) + # print "Creating %s with config '%s'" % (self.uri, create_params) self.session.create(self.uri, create_params) c = self.session.open_cursor(self.uri, None) @@ -56,7 +63,7 @@ class test_txn08(wttest.WiredTigerTestCase, suite_subprocess): value = u'\u0001\u0002abcd\u0003\u0004' self.session.begin_transaction() - for k in range(5): + for k in range(1, 6): c[k] = value self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_txn13.py b/src/third_party/wiredtiger/test/suite/test_txn13.py index 72bfe35386d..b11a3cb41be 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn13.py +++ b/src/third_party/wiredtiger/test/suite/test_txn13.py @@ -43,14 +43,20 @@ class test_txn13(wttest.WiredTigerTestCase, suite_subprocess): # We use 8 ops here to get around the 10 operation check done by WiredTiger to determine if # a transaction is blocking or not. nops = 8 - create_params = 'key_format=i,value_format=S' + + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] # The 1gb, 2gb and 4gb scenario names refer to the valuesize * nops. - scenarios = make_scenarios([ + size_values = [ ('1gb', dict(expect_err=False, valuesize=134217728)), ('2gb', dict(expect_err=False, valuesize=268435456)), ('4gb', dict(expect_err=True, valuesize=536870912)) - ]) + ] + + scenarios = make_scenarios(key_format_values, size_values) # Turn on logging for this test. def conn_config(self): @@ -59,9 +65,11 @@ class test_txn13(wttest.WiredTigerTestCase, suite_subprocess): @wttest.longtest('txn tests with huge values') def test_large_values(self): - # print "Creating %s with config '%s'" % (self.uri, self.create_params) + create_params = 'key_format={},value_format=S'.format(self.key_format) + + # print "Creating %s with config '%s'" % (self.uri, create_params) # print "Running with %d" % (self.valuesize) - self.session.create(self.uri, self.create_params) + self.session.create(self.uri, create_params) c = self.session.open_cursor(self.uri, None) # We want to test very large values. Generate 'nops' records within @@ -70,7 +78,7 @@ class test_txn13(wttest.WiredTigerTestCase, suite_subprocess): gotException = False self.session.begin_transaction() - for k in range(self.nops): + for k in range(1, self.nops + 1): value = valuepfx + str(k) c[k] = value diff --git a/src/third_party/wiredtiger/test/suite/test_txn14.py b/src/third_party/wiredtiger/test/suite/test_txn14.py index 24c51023cb4..f3eaac6cd6b 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn14.py +++ b/src/third_party/wiredtiger/test/suite/test_txn14.py @@ -38,7 +38,6 @@ import wttest class test_txn14(wttest.WiredTigerTestCase, suite_subprocess): t1 = 'table:test_txn14_1' - create_params = 'key_format=i,value_format=i' entries = 10000 extra_entries = 5 conn_config = 'log=(archive=false,enabled,file_max=100K)' @@ -46,9 +45,12 @@ class test_txn14(wttest.WiredTigerTestCase, suite_subprocess): sync_list = [ ('write', dict(sync='off')), ('sync', dict(sync='on')), - ('bg', dict(sync='background')), ] - scenarios = make_scenarios(sync_list) + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(sync_list, key_format_values) def test_log_flush(self): # Here's the strategy: @@ -59,32 +61,27 @@ class test_txn14(wttest.WiredTigerTestCase, suite_subprocess): # - Make recovery run. # - Confirm flushed data is in the table. # - self.session.create(self.t1, self.create_params) + create_params = 'key_format={},value_format=i'.format(self.key_format) + self.session.create(self.t1, create_params) c = self.session.open_cursor(self.t1, None, None) - for i in range(self.entries): + for i in range(1, self.entries + 1): c[i] = i + 1 cfgarg='sync=%s' % self.sync self.pr('cfgarg ' + cfgarg) self.session.log_flush(cfgarg) - for i in range(self.extra_entries): + for i in range(1, self.extra_entries + 1): c[i+self.entries] = i + self.entries + 1 c.close() self.session.log_flush(cfgarg) - if self.sync == 'background': - # If doing a background flush, wait 30 seconds. I have seen an - # individual log file's fsync take more than a second on some - # systems, and we've seen timeouts at 10 seconds on systems - # with slow I/O. So give it time to flush perhaps a few files. - self.session.transaction_sync('timeout_ms=30000') simulate_crash_restart(self, ".", "RESTART") c = self.session.open_cursor(self.t1, None, None) - i = 0 + i = 1 for key, value in c: self.assertEqual(i, key) self.assertEqual(i+1, value) i += 1 all = self.entries + self.extra_entries - self.assertEqual(i, all) + self.assertEqual(i, all + 1) c.close() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_txn15.py b/src/third_party/wiredtiger/test/suite/test_txn15.py index 319f9a9ec3a..a266bf37582 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn15.py +++ b/src/third_party/wiredtiger/test/suite/test_txn15.py @@ -38,7 +38,6 @@ import wttest class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:test_txn15_1' - create_params = 'key_format=i,value_format=i' entries = 100 # Turn on logging for this test. def conn_config(self): @@ -48,6 +47,10 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): 'transaction_sync=(enabled=%s),' % self.conn_enable + \ 'transaction_sync=(method=%s),' % self.conn_method + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] conn_sync_enabled = [ ('en_off', dict(conn_enable='false')), ('en_on', dict(conn_enable='true')), @@ -71,7 +74,7 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): ('c_none', dict(commit_sync=None)), ('c_off', dict(commit_sync='sync=off')), ] - scenarios = make_scenarios(conn_sync_enabled, conn_sync_method, + scenarios = make_scenarios(key_format_values, conn_sync_enabled, conn_sync_method, begin_sync, commit_sync) # Given the different configuration settings determine if this group @@ -111,7 +114,8 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): if self.begin_sync != None and self.commit_sync != None: return - self.session.create(self.uri, self.create_params) + create_params = 'key_format={},value_format=i'.format(self.key_format) + self.session.create(self.uri, create_params) stat_cursor = self.session.open_cursor('statistics:', None, None) # @@ -126,7 +130,7 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(self.uri, None, None) self.session.begin_transaction(self.begin_sync) - for i in range(self.entries): + for i in range(1, self.entries + 1): c[i] = i + 1 self.session.commit_transaction(self.commit_sync) c.close() diff --git a/src/third_party/wiredtiger/test/suite/test_txn17.py b/src/third_party/wiredtiger/test/suite/test_txn17.py index 84b7a5feaa3..f1f2c8a2b49 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn17.py +++ b/src/third_party/wiredtiger/test/suite/test_txn17.py @@ -35,16 +35,13 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_txn17(wttest.WiredTigerTestCase, suite_subprocess): def test_txn_api(self): # Test API functionality tagged as requires_transaction. # Cannot set a timestamp on a non-running transaction. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(1 << 5000)), + 'commit_timestamp=' + self.timestamp_str(1 << 5000)), '/only permitted in a running/') # Cannot call commit on a non-running transaction. @@ -71,12 +68,5 @@ class test_txn17(wttest.WiredTigerTestCase, suite_subprocess): '/not permitted in a running transaction/') self.session.rollback_transaction() - # Cannot call transaction_sync while a transaction is running. - self.session.begin_transaction() - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.transaction_sync(), - '/not permitted in a running transaction/') - self.session.rollback_transaction() - if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_txn18.py b/src/third_party/wiredtiger/test/suite/test_txn18.py index c764993d682..e8bb10bdbcf 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn18.py +++ b/src/third_party/wiredtiger/test/suite/test_txn18.py @@ -33,15 +33,21 @@ import fnmatch, os, shutil, time from suite_subprocess import suite_subprocess import wiredtiger, wttest +from wtscenario import make_scenarios class test_txn18(wttest.WiredTigerTestCase, suite_subprocess): t1 = 'table:test_txn18' - create_params = 'key_format=i,value_format=i' conn_config = 'log=(archive=false,enabled,file_max=100K),' + \ 'transaction_sync=(method=dsync,enabled)' conn_recerror = conn_config + ',log=(recover=error)' conn_recon = conn_config + ',log=(recover=on)' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def simulate_crash(self, olddir, newdir): ''' Simulate a crash from olddir and restart in newdir. ''' # with the connection still open, copy files to new directory @@ -71,13 +77,14 @@ class test_txn18(wttest.WiredTigerTestCase, suite_subprocess): # # If we aren't tracking file IDs properly, it's possible that # we'd end up apply the log records for t2 to table t1. - self.session.create(self.t1, self.create_params) + create_params = 'key_format={},value_format=i'.format(self.key_format) + self.session.create(self.t1, create_params) # # Since we're logging, we need to flush out the meta-data file # from the create. self.session.checkpoint() c = self.session.open_cursor(self.t1, None, None) - for i in range(10000): + for i in range(1, 10001): c[i] = i + 1 c.close() olddir = "." @@ -103,12 +110,12 @@ class test_txn18(wttest.WiredTigerTestCase, suite_subprocess): # Make sure the data we added originally is there self.session = self.setUpSessionOpen(self.conn) c = self.session.open_cursor(self.t1, None, None) - i = 0 + i = 1 for key, value in c: self.assertEqual(i, key) self.assertEqual(i+1, value) i += 1 - self.assertEqual(i, 10000) + self.assertEqual(i, 10001) c.close() self.close_conn() # Reopening with recover=error after a clean shutdown should succeed. diff --git a/src/third_party/wiredtiger/test/suite/test_txn19.py b/src/third_party/wiredtiger/test/suite/test_txn19.py index 18c0636d603..d975dc43a7c 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn19.py +++ b/src/third_party/wiredtiger/test/suite/test_txn19.py @@ -113,6 +113,11 @@ class test_txn19(wttest.WiredTigerTestCase, suite_subprocess): nrecords = [('nrecords=10', dict(nrecords=10)), ('nrecords=11', dict(nrecords=11))] + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + # This function prunes out unnecessary or problematic test cases # from the list of scenarios. def includeFunc(name, dictarg): @@ -131,11 +136,10 @@ class test_txn19(wttest.WiredTigerTestCase, suite_subprocess): return True scenarios = make_scenarios( - corruption_type, corruption_pos, nrecords, + key_format_values, corruption_type, corruption_pos, nrecords, include=includeFunc, prune=20, prunelong=1000) uri = 'table:test_txn19' - create_params = 'key_format=i,value_format=S' # Return the log file number that contains the given record # number. In this test, two records fit into each log file, and @@ -282,7 +286,8 @@ class test_txn19(wttest.WiredTigerTestCase, suite_subprocess): # Then does a restart with recovery, then starts again with salvage, # and finally starts again with recovery (adding new records). - self.session.create(self.uri, self.create_params) + create_params = 'key_format=i,value_format=S'.format(self.key_format) + self.session.create(self.uri, create_params) self.inserts([x for x in range(0, self.nrecords)]) newdir = "RESTART" copy_for_crash_restart(self.home, newdir) @@ -385,6 +390,11 @@ class test_txn19_meta(wttest.WiredTigerTestCase, suite_subprocess): ('WiredTiger.wt', dict(filename='WiredTiger.wt')), ('WiredTigerHS.wt', dict(filename='WiredTigerHS.wt')), ] + # Configure the database type. + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] # In many cases, wiredtiger_open without any salvage options will # just work. We list those cases here. @@ -433,10 +443,9 @@ class test_txn19_meta(wttest.WiredTigerTestCase, suite_subprocess): "garbage-end:WiredTiger.basecfg", ] - scenarios = make_scenarios(corruption_scenarios, filename_scenarios) + scenarios = make_scenarios(key_format_values, corruption_scenarios, filename_scenarios) uri = 'table:test_txn19_meta_' ntables = 5 - create_params = 'key_format=i,value_format=S' nrecords = 1000 # records per table. suffixes = [ str(x) for x in range(0, ntables)] # [ '0', '1', ... ] @@ -507,11 +516,12 @@ class test_txn19_meta(wttest.WiredTigerTestCase, suite_subprocess): def test_corrupt_meta(self): newdir = "RESTART" newdir2 = "RESTART2" - expect = list(range(0, self.nrecords)) + expect = list(range(1, self.nrecords + 1)) salvage_config = self.base_config + ',salvage=true' + create_params = 'key_format={},value_format=S'.format(self.key_format) for suffix in self.suffixes: - self.session.create(self.uri + suffix, self.create_params) + self.session.create(self.uri + suffix, create_params) self.inserts(expect) # Simulate a crash by copying the contents of the directory diff --git a/src/third_party/wiredtiger/test/suite/test_txn20.py b/src/third_party/wiredtiger/test/suite/test_txn20.py index d51e180fe20..b6d0c8306f2 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn20.py +++ b/src/third_party/wiredtiger/test/suite/test_txn20.py @@ -36,18 +36,21 @@ from wtscenario import make_scenarios class test_txn20(wttest.WiredTigerTestCase): uri = 'table:test_txn' + key_format_values = [ + ('string-row', dict(key_format='S', key='key')), + ('column', dict(key_format='r', key=12)), + ] iso_types = [ ('isolation_read_uncommitted', dict(isolation='read-uncommitted')), ('isolation_read_committed', dict(isolation='read-committed')), ('isolation_snapshot', dict(isolation='snapshot')) ] - scenarios = make_scenarios(iso_types) - key = 'key' + scenarios = make_scenarios(key_format_values, iso_types) old_value = 'value: old' new_value = 'value: new' def test_isolation_level(self): - self.session.create(self.uri, 'key_format=S,value_format=S') + self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(self.uri, None) cursor[self.key] = self.old_value diff --git a/src/third_party/wiredtiger/test/suite/test_txn22.py b/src/third_party/wiredtiger/test/suite/test_txn22.py index 310c6f8a829..42da29540e4 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn22.py +++ b/src/third_party/wiredtiger/test/suite/test_txn22.py @@ -52,6 +52,11 @@ class test_txn22(wttest.WiredTigerTestCase, suite_subprocess): base_config = 'cache_size=1GB' conn_config = base_config + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + # File to be corrupted filename_scenarios = [ ('WiredTiger', dict(filename='WiredTiger')), @@ -75,9 +80,8 @@ class test_txn22(wttest.WiredTigerTestCase, suite_subprocess): "removal:WiredTiger.wt", ] - scenarios = make_scenarios(filename_scenarios) + scenarios = make_scenarios(key_format_values, filename_scenarios) uri = 'table:test_txn22' - create_params = 'key_format=i,value_format=S' nrecords = 1000 # records per table. def valuegen(self, i): @@ -115,10 +119,11 @@ class test_txn22(wttest.WiredTigerTestCase, suite_subprocess): def test_corrupt_meta(self): newdir = "RESTART" newdir2 = "RESTART2" - expect = list(range(0, self.nrecords)) + expect = list(range(1, self.nrecords + 1)) salvage_config = self.base_config + ',salvage=true' - self.session.create(self.uri, self.create_params) + create_params = 'key_format={},value_format=S'.format(self.key_format) + self.session.create(self.uri, create_params) self.inserts(expect) # Simulate a crash by copying the contents of the directory diff --git a/src/third_party/wiredtiger/test/suite/test_txn23.py b/src/third_party/wiredtiger/test/suite/test_txn23.py index 87c2c53a103..c0d420ca92b 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn23.py +++ b/src/third_party/wiredtiger/test/suite/test_txn23.py @@ -32,26 +32,30 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_txn23(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' conn_config = 'cache_size=5MB' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def large_updates(self, uri, value, ds, nrows, commit_ts): # Update a large number of records. cursor = self.session.open_cursor(uri) - for i in range(0, nrows): + for i in range(1, nrows + 1): self.session.begin_transaction() cursor[ds.key(i)] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, check_value, uri, ds, nrows, read_ts): - for i in range(0, nrows): - self.session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + for i in range(1, nrows + 1): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = self.session.open_cursor(uri) self.assertEqual(cursor[ds.key(i)], check_value) cursor.close() @@ -63,18 +67,18 @@ class test_txn23(wttest.WiredTigerTestCase): # Create a table. uri_1 = "table:txn23_1" ds_1 = SimpleDataSet( - self, uri_1, 0, key_format="i", value_format="S") + self, uri_1, 0, key_format=self.key_format, value_format="S") ds_1.populate() # Create another table. uri_2 = "table:txn23_2" ds_2 = SimpleDataSet( - self, uri_2, 0, key_format="i", value_format="S") + self, uri_2, 0, key_format=self.key_format, value_format="S") ds_2.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_txn24.py b/src/third_party/wiredtiger/test/suite/test_txn24.py index d6196d6fe85..11bb8a02f4e 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn24.py +++ b/src/third_party/wiredtiger/test/suite/test_txn24.py @@ -33,11 +33,18 @@ import wiredtiger, wttest import time +from wtscenario import make_scenarios class test_txn24(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def conn_config(self): # We want to either eliminate or keep the application thread role in eviction to minimum. # This will ensure that the dedicated eviction threads are doing the heavy lifting. @@ -49,14 +56,14 @@ class test_txn24(wttest.WiredTigerTestCase): # Create and populate a table. uri = "table:test_txn24" - table_params = 'key_format=i,value_format=S' + table_params = 'key_format={},value_format=S'.format(self.key_format) default_val = 'ABCD' * 60 new_val = 'YYYY' * 60 n_rows = 480000 self.session.create(uri, table_params) cursor = self.session.open_cursor(uri, None) - for i in range(0, n_rows): + for i in range(1, n_rows + 1): cursor[i] = default_val cursor.close() @@ -66,7 +73,7 @@ class test_txn24(wttest.WiredTigerTestCase): # Start a transaction, make an update and keep it running. cursor = self.session.open_cursor(uri, None) self.session.begin_transaction('isolation=snapshot') - cursor[0] = new_val + cursor[1] = new_val # Start few sessions and transactions, make updates and try committing them. session2 = self.setUpSessionOpen(self.conn) @@ -94,7 +101,7 @@ class test_txn24(wttest.WiredTigerTestCase): session4 = self.setUpSessionOpen(self.conn) cursor4 = session4.open_cursor(uri) - start_row = 1 + start_row = 2 for i in range(0, 120000): cursor4[start_row] = new_val start_row += 1 diff --git a/src/third_party/wiredtiger/test/suite/test_txn25.py b/src/third_party/wiredtiger/test/suite/test_txn25.py index c8cebdeb05e..6f1b25d4eec 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn25.py +++ b/src/third_party/wiredtiger/test/suite/test_txn25.py @@ -31,17 +31,31 @@ # import wiredtiger, wttest +from wtscenario import make_scenarios class test_txn25(wttest.WiredTigerTestCase): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' + key_format_values = [ + ('string-row', dict(key_format='S', usestrings=True)), + ('column', dict(key_format='r', usestrings=False)), + ] + scenarios = make_scenarios(key_format_values) + + def getkey(self, i): + if self.usestrings: + return str(i) + else: + return i + def test_txn25(self): uri = 'file:test_txn25' - create_config = 'allocation_size=512,key_format=S,value_format=S' + create_config = 'allocation_size=512,key_format={},value_format=S'.format(self.key_format) self.session.create(uri, create_config) # Populate the file and ensure that we start seeing some high transaction IDs in the system. + nrows = 1000 value1 = 'aaaaa' * 100 value2 = 'bbbbb' * 100 value3 = 'ccccc' * 100 @@ -51,19 +65,19 @@ class test_txn25(wttest.WiredTigerTestCase): session2.begin_transaction() cursor = self.session.open_cursor(uri) - for i in range(1, 1000): + for i in range(1, nrows): self.session.begin_transaction() - cursor[str(i)] = value1 + cursor[self.getkey(i)] = value1 self.session.commit_transaction() - for i in range(1, 1000): + for i in range(1, nrows): self.session.begin_transaction() - cursor[str(i)] = value2 + cursor[self.getkey(i)] = value2 self.session.commit_transaction() - for i in range(1, 1000): + for i in range(1, nrows): self.session.begin_transaction() - cursor[str(i)] = value3 + cursor[self.getkey(i)] = value3 self.session.commit_transaction() session2.rollback_transaction() @@ -81,6 +95,6 @@ class test_txn25(wttest.WiredTigerTestCase): # so we have to wipe the cell's transaction IDs in order to see them. cursor = self.session.open_cursor(uri) self.session.begin_transaction() - for i in range(1, 1000): - self.assertEqual(cursor[str(i)], value3) + for i in range(1, nrows): + self.assertEqual(cursor[self.getkey(i)], value3) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_txn26.py b/src/third_party/wiredtiger/test/suite/test_txn26.py index 4618585b7f1..e5b0a719867 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn26.py +++ b/src/third_party/wiredtiger/test/suite/test_txn26.py @@ -31,37 +31,44 @@ # [END_TAGS] import wiredtiger, wttest +from wtscenario import make_scenarios # test_txn26.py -# Test that commit should fail if commit timestamp is smaller or equal to the active timestamp. Our handling of out of order timestamp relies on this to ensure repeated reads are working as expected. -def timestamp_str(t): - return '%x' % t +# Test that commit should fail if commit timestamp is smaller or equal to the active timestamp. +# Our handling of out of order timestamp relies on this to ensure repeated reads are working as +# expected. class test_txn26(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('string-row', dict(key_format='S', key=str(0))), + ('column', dict(key_format='r', key=16)), + ] + scenarios = make_scenarios(key_format_values) + def test_commit_larger_than_active_timestamp(self): if not wiredtiger.diagnostic_build(): self.skipTest('requires a diagnostic build') uri = 'table:test_txn26' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) value = 'a' # Start a session with timestamp 10 session2 = self.conn.open_session(self.session_config) - session2.begin_transaction('read_timestamp=' + timestamp_str(10)) + session2.begin_transaction('read_timestamp=' + self.timestamp_str(10)) # Try to commit at timestamp 10 self.session.begin_transaction() - cursor[str(0)] = value + cursor[self.key] = value with self.expectedStderrPattern("must be greater than the latest active read timestamp"): try: - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) except wiredtiger.WiredTigerError as e: gotException = True self.pr('got expected exception: ' + str(e)) diff --git a/src/third_party/wiredtiger/test/suite/test_util01.py b/src/third_party/wiredtiger/test/suite/test_util01.py index f01eb310c23..20c6cea62d6 100755 --- a/src/third_party/wiredtiger/test/suite/test_util01.py +++ b/src/third_party/wiredtiger/test/suite/test_util01.py @@ -30,9 +30,6 @@ import string, os, sys, random from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - # test_util01.py # Utilities: wt dump, as well as the dump cursor class test_util01(wttest.WiredTigerTestCase, suite_subprocess): @@ -164,7 +161,7 @@ class test_util01(wttest.WiredTigerTestCase, suite_subprocess): expectout.write(self.dumpstr(key, hexoutput)) expectout.write(self.dumpstr(value, hexoutput)) if commit_timestamp is not None: - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_timestamp)) def dump(self, usingapi, hexoutput, commit_timestamp, read_timestamp): params = self.table_config() diff --git a/src/third_party/wiredtiger/test/suite/test_util21.py b/src/third_party/wiredtiger/test/suite/test_util21.py index 2271ad8b312..6dc3e05e042 100644 --- a/src/third_party/wiredtiger/test/suite/test_util21.py +++ b/src/third_party/wiredtiger/test/suite/test_util21.py @@ -30,9 +30,6 @@ import wttest from suite_subprocess import suite_subprocess from helper import compare_files -def timestamp_str(t): - return '%x' % t - # test_util21.py # Ensure that wt dump can dump obsolete data in the history store. class test_util21(wttest.WiredTigerTestCase, suite_subprocess): @@ -45,7 +42,7 @@ class test_util21(wttest.WiredTigerTestCase, suite_subprocess): for i in range(1, 5): self.session.begin_transaction() cursor[str(i)] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def test_dump_obsolete_data(self): @@ -58,7 +55,7 @@ class test_util21(wttest.WiredTigerTestCase, suite_subprocess): value3 = 'c' * 100 value4 = 'd' * 100 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) self.add_data_with_timestamp(uri, value1, 2) self.add_data_with_timestamp(uri, value2, 3) @@ -68,14 +65,14 @@ class test_util21(wttest.WiredTigerTestCase, suite_subprocess): self.session.checkpoint() # Set stable timestamp, so we don't lose data when closing/opening connection when using wt dump. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Call dump on the values before the oldest timestamp is set self.runWt(['dump', 'file:WiredTigerHS.wt'], outfilename="before_oldest") # Set oldest timestamp, and checkpoint, the obsolete data should not removed as # the pages are clean. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(6)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(6)) self.session.checkpoint() self.runWt(['dump', 'file:WiredTigerHS.wt'], outfilename="after_oldest") diff --git a/src/third_party/wiredtiger/test/suite/wtbackup.py b/src/third_party/wiredtiger/test/suite/wtbackup.py index fe34d563da8..25de77ba7fa 100644 --- a/src/third_party/wiredtiger/test/suite/wtbackup.py +++ b/src/third_party/wiredtiger/test/suite/wtbackup.py @@ -202,16 +202,21 @@ class backup_base(wttest.WiredTigerTestCase, suite_subprocess): # # Perform a block range copy for a given offset and file. # - def range_copy(self, filename, offset, size, backup_incr_dir): + def range_copy(self, filename, offset, size, backup_incr_dir, consolidate): read_from = filename write_to = backup_incr_dir + '/' + filename rfp = open(read_from, "rb") rfp.seek(offset, 0) buf = rfp.read(size) # Perform between previous incremental directory, to check that - # the old file and the new file is different. + # the old file and the new file is different. We can only ensure + # the data are different if running in consolidate mode. It's + # possible that we change multiple blocks in a single write and + # some of the blocks are the same as before. If we are not running + # in consolidate mode, these blocks which are copied separately one + # by one will trigger this assert. old_to = self.home_tmp + '/' + filename - if os.path.exists(old_to): + if os.path.exists(old_to) and consolidate: self.pr('RANGE CHECK file ' + old_to + ' offset ' + str(offset) + ' len ' + str(size)) old_rfp = open(old_to, "rb") old_rfp.seek(offset, 0) @@ -238,7 +243,7 @@ class backup_base(wttest.WiredTigerTestCase, suite_subprocess): # # Note: we return the sizes of WT_BACKUP_RANGE type files for tests that check for consolidate config. # - def take_incr_backup_block(self, bkup_c, newfile, backup_incr_dir): + def take_incr_backup_block(self, bkup_c, newfile, backup_incr_dir, consolidate): config = 'incremental=(file=' + newfile + ')' self.pr('Open incremental cursor with ' + config) # For each file listed, open a duplicate backup cursor and copy the blocks. @@ -262,7 +267,7 @@ class backup_base(wttest.WiredTigerTestCase, suite_subprocess): else: # Copy the block range. self.pr('Range copy file ' + newfile + ' offset ' + str(offset) + ' len ' + str(size)) - self.range_copy(newfile, offset, size, backup_incr_dir) + self.range_copy(newfile, offset, size, backup_incr_dir, consolidate) lens.append(size) incr_c.close() return lens @@ -319,7 +324,7 @@ class backup_base(wttest.WiredTigerTestCase, suite_subprocess): # If that changes then this, and the use of the duplicate below can change. while bkup_c.next() == 0: newfile = bkup_c.get_key() - file_sizes += self.take_incr_backup_block(bkup_c, newfile, backup_incr_dir) + file_sizes += self.take_incr_backup_block(bkup_c, newfile, backup_incr_dir, consolidate) file_names.append(newfile) # Copy into temp directory for tests that require further iterations of incremental backups. self.copy_file(newfile, self.home_tmp) diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 1d22162f535..01c4f315f9c 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -653,6 +653,9 @@ class WiredTigerTestCase(unittest.TestCase): """ self.assertEqual(int(ts1, 16), int(ts2, 16)) + def timestamp_str(self, t): + return '%x' % t + def exceptionToStderr(self, expr): """ Used by assertRaisesHavingMessage to convert an expression |
