diff options
author | Luke Chen <luke.chen@mongodb.com> | 2020-08-10 13:47:18 +1000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-08-10 04:00:15 +0000 |
commit | 78207ca380688c73b1a217f23d5b7c8803bef9cd (patch) | |
tree | 12572958536b96c3a73e3bd545776768c1f93926 /src/third_party/wiredtiger/test | |
parent | e6e4fb531c545ce538e84116c7cb6046de009bb0 (diff) | |
download | mongo-78207ca380688c73b1a217f23d5b7c8803bef9cd.tar.gz |
Import wiredtiger: 9c79fedf125e1d63250d767a8bddffcfa9fb7757 from branch mongodb-4.4
ref: 8092bf0a39..9c79fedf12
for: 4.4.1
WT-5571 Evergreen loses compiler warnings, need way to add to CFLAGS
WT-5717 Reenable history store salvage test
WT-5945 Enable stricter performance for test_wt2853_perf
WT-5970 Update test_wt4333_handle_locks to use 1GB cache size
WT-6215 Clear backup block information on rename
WT-6233 Add statistic to track the number of times we remove keys from the history store due to inserting 0 timestamped entries
WT-6274 SESSIONs shouldn't nest calls to get/release hs cursor
WT-6325 Fast truncate can lead to never resolved prepared operations
WT-6331 Set oldest timestamp on startup of WiredTiger
WT-6421 Avoid parsing metadata checkpoint for clean files
WT-6425 Turn off log archiving to avoid file copy race
WT-6440 Use onpage visibility check helpers in __wt_txn_update_check
WT-6466 Don't reset the WT_SESSION timers when HS cursors are closed
WT-6468 Discard a deleted page that is cleaned by a checkpoint
WT-6473 format configuration with tiny caches can result in empty objects
WT-6474 Configure the global RNG before doing any command-line or configuration file processing
WT-6479 Don't insert updates after the first globally visible update to the history store
WT-6488 Fail update restore eviction if no new updates were written
WT-6491 No need to append the onpage value if it is prepared
WT-6492 Use WT_UPDATE_RESTORED_FROM_DS to determine if the onpage value is on the update chain or not
WT-6493 Retry processing update list if a prepared update is modified concurrently
WT-6499 Ignore evict priority for btrees that are dominating cache usage
WT-6508 Perform checkpoint cleanup on data store as part of checkpoint
WT-6511 cursor join: explicitly advance the iterator when finished with a clause
WT-6514 Fix description of eviction_updates_trigger in the documentation
WT-6517 Update test_txn13 to avoid getting a rollback error
WT-6519 Add testing coverage for mongodb-4.6 branch in compatibility test
WT-6521 Don't perform checkpoint cleanup during server startup and shutdown phase
WT-6527 When a thread cannot be allocated, return cleanly from WT API
WT-6528 Remove offensive terminology in WiredTiger API and source code
WT-6529 Use helper function in __rollback_abort_row_ondisk_kv
WT-6533 No need to do obsolete check before inserting to history store
WT-6539 Fix backup and rename memory leak
WT-6543 Distinguish Evergreen artifacts links across restarts for testing tasks
WT-6551 Avoid a timing problem at the beginning of a test for log preallocation
WT-6552 Fix configure flags in evergreen.yml for failing PPC tests
Diffstat (limited to 'src/third_party/wiredtiger/test')
21 files changed, 839 insertions, 81 deletions
diff --git a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c index e260081b78c..0dd403be91e 100644 --- a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c +++ b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c @@ -59,7 +59,7 @@ static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static bool slow_incremental = false; static bool do_drop = true; -static bool do_rename = false; +static bool do_rename = true; #define VERBOSE(level, fmt, ...) \ do { \ @@ -547,7 +547,7 @@ reopen_file(int *fdp, char *buf, size_t buflen, const char *filename, int oflag) */ static void incr_backup(WT_CONNECTION *conn, const char *home, const char *backup_home, TABLE_INFO *tinfo, - ACTIVE_FILES *master_active) + ACTIVE_FILES *current_active) { ACTIVE_FILES active; WT_CURSOR *cursor, *file_cursor; @@ -560,7 +560,7 @@ incr_backup(WT_CONNECTION *conn, const char *home, const char *backup_home, TABL char *filename; VERBOSE(2, "INCREMENTAL BACKUP: %s\n", backup_home); - active_files_print(master_active, "master list before incremental backup"); + active_files_print(current_active, "current list before incremental backup"); WT_CLEAR(rbuf); WT_CLEAR(wbuf); rfd = wfd = -1; @@ -637,10 +637,10 @@ incr_backup(WT_CONNECTION *conn, const char *home, const char *backup_home, TABL VERBOSE(2, " finished incremental backup: %d files, %d range copy, %d file copy\n", nfiles, nrange, ncopy); active_files_sort(&active); - active_files_remove_missing(master_active, &active, backup_home); + active_files_remove_missing(current_active, &active, backup_home); - /* Move the current active list to the master list */ - active_files_move(master_active, &active); + /* Move the active list to the current list. */ + active_files_move(current_active, &active); } static void diff --git a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c index ff59c66fbb7..9005c474801 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c @@ -44,8 +44,8 @@ static void *thread_insert(void *); static void *thread_get(void *); #define BLOOM false -#define GAP_DISPLAY 5 /* Threshold for seconds of gap to be displayed */ -#define GAP_ERROR 25 /* Threshold for seconds of gap to be treated as error */ +#define GAP_DISPLAY 3 /* Threshold for seconds of gap to be displayed */ +#define GAP_ERROR 7 /* Threshold for seconds of gap to be treated as error */ #define N_RECORDS 10000 #define N_INSERT 1000000 #define N_INSERT_THREAD 1 diff --git a/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c b/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c index 9a263eb9ef6..b91d3c2222e 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c @@ -259,7 +259,7 @@ runone(bool config_cache) testutil_check(__wt_snprintf(buf, sizeof(buf), "create" ", cache_cursors=%s" - ", cache_size=5GB" + ", cache_size=1GB" ", checkpoint_sync=true" ", eviction=(threads_max=5)" ", file_manager=(" @@ -306,10 +306,8 @@ run(int argc, char *argv[]) bool cache_cursors; } runs[] = { {1, 1, false}, {1, 1, true}, {8, 1, false}, {8, 1, true}, {16, 1, false}, {16, 1, true}, - {16, WT_ELEMENTS(uri_list), false}, {16, WT_ELEMENTS(uri_list), true}, {200, 100, false}, - {200, 100, true}, {200, WT_ELEMENTS(uri_list), false}, {200, WT_ELEMENTS(uri_list), true}, - {300, 100, false}, {300, 100, true}, {600, WT_ELEMENTS(uri_list), false}, - {600, WT_ELEMENTS(uri_list), true}, + {16, WT_ELEMENTS(uri_list), false}, {16, WT_ELEMENTS(uri_list), true}, {64, 100, false}, + {64, 100, true}, {64, WT_ELEMENTS(uri_list), false}, {64, WT_ELEMENTS(uri_list), true}, }; WT_RAND_STATE rnd; u_int i, n; diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 9bf45be39dd..68ac9001277 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -55,7 +55,7 @@ functions: set -o verbose if [ "$OS" != "Windows_NT" ]; then sh reconf - ${configure_env_vars|CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC"} \ + ${configure_env_vars|CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH ADD_CFLAGS="-ggdb -fPIC"} \ ../configure ${configure_python_setting|} \ ${posix_configure_flags|--enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL} fi @@ -122,7 +122,7 @@ functions: set -o verbose sh reconf if [ "$OS" != "Windows_NT" ]; then - CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC" \ + CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH ADD_CFLAGS="-ggdb -fPIC" \ ../configure ${configure_python_setting|} \ --enable-strict --enable-diagnostic --with-builtins=lz4,snappy,zlib fi @@ -262,7 +262,7 @@ functions: permissions: public-read content_type: application/tar display_name: Artifacts - remote_file: wiredtiger/${build_variant}/${revision}/artifacts/${task_name}_${build_id}.tgz + remote_file: wiredtiger/${build_variant}/${revision}/artifacts/${task_name}_${build_id}${postfix|}.tgz "cleanup": command: shell.exec params: @@ -371,7 +371,7 @@ variables: configure_env_vars: CC="/opt/mongodbtoolchain/v3/bin/clang -fsanitize=address" PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC -fno-omit-frame-pointer" - posix_configure_flags: --enable-diagnostic --with-builtins=lz4,snappy,zlib + posix_configure_flags: --enable-strict --enable-diagnostic --with-builtins=lz4,snappy,zlib - func: "format test script" vars: format_test_script_args: -R -t 360 @@ -388,12 +388,14 @@ variables: posix_configure_flags: --enable-strict --enable-diagnostic --with-builtins=lz4,snappy,zlib - func: "recovery stress test script" vars: - times: 40 + times: 25 pre: - func: "cleanup" post: - func: "upload artifact" + vars: + postfix: -${execution} - func: "cleanup" tasks: @@ -403,6 +405,8 @@ tasks: commands: - func: "get project" - func: "compile wiredtiger" + - func: "upload artifact" + - func: "cleanup" - name: compile-asan tags: ["pull_request"] @@ -412,6 +416,8 @@ tasks: vars: configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-fsanitize=address -fno-omit-frame-pointer -ggdb" posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static + - func: "upload artifact" + - func: "cleanup" - name: compile-msan commands: @@ -420,6 +426,8 @@ tasks: vars: configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-fsanitize=memory -ggdb" posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static + - func: "upload artifact" + - func: "cleanup" - name: compile-ubsan commands: @@ -428,6 +436,8 @@ tasks: vars: configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-fsanitize=undefined -ggdb" posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic + - func: "upload artifact" + - func: "cleanup" # production build with --disable-shared - name: compile-production-disable-shared @@ -437,6 +447,8 @@ tasks: - func: "compile wiredtiger" vars: posix_configure_flags: --enable-silent-rules --enable-strict --disable-shared + - func: "upload artifact" + - func: "cleanup" # production build with --disable-static - name: compile-production-disable-static @@ -446,6 +458,8 @@ tasks: - func: "compile wiredtiger" vars: posix_configure_flags: --enable-silent-rules --enable-strict --disable-static --enable-lz4 --enable-snappy --enable-zlib --enable-zstd --enable-python + - func: "upload artifact" + - func: "cleanup" - name: compile-linux-no-ftruncate commands: @@ -453,6 +467,8 @@ tasks: - func: "compile wiredtiger no linux ftruncate" vars: posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-strict --enable-python + - func: "upload artifact" + - func: "cleanup" - name: compile-wtperf commands: @@ -460,6 +476,8 @@ tasks: - func: "compile wiredtiger" vars: posix_configure_flags: --enable-strict --enable-diagnostic + - func: "upload artifact" + - func: "cleanup" - name: compile-gcc tags: ["pull_request", "pull_request_compilers"] @@ -467,25 +485,25 @@ tasks: - func: "get project" - func: "compile wiredtiger" vars: - configure_env_vars: CC=gcc-4.8 CXX=g++-4.8 CFLAGS="-ggdb -fPIC" + configure_env_vars: CC=gcc-4.8 CXX=g++-4.8 ADD_CFLAGS="-ggdb -fPIC" - func: "compile wiredtiger" vars: - configure_env_vars: CC=gcc-5 CXX=g++-5 CFLAGS="-ggdb -fPIC" + configure_env_vars: CC=gcc-5 CXX=g++-5 ADD_CFLAGS="-ggdb -fPIC" - func: "compile wiredtiger" vars: - configure_env_vars: CC=gcc-6 CXX=g++-6 CFLAGS="-ggdb -fPIC" + configure_env_vars: CC=gcc-6 CXX=g++-6 ADD_CFLAGS="-ggdb -fPIC" - func: "compile wiredtiger" vars: - configure_env_vars: CC=gcc-6 CXX=g++-6 CFLAGS="-ggdb -fPIC" CPPFLAGS=-fvisibility=hidden + configure_env_vars: CC=gcc-6 CXX=g++-6 ADD_CFLAGS="-ggdb -fPIC" CPPFLAGS=-fvisibility=hidden - func: "compile wiredtiger" vars: - configure_env_vars: CC=gcc-7 CXX=g++-7 CFLAGS="-ggdb -fPIC" + configure_env_vars: CC=gcc-7 CXX=g++-7 ADD_CFLAGS="-ggdb -fPIC" - func: "compile wiredtiger" vars: - configure_env_vars: CC=gcc-8 CXX=g++-8 CFLAGS="-ggdb -fPIC" + configure_env_vars: CC=gcc-8 CXX=g++-8 ADD_CFLAGS="-ggdb -fPIC" - func: "compile wiredtiger" vars: - configure_env_vars: CC=gcc-9 CXX=g++-9 CFLAGS="-ggdb -fPIC" + configure_env_vars: CC=gcc-9 CXX=g++-9 ADD_CFLAGS="-ggdb -fPIC" - name: compile-clang tags: ["pull_request", "pull_request_compilers"] @@ -493,22 +511,22 @@ tasks: - func: "get project" - func: "compile wiredtiger" vars: - configure_env_vars: CC=clang-3.9 CXX=clang++-3.9 CFLAGS="-ggdb -fPIC" + configure_env_vars: CC=clang-3.9 CXX=clang++-3.9 ADD_CFLAGS="-ggdb -fPIC" - func: "compile wiredtiger" vars: - configure_env_vars: CC=clang-4.0 CXX=clang++-4.0 CFLAGS="-ggdb -fPIC" + configure_env_vars: CC=clang-4.0 CXX=clang++-4.0 ADD_CFLAGS="-ggdb -fPIC" - func: "compile wiredtiger" vars: - configure_env_vars: CC=clang-5.0 CXX=clang++-5.0 CFLAGS="-ggdb -fPIC" + configure_env_vars: CC=clang-5.0 CXX=clang++-5.0 ADD_CFLAGS="-ggdb -fPIC" - func: "compile wiredtiger" vars: - configure_env_vars: CC=clang-6.0 CXX=clang++-6.0 CFLAGS="-ggdb -fPIC" + configure_env_vars: CC=clang-6.0 CXX=clang++-6.0 ADD_CFLAGS="-ggdb -fPIC" - func: "compile wiredtiger" vars: - configure_env_vars: CC=clang-7 CXX=clang++-7 CFLAGS="-ggdb -fPIC" + configure_env_vars: CC=clang-7 CXX=clang++-7 ADD_CFLAGS="-ggdb -fPIC" - func: "compile wiredtiger" vars: - configure_env_vars: CC=clang-8 CXX=clang++-8 CFLAGS="-ggdb -fPIC" + configure_env_vars: CC=clang-8 CXX=clang++-8 ADD_CFLAGS="-ggdb -fPIC" - name: make-check-test depends_on: @@ -770,6 +788,8 @@ tasks: - name: compile-ubsan commands: - func: "fetch artifacts" + vars: + dependent_task: compile-ubsan - func: "compile wiredtiger" vars: configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-fsanitize=undefined -ggdb" @@ -1782,8 +1802,8 @@ tasks: # Handle special build combination for running all the diagnostic tests. - func: "configure wiredtiger" vars: - configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-g -Werror" - posix_configure_flags: --enable-silent-rules --enable-diagnostic --disable-static + configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH ADD_CFLAGS="-g -Werror" + posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static - func: "make wiredtiger" vars: make_command: make all @@ -1874,7 +1894,7 @@ tasks: - func: "get project" - func: "compile wiredtiger" vars: - configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="--coverage -fPIC -ggdb" LDFLAGS=--coverage + configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH ADD_CFLAGS="--coverage -fPIC -ggdb" LDFLAGS=--coverage posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-strict --enable-python --with-builtins=lz4,snappy,zlib - func: "make check all" - func: "unit test" @@ -1930,7 +1950,7 @@ tasks: permissions: public-read content_type: text/html display_name: Coverage report - remote_file: wiredtiger/${build_variant}/${revision}/coverage_report/coverage_report_${build_id}.html + remote_file: wiredtiger/${build_variant}/${revision}/coverage_report/coverage_report_${build_id}-${execution}.html - name: spinlock-gcc-test commands: @@ -1982,7 +2002,7 @@ tasks: - func: "get project" - func: "compile wiredtiger" vars: - posix_configure_flags: ac_cv_func_ftruncate=no + posix_configure_flags: --enable-strict ac_cv_func_ftruncate=no - command: shell.exec params: working_dir: "wiredtiger/build_posix" @@ -1998,8 +2018,8 @@ tasks: - func: "get project" - func: "configure wiredtiger" vars: - configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-g -Werror" - posix_configure_flags: --enable-silent-rules --enable-diagnostic --disable-static + configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH ADD_CFLAGS="-g -Werror" + posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static - func: "make wiredtiger" # Run the long version of make check, that includes the full csuite tests @@ -2121,7 +2141,7 @@ tasks: - func: "get project" - func: "compile wiredtiger" vars: - posix_configure_flags: --disable-shared --with-builtins=snappy,zlib + posix_configure_flags: --enable-strict --disable-shared --with-builtins=snappy,zlib - command: shell.exec params: working_dir: "wiredtiger/build_posix" @@ -2133,7 +2153,7 @@ tasks: # Delete wt util rm -f wt # Build static wt util - make CC='eval "g++ -static"' ${smp_command|} + ${make_command|make} CC='eval "g++ -static"' ${smp_command|} # -V option displays Wiredtiger library version ./wt -V @@ -2185,7 +2205,7 @@ tasks: vars: configure_env_vars: CXX=/opt/mongodbtoolchain/v3/bin/g++ - PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC" + PATH=/opt/mongodbtoolchain/v3/bin:$PATH ADD_CFLAGS="-ggdb -fPIC" - command: shell.exec params: working_dir: "wiredtiger/bench/workgen/runner" @@ -2313,6 +2333,9 @@ tasks: - <<: *recovery-stress-test name: recovery-stress-test-2 tags: ["stress-test-2", "stress-test-zseries-2"] + - <<: *recovery-stress-test + name: recovery-stress-test-3 + tags: ["stress-test-3", "stress-test-zseries-3"] buildvariants: @@ -2488,7 +2511,7 @@ buildvariants: run_on: - macos-1012 expansions: - configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC" + configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH ADD_CFLAGS="-ggdb -fPIC" posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL smp_command: -j $(sysctl -n hw.logicalcpu) make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH ARCHFLAGS=-Wno-error=unused-command-line-argument-hard-error-in-future make @@ -2568,3 +2591,4 @@ buildvariants: - name: unit-test - name: ".stress-test-zseries-1" - name: ".stress-test-zseries-2" + - name: ".stress-test-zseries-3" diff --git a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh index 5a396aad3dc..551bbd78764 100755 --- a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh +++ b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh @@ -14,6 +14,7 @@ bflag() { # Return if the branch's format command takes the -B flag for backward compatibility. test "$1" = "develop" && echo "-B " + test "$1" = "mongodb-4.6" && echo "-B " test "$1" = "mongodb-4.4" && echo "-B " return 0 } @@ -186,10 +187,11 @@ cd "$top" if [ "$long" = true ]; then (build_branch mongodb-3.4) (build_branch mongodb-3.6) + (build_branch mongodb-4.0) fi -(build_branch mongodb-4.0) (build_branch mongodb-4.2) (build_branch mongodb-4.4) +(build_branch mongodb-4.6) (build_branch develop) # Get the names of the last two WiredTiger releases, wt1 is the most recent release, wt2 is the @@ -206,10 +208,11 @@ fi if [ "$long" = true ]; then (run_format mongodb-3.4 "fix row var") (run_format mongodb-3.6 "fix row var") + (run_format mongodb-4.0 "fix row var") fi -(run_format mongodb-4.0 "fix row var") (run_format mongodb-4.2 "fix row var") (run_format mongodb-4.4 "row") +(run_format mongodb-4.6 "row") (run_format develop "row") if [ "$long" = true ]; then (run_format "$wt1" "fix row var") @@ -220,11 +223,11 @@ fi if [ "$long" = true ]; then (verify_branches mongodb-3.6 mongodb-3.4 "fix row var") (verify_branches mongodb-4.0 mongodb-3.6 "fix row var") + (verify_branches mongodb-4.2 mongodb-4.0 "fix row var") fi -(verify_branches mongodb-4.2 mongodb-4.0 "fix row var") (verify_branches mongodb-4.4 mongodb-4.2 "fix row var") -(verify_branches develop mongodb-4.4 "row") -(verify_branches develop mongodb-4.2 "row") +(verify_branches mongodb-4.6 mongodb-4.4 "row") +(verify_branches develop mongodb-4.6 "row") if [ "$long" = true ]; then (verify_branches "$wt1" "$wt2" "row") (verify_branches develop "$wt1" "row") @@ -232,12 +235,12 @@ fi # Verify forward compatibility for supported access methods. (verify_branches mongodb-4.2 mongodb-4.4 "row") -(verify_branches mongodb-4.2 develop "row") -(verify_branches mongodb-4.4 develop "row") +(verify_branches mongodb-4.4 mongodb-4.6 "row") +(verify_branches mongodb-4.6 develop "row") # Upgrade/downgrade testing for supported access methods. (upgrade_downgrade mongodb-4.2 mongodb-4.4 "row") -(upgrade_downgrade mongodb-4.2 develop "row") -(upgrade_downgrade mongodb-4.4 develop "row") +(upgrade_downgrade mongodb-4.4 mongodb-4.6"row") +(upgrade_downgrade mongodb-4.6 develop "row") exit 0 diff --git a/src/third_party/wiredtiger/test/format/bulk.c b/src/third_party/wiredtiger/test/format/bulk.c index 72c17366639..bfbddaffaa5 100644 --- a/src/third_party/wiredtiger/test/format/bulk.c +++ b/src/third_party/wiredtiger/test/format/bulk.c @@ -171,8 +171,12 @@ wts_load(void) break; } - /* Restart the enclosing transaction every 5K operations so we don't overflow the cache. */ - if (keyno % 5000 == 0) { + /* + * When first starting up, restart the enclosing transaction every 10 operations so we never + * end up with an empty object. After 5K records, restart the transaction every 5K records + * so we don't overflow the cache. + */ + if ((keyno < 5000 && keyno % 10 == 0) || keyno % 5000 == 0) { /* Report on progress. */ track("bulk load", keyno, NULL); @@ -190,6 +194,8 @@ wts_load(void) * aren't surprised). */ if (keyno != g.c_rows + 1) { + testutil_assert(committed_keyno > 0); + g.rows = committed_keyno; g.c_rows = (uint32_t)committed_keyno; config_print(false); diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 5b4e8589b8c..4f7db16a3c0 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -402,6 +402,7 @@ void operations(u_int, bool); void path_setup(const char *); void set_alarm(u_int); void set_core_off(void); +void set_oldest_timestamp(void); void snap_init(TINFO *); void snap_teardown(TINFO *); void snap_op_init(TINFO *, uint64_t, bool); diff --git a/src/third_party/wiredtiger/test/format/t.c b/src/third_party/wiredtiger/test/format/t.c index 11d3d48db32..fadbb1c79c9 100644 --- a/src/third_party/wiredtiger/test/format/t.c +++ b/src/third_party/wiredtiger/test/format/t.c @@ -165,6 +165,8 @@ main(int argc, char *argv[]) format_process_env(); + __wt_random_init_seed(NULL, &g.rnd); /* Initialize the RNG. */ + /* Set values from the command line. */ home = NULL; one_flag = quiet_flag = false; @@ -256,8 +258,6 @@ main(int argc, char *argv[]) */ ops_seconds = g.c_timer == 0 ? 0 : ((g.c_timer * 60) - 15) / FORMAT_OPERATION_REPS; - __wt_random_init_seed(NULL, &g.rnd); /* Initialize the RNG. */ - testutil_check(__wt_thread_str(g.tidbuf, sizeof(g.tidbuf))); while (++g.run_cnt <= g.c_runs || g.c_runs == 0) { @@ -269,6 +269,7 @@ main(int argc, char *argv[]) if (g.reopen) { config_final(); wts_open(g.home, &g.wts_conn, &g.wts_session, true); + set_oldest_timestamp(); } else { wts_create(g.home); config_final(); diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index d2ae281e281..069340df59b 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -303,7 +303,8 @@ timestamp_parse(WT_SESSION *session, const char *str, uint64_t *tsp) char *p; *tsp = strtoull(str, &p, 16); - WT_ASSERT((WT_SESSION_IMPL *)session, p - str <= 16); + if (session != NULL) + WT_ASSERT((WT_SESSION_IMPL *)session, p - str <= 16); } /* @@ -356,6 +357,39 @@ timestamp(void *arg) } /* + * set_oldest_timestamp -- + * Query the oldest timestamp from wiredtiger and set it as our global oldest timestamp. This + * should only be called on runs for pre existing databases. + */ +void +set_oldest_timestamp(void) +{ + static const char *oldest_timestamp_str = "oldest_timestamp="; + + WT_CONNECTION *conn; + WT_DECL_RET; + uint64_t oldest_ts; + char buf[WT_TS_HEX_STRING_SIZE * 2 + 64], tsbuf[WT_TS_HEX_STRING_SIZE]; + + conn = g.wts_conn; + + if ((ret = conn->query_timestamp(conn, tsbuf, "get=oldest")) == 0) { + timestamp_parse(NULL, tsbuf, &oldest_ts); + g.timestamp = oldest_ts; + testutil_check( + __wt_snprintf(buf, sizeof(buf), "%s%" PRIx64, oldest_timestamp_str, g.oldest_timestamp)); + } else if (ret != WT_NOTFOUND) + /* + * Its possible there may not be an oldest timestamp as such we could get not found. This + * should be okay assuming timestamps are not configured if they are, it's still okay as we + * could have configured timestamps after not running with timestamps. As such only error if + * we get a non not found error. If we were supposed to fail with not found we'll see an + * error later on anyway. + */ + testutil_die(ret, "unable to query oldest timestamp"); +} + +/* * lock_init -- * Initialize abstract lock that can use either pthread of wt reader-writer locks. */ diff --git a/src/third_party/wiredtiger/test/suite/test_bug019.py b/src/third_party/wiredtiger/test/suite/test_bug019.py index abf68827959..a5af1f33d0d 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_bug019.py +++ b/src/third_party/wiredtiger/test/suite/test_bug019.py @@ -37,6 +37,7 @@ class test_bug019(wttest.WiredTigerTestCase): conn_config = 'log=(enabled,file_max=100K),statistics=(fast)' uri = "table:bug019" entries = 5000 + max_initial_entries = 50000 max_prealloc = 1 # Modify rows so we write log records. We're writing a lot more than a @@ -51,15 +52,20 @@ class test_bug019(wttest.WiredTigerTestCase): def populate(self, nentries): c = self.session.open_cursor(self.uri, None, None) for i in range(0, nentries): - # Make the values about 200 bytes. That's about 1MB of data for - # 5000 records, generating 10 log files used plus more for overhead. - c[i] = "abcde" * 40 - if i % 500 == 0: + # Make the values about 2000 bytes. When called with 5000 records + # that's about 10MB of data, generating 100 log files used plus more for overhead. + # Typically the huge traffic causes the preallocation statistic to + # increase. We'll quit when it does, as that's our goal here. + # For the initial populate, we'll insert up to 10x as many records, + # so up to 1000 log files. + c[i] = "abcde" * 400 + if i % 50 == 0: prealloc = self.get_prealloc_stat() if prealloc > self.max_prealloc: - self.pr("Updating max_prealloc from " + str(self.max_prealloc)) - self.pr(" to new prealloc " + str(prealloc)) + self.pr("Updating max_prealloc from {} to {} after {} inserts". + format(self.max_prealloc, prealloc, i)) self.max_prealloc = prealloc + break c.close() # Wait for a log file to be pre-allocated. Avoid timing problems, but @@ -75,10 +81,14 @@ class test_bug019(wttest.WiredTigerTestCase): # There was a bug where pre-allocated log files accumulated on # Windows systems due to an issue with the directory list code. def test_bug019(self): - # Create a table just to write something into the log. - self.session.create(self.uri, 'key_format=i,value_format=S') start_prealloc = self.get_prealloc_stat() - self.populate(self.entries) + self.max_prealloc = start_prealloc + + # Populate a new table to generate log traffic. This typically + # increase the max number of log files preallocated, as indicated by + # the statistic. + self.session.create(self.uri, 'key_format=i,value_format=S') + self.populate(self.max_initial_entries) self.session.checkpoint() if self.max_prealloc <= start_prealloc: self.pr("FAILURE: max_prealloc " + str(self.max_prealloc)) diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint05.py b/src/third_party/wiredtiger/test/suite/test_checkpoint05.py index 58af3003a60..c28d634d54f 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint05.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint05.py @@ -44,7 +44,7 @@ class test_checkpoint05(wttest.WiredTigerTestCase): while metadata_cursor.next() == 0: key = metadata_cursor.get_key() value = metadata_cursor[key] - nckpt = nckpt + value.count("WiredTigerCheckpoint") + nckpt += value.count("WiredTigerCheckpoint") metadata_cursor.close() return nckpt diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint07.py b/src/third_party/wiredtiger/test/suite/test_checkpoint07.py new file mode 100755 index 00000000000..52b3104f69b --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint07.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_checkpoint07.py +# Test that the checkpoints timing statistics are populated as expected. + +import wiredtiger, wttest +from wiredtiger import stat +from wtdataset import SimpleDataSet + +def timestamp_str(t): + return '%x' % t + +class test_checkpoint07(wttest.WiredTigerTestCase): + conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + session_config = 'isolation=snapshot' + + def get_stat(self, uri): + stat_uri = 'statistics:' + uri + stat_cursor = self.session.open_cursor(stat_uri) + val = stat_cursor[stat.dsrc.btree_clean_checkpoint_timer][2] + stat_cursor.close() + return val + + def test_checkpoint07(self): + self.uri1 = 'table:ckpt05.1' + self.file1 = 'file:ckpt05.1.wt' + self.uri2 = 'table:ckpt05.2' + self.file2 = 'file:ckpt05.2.wt' + self.uri3 = 'table:ckpt05.3' + self.file3 = 'file:ckpt05.3.wt' + self.session.create(self.uri1, 'key_format=i,value_format=i') + self.session.create(self.uri2, 'key_format=i,value_format=i') + self.session.create(self.uri3, 'key_format=i,value_format=i') + + # Setup: Insert some data and checkpoint it. Then modify only + # the data in the first table and checkpoint. Verify the clean skip + # timer is not set for the modified table and is set for the clean one. + c1 = self.session.open_cursor(self.uri1, None) + c2 = self.session.open_cursor(self.uri2, None) + c3 = self.session.open_cursor(self.uri3, None) + c1[1] = 1 + c2[1] = 1 + c3[1] = 1 + self.session.checkpoint(None) + c1[2] = 2 + self.session.checkpoint(None) + val1 = self.get_stat(self.file1) + self.assertEqual(val1, 0) + val2 = self.get_stat(self.file2) + self.assertNotEqual(val2, 0) + val3 = self.get_stat(self.file3) + self.assertNotEqual(val3, 0) + # It is possible that we could span the second timer when processing table + # two and table three during the checkpoint. If they're different check + # they are within 1 second of each other. + if val2 != val3: + self.assertTrue(val2 == val3 - 1 or val3 == val2 - 1) + + # Now force a checkpoint on clean tables. No clean timer should be set. + self.session.checkpoint('force=true') + val = self.get_stat(self.uri1) + self.assertEqual(val, 0) + val = self.get_stat(self.uri2) + self.assertEqual(val, 0) + val = self.get_stat(self.uri3) + self.assertEqual(val, 0) + + # Modify the first two tables and reverify all three. + c1[3] = 3 + c2[3] = 3 + self.session.checkpoint(None) + val = self.get_stat(self.uri1) + self.assertEqual(val, 0) + val = self.get_stat(self.uri2) + self.assertEqual(val, 0) + val = self.get_stat(self.uri3) + self.assertNotEqual(val, 0) + + # Open a backup cursor. This will pin the most recent checkpoint. + # Modify table 1 and checkpoint, then modify table 2 and checkpoint. + # The open backup cursor will cause table 1 to get the smaller timer. + backup_cursor = self.session.open_cursor('backup:', None, None) + c1[4] = 4 + self.session.checkpoint(None) + val = self.get_stat(self.uri1) + self.assertEqual(val, 0) + + c2[4] = 4 + self.session.checkpoint(None) + val2 = self.get_stat(self.uri2) + self.assertEqual(val2, 0) + + val1 = self.get_stat(self.uri1) + val3 = self.get_stat(self.uri3) + # Assert table 1 does not have the forever timer value, but it is set. + # This assumes table 3 has the forever value. + self.assertNotEqual(val1, 0) + self.assertNotEqual(val3, 0) + self.assertLess(val1, val3) + # Save the old forever value from table 3. + oldval3 = val3 + + # Force a checkpoint while the backup cursor is open. Then write again + # to table 2. Since table 1 and table 3 are clean again, this should + # force both table 1 and table 3 to have the smaller timer. + self.session.checkpoint('force=true') + val1 = self.get_stat(self.uri1) + val3 = self.get_stat(self.uri3) + self.assertEqual(val1, 0) + self.assertEqual(val3, 0) + c2[5] = 5 + self.session.checkpoint(None) + val2 = self.get_stat(self.uri2) + self.assertEqual(val2, 0) + + val1 = self.get_stat(self.uri1) + val3 = self.get_stat(self.uri3) + self.assertNotEqual(val1, 0) + self.assertNotEqual(val3, 0) + self.assertLess(val3, oldval3) + # It is possible that we could span the second timer when processing table + # two and table three during the checkpoint. If they're different check + # they are within 1 second of each other. + if val1 != val3: + self.assertTrue(val1 == val3 - 1 or val3 == val1 - 1) + + backup_cursor.close() + + # Repeat the sequence of forcing a checkpoint and then modifying after + # closing the backup cursor to check that both tables are now marked + # with the forever value. + self.session.checkpoint('force=true') + val1 = self.get_stat(self.uri1) + val3 = self.get_stat(self.uri3) + self.assertEqual(val1, 0) + self.assertEqual(val3, 0) + c2[6] = 6 + self.session.checkpoint(None) + val2 = self.get_stat(self.uri2) + self.assertEqual(val2, 0) + + val1 = self.get_stat(self.uri1) + val3 = self.get_stat(self.uri3) + self.assertNotEqual(val1, 0) + self.assertNotEqual(val3, 0) + # It is possible that we could span the second timer when processing table + # two and table three during the checkpoint. If they're different check + # they are within 1 second of each other. + if val1 != val3: + self.assertTrue(val1 == val3 - 1 or val3 == val1 - 1) + self.assertEqual(val3, oldval3) + + self.session.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor13.py b/src/third_party/wiredtiger/test/suite/test_cursor13.py index c4bb50c5bad..c90db17e299 100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor13.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor13.py @@ -65,16 +65,18 @@ class test_cursor13_base(wttest.WiredTigerTestCase): if expect_change: self.assertGreater(stats[0], self.stat_cursor_cache) self.stat_cursor_cache = stats[0] - else: - self.assertEqual(stats[0], self.stat_cursor_cache) + # Stats may change due to background operations in history store cursor + #else: + # self.assertEqual(stats[0], self.stat_cursor_cache) def assert_cursor_reopened(self, expect_change): stats = self.caching_stats() if expect_change: self.assertGreater(stats[1], self.stat_cursor_reopen) self.stat_cursor_reopen = stats[1] - else: - self.assertEqual(stats[1], self.stat_cursor_reopen) + # Stats may change due to background operations in history store cursor + #else: + # self.assertEqual(stats[1], self.stat_cursor_reopen) def cursor_stats_init(self): stats = self.caching_stats() @@ -447,8 +449,11 @@ class test_cursor13_big(test_cursor13_big_base): #self.tty('opens = ' + str(self.opencount) + \ # ', closes = ' + str(self.closecount)) #self.tty('stats after = ' + str(end_stats)) - self.assertEquals(end_stats[0] - begin_stats[0], self.closecount) - self.assertEquals(end_stats[1] - begin_stats[1], self.opencount) + + # Stats won't be exact because they may include operations triggered by other + # threads (e.g., eviction) opening and closing history store cursors. + self.assertGreaterEqual(end_stats[0] - begin_stats[0], self.closecount) + self.assertGreaterEqual(end_stats[1] - begin_stats[1], self.opencount) class test_cursor13_sweep(test_cursor13_big_base): # Set dhandle sweep configuration so that dhandles should be closed within @@ -507,7 +512,7 @@ class test_cursor13_sweep(test_cursor13_big_base): # ', closes = ' + str(self.closecount)) #self.tty('stats after = ' + str(end_stats)) #self.tty('sweep stats after = ' + str(end_sweep_stats)) - self.assertEquals(end_stats[0] - begin_stats[0], self.closecount) + self.assertGreaterEqual(end_stats[0] - begin_stats[0], self.closecount) swept = end_sweep_stats[3] - begin_sweep_stats[3] # Although this is subject to tuning parameters, we know that diff --git a/src/third_party/wiredtiger/test/suite/test_cursor16.py b/src/third_party/wiredtiger/test/suite/test_cursor16.py index 6ce58f01ae8..b5fbd7996eb 100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor16.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor16.py @@ -64,7 +64,8 @@ class test_cursor16(wttest.WiredTigerTestCase): for j in range(0, 10): cursor[str(j)] = str(j) - self.assertEqual(0, self.cached_stats()) + # Skewed by internal history store operations + #self.assertEqual(0, self.cached_stats()) sessions = [] for i in range(0, self.session_count): @@ -89,7 +90,8 @@ class test_cursor16(wttest.WiredTigerTestCase): session.close() #self.tty('end cursors cached=' + str(self.cached_stats())) - self.assertEqual(0, self.cached_stats()) + # Skewed by internal history store operations + #self.assertEqual(0, self.cached_stats()) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_hs15.py b/src/third_party/wiredtiger/test/suite/test_hs15.py new file mode 100644 index 00000000000..8124f1d5f1e --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hs15.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import time, wiredtiger, wttest + +def timestamp_str(t): + return '%x' % t + +# test_hs15.py +# Ensure eviction doesn't clear the history store again after checkpoint has done so because of the same update without timestamp. +class test_hs14(wttest.WiredTigerTestCase): + conn_config = 'cache_size=5MB' + session_config = 'isolation=snapshot' + + def test_hs15(self): + uri = 'table:test_hs15' + self.session.create(uri, 'key_format=S,value_format=S') + cursor = self.session.open_cursor(uri) + + value1 = 'a' * 500 + value2 = 'b' * 500 + value3 = 'c' * 500 + + # Insert an update without timestamp + self.session.begin_transaction() + cursor[str(0)] = value1 + self.session.commit_transaction() + + # Insert a bunch of other contents to trigger eviction + for i in range(1, 1000): + self.session.begin_transaction() + cursor[str(i)] = value2 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + + # Do a modify and an update with timestamps + self.session.begin_transaction() + cursor.set_key(str(0)) + mods = [wiredtiger.Modify('B', 100, 1)] + self.assertEqual(cursor.modify(mods), 0) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + + self.session.begin_transaction() + cursor[str(0)] = value2 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + + # Make the modify with timestamp and the update without timestamp obsolete + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + + # Do a checkpoint + self.session.checkpoint() + + self.session.begin_transaction() + cursor[str(0)] = value3 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + + # Insert a bunch of other contents to trigger eviction + for i in range(1, 1000): + self.session.begin_transaction() + cursor[str(i)] = value3 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + + expected = list(value1) + expected[100] = 'B' + expected = str().join(expected) + self.session.begin_transaction('read_timestamp=' + timestamp_str(1)) + self.assertEqual(cursor[str(0)], expected) + self.session.rollback_transaction() + + self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) + self.assertEqual(cursor[str(0)], value2) + self.session.rollback_transaction() + + self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.assertEqual(cursor[str(0)], value3) + self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_join10.py b/src/third_party/wiredtiger/test/suite/test_join10.py new file mode 100755 index 00000000000..852ef29437d --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_join10.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_join10.py +# Test modeled on the C example for join. +class test_join10(wttest.WiredTigerTestCase): + + # We need statistics for these tests. + conn_config = 'statistics=(all)' + + def check_stats(self, jc, expect): + statcursor = self.session.open_cursor('statistics:join', jc, None) + for id, desc, valstr, val in statcursor: + if desc in expect: + expect_val = expect.pop(desc) + self.assertEqual(val, expect_val) + self.assertTrue(len(expect) == 0, + 'missing expected values in stats: ' + str(expect)) + statcursor.close() + + def test_country(self): + session = self.session + pop_data = [["AU", 1900, 4000000], + ["AU", 1950, 8267337], + ["AU", 2000, 19053186], + ["CAN", 1900, 5500000], + ["CAN", 1950, 14011422], + ["CAN", 2000, 31099561], + ["UK", 1900, 369000000], + ["UK", 1950, 50127000], + ["UK", 2000, 59522468], + ["USA", 1900, 76212168], + ["USA", 1950, 150697361], + ["USA", 2000, 301279593]] + + session.create("table:poptable", \ + "key_format=r,value_format=SHQ," + + "columns=(id,country,year,population),colgroups=(main,population)") + + session.create("colgroup:poptable:main", "columns=(country,year,population)") + session.create("colgroup:poptable:population", "columns=(population)") + + session.create("index:poptable:country", "columns=(country)") + session.create("index:poptable:immutable_year", "columns=(year),immutable") + + cursor = session.open_cursor("table:poptable", None, "append") + for p in pop_data: + cursor.set_value(p[0], p[1], p[2]) + cursor.insert() + cursor.close() + + join_cursor = session.open_cursor("join:table:poptable") + country_cursor = session.open_cursor("index:poptable:country") + year_cursor = session.open_cursor("index:poptable:immutable_year") + + # select values WHERE country == "AU" AND year > 1900 + country_cursor.set_key("AU") + self.assertEqual(country_cursor.search(), 0) + session.join(join_cursor, country_cursor, "compare=eq,count=10") + year_cursor.set_key(1900) + self.assertEqual(year_cursor.search(), 0) + session.join(join_cursor, year_cursor, "compare=gt,count=10,strategy=bloom") + + # Check results + expect = [[c,y,p] for c,y,p in pop_data if c == "AU" and y > 1900] + got = [] + for recno, country, year, population in join_cursor: + got.append([country, year, population]) + self.assertEqual(expect, got) + + # Check statistics + # It may seem odd to encode specific values to check against, but each of these + # statistics represent significant and predictable events in the code, and we + # want to know if anything changes. + expect_stat = dict() + pfxc = 'join: index:poptable:country: ' + pfxy = 'join: index:poptable:immutable_year: ' + + expect_stat[pfxc + 'accesses to the main table'] = 2 + expect_stat[pfxc + 'bloom filter false positives'] = 0 + expect_stat[pfxc + 'checks that conditions of membership are satisfied'] = 4 + expect_stat[pfxc + 'items inserted into a bloom filter'] = 0 + expect_stat[pfxc + 'items iterated'] = 4 + + # We're using a bloom filter on this one, but we don't check for bloom filter + # false positives, it's a bit tied to implementation details. + expect_stat[pfxy + 'accesses to the main table'] = 2 + expect_stat[pfxy + 'checks that conditions of membership are satisfied'] = 3 + expect_stat[pfxy + 'items inserted into a bloom filter'] = 8 + expect_stat[pfxy + 'items iterated'] = 12 + + self.check_stats(join_cursor, expect_stat) + + join_cursor.close() + year_cursor.close() + country_cursor.close() + + # Complex join cursors + join_cursor = session.open_cursor("join:table:poptable") + subjoin_cursor = session.open_cursor("join:table:poptable") + + country_cursor = session.open_cursor("index:poptable:country") + country_cursor2 = session.open_cursor("index:poptable:country") + year_cursor = session.open_cursor("index:poptable:immutable_year") + + # select values WHERE (country == "AU" OR country == "UK") + # AND year > 1900 + + # First, set up the join representing the country clause. + country_cursor.set_key("AU") + self.assertEqual(country_cursor.search(), 0) + session.join(subjoin_cursor, country_cursor, "operation=or,compare=eq,count=10") + + country_cursor2.set_key("UK") + self.assertEqual(country_cursor2.search(), 0) + session.join(subjoin_cursor, country_cursor2, "operation=or,compare=eq,count=10") + + # Join that to the top join, and add the year clause + session.join(join_cursor, subjoin_cursor) + year_cursor.set_key(1900) + self.assertEqual(year_cursor.search(), 0) + + session.join(join_cursor, year_cursor, "compare=gt,count=10,strategy=bloom") + + # Check results + expect = [[c,y,p] for c,y,p in pop_data if (c == "AU" or c == "UK") and y > 1900] + got = [] + for recno, country, year, population in join_cursor: + got.append([country, year, population]) + self.assertEqual(expect, got) + + expect_stat = dict() + + # Note: the stats collected for the clause (country == "AU" OR country == "UK") + # are in a join entry that is a "subjoin". Due to a quirk in the implementation + # of join statistics, subjoin statistics are returned using the main table prefix. + pfxm = 'join: table:poptable: ' + pfxy = 'join: index:poptable:immutable_year: ' + + expect_stat[pfxm + 'accesses to the main table'] = 4 + expect_stat[pfxm + 'bloom filter false positives'] = 0 + expect_stat[pfxm + 'checks that conditions of membership are satisfied'] = 12 + expect_stat[pfxm + 'items inserted into a bloom filter'] = 0 + expect_stat[pfxm + 'items iterated'] = 0 + + expect_stat[pfxy + 'accesses to the main table'] = 6 + expect_stat[pfxy + 'bloom filter false positives'] = 0 + expect_stat[pfxy + 'checks that conditions of membership are satisfied'] = 6 + expect_stat[pfxy + 'items inserted into a bloom filter'] = 0 + expect_stat[pfxy + 'items iterated'] = 0 + + self.check_stats(join_cursor, expect_stat) + + join_cursor.close() + subjoin_cursor.close() + country_cursor.close() + country_cursor2.close() + year_cursor.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py b/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py new file mode 100644 index 00000000000..267aafd79cd --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_prepare_conflict.py +# Evict a page while in the fast-truncate state. +# + +import wiredtiger, wttest +from wtdataset import simple_key, simple_value + +def timestamp_str(t): + return '%x' % t + +class test_prepare_conflict(wttest.WiredTigerTestCase): + def test_prepare(self): + # Create a large table with lots of pages. + uri = "table:test_prepare_conflict" + config = 'allocation_size=512,leaf_page_max=512,key_format=S,value_format=S' + self.session.create(uri, config) + cursor = self.session.open_cursor(uri) + for i in range(1, 80000): + cursor[simple_key(cursor, i)] = simple_value(cursor, i) + cursor.close() + + # Force to disk. + self.reopen_conn() + + # Start a transaction. + self.session.begin_transaction('isolation=snapshot') + + # Truncate the middle chunk. + c1 = self.session.open_cursor(uri, None) + c1.set_key(simple_key(c1, 10000)) + c2 = self.session.open_cursor(uri, None) + c2.set_key(simple_key(c1, 70000)) + self.session.truncate(None, c1, c2, None) + c1.close() + c2.close() + + # Modify a record on a fast-truncate page. + cursor = self.session.open_cursor(uri) + cursor[simple_key(cursor, 40000)] = "replacement_value" + cursor.close() + + # Prepare and commit the transaction. + self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) + self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(20)) + self.session.commit_transaction() + + # WT-6325 reports WT_PREPARE_CONFLICT while iterating the cursor. + # Walk the table, the bug will cause a prepared conflict return. + cursor = self.session.open_cursor(uri) + while cursor.next() == 0: + continue + cursor.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py index eed6d8e0c26..e8db0550960 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py @@ -49,7 +49,7 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): scenarios = make_scenarios(prepare_values) def conn_config(self): - config = 'cache_size=1MB,statistics=(all),log=(enabled=true)' + config = 'cache_size=1MB,statistics=(all),log=(archive=false,enabled=true)' return config def simulate_crash_restart(self, olddir, newdir): diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp19.py b/src/third_party/wiredtiger/test/suite/test_timestamp19.py new file mode 100644 index 00000000000..54e121291d0 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_timestamp19.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_timestamp19.py +# Use the oldest timestamp in the metadata as the oldest timestamp on restart. +import wiredtiger, wttest +from wtdataset import SimpleDataSet + +def timestamp_str(t): + return '%x' % t + +class test_timestamp19(wttest.WiredTigerTestCase): + conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + session_config = 'isolation=snapshot' + + def updates(self, uri, value, ds, nrows, commit_ts): + session = self.session + cursor = session.open_cursor(uri) + for i in range(0, nrows): + session.begin_transaction() + cursor[ds.key(i)] = value + session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + cursor.close() + + def test_timestamp(self): + uri = "table:test_timestamp19" + create_params = 'value_format=S,key_format=i' + self.session.create(uri, create_params) + + ds = SimpleDataSet( + self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + ds.populate() + + nrows = 1000 + value_x = 'x' * 1000 + value_y = 'y' * 1000 + value_z = 'z' * 1000 + + # Set the oldest and stable timestamps to 10. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + + ', stable_timestamp=' + timestamp_str(10)) + + # Insert values with varying timestamps. + self.updates(uri, value_x, ds, nrows, 20) + self.updates(uri, value_y, ds, nrows, 30) + self.updates(uri, value_z, ds, nrows, 40) + + # Perform a checkpoint. + self.session.checkpoint('use_timestamp=true') + + # Move the oldest and stable timestamps to 40. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(40) + + ', stable_timestamp=' + timestamp_str(40)) + + # Update values. + self.updates(uri, value_z, ds, nrows, 50) + self.updates(uri, value_x, ds, nrows, 60) + self.updates(uri, value_y, ds, nrows, 70) + + # Perform a checkpoint. + self.session.checkpoint('use_timestamp=true') + + # Close and reopen the connection. + self.close_conn() + self.conn = self.setUpConnectionOpen('.') + self.session = self.setUpSessionOpen(self.conn) + + # The oldest timestamp on recovery is 40. Trying to set it earlier is a no-op. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), timestamp_str(40)) + + # Trying to set an earlier stable timestamp is an error. + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)), + '/oldest timestamp \(0, 40\) must not be later than stable timestamp \(0, 10\)/') + self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), timestamp_str(40)) + + # Move the oldest and stable timestamps to 70. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(70) + + ', stable_timestamp=' + timestamp_str(70)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), timestamp_str(70)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), timestamp_str(70)) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_txn13.py b/src/third_party/wiredtiger/test/suite/test_txn13.py index 541017804c9..fcd1dae95da 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn13.py +++ b/src/third_party/wiredtiger/test/suite/test_txn13.py @@ -40,13 +40,16 @@ class test_txn13(wttest.WiredTigerTestCase, suite_subprocess): logmax = "100K" tablename = 'test_txn13' uri = 'table:' + tablename - nops = 1024 + # We use 8 ops here to get around the 10 operation check done by WiredTiger to determine if + # a transaction is blocking or not. + nops = 8 create_params = 'key_format=i,value_format=S' + # The 1gb, 2gb and 4gb scenario names refer to the valuesize * nops. scenarios = make_scenarios([ - ('1gb', dict(expect_err=False, valuesize=1048576)), - ('2gb', dict(expect_err=False, valuesize=2097152)), - ('4gb', dict(expect_err=True, valuesize=4194304)) + ('1gb', dict(expect_err=False, valuesize=134217728)), + ('2gb', dict(expect_err=False, valuesize=268435456)), + ('4gb', dict(expect_err=True, valuesize=536870912)) ]) # Turn on logging for this test. diff --git a/src/third_party/wiredtiger/test/suite/test_txn19.py b/src/third_party/wiredtiger/test/suite/test_txn19.py index 004e849671e..3d81e2e9e6f 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn19.py +++ b/src/third_party/wiredtiger/test/suite/test_txn19.py @@ -481,7 +481,13 @@ class test_txn19_meta(wttest.WiredTigerTestCase, suite_subprocess): closeconn=False) if expect_fail: - self.check_file_contains_one_of(errfile, ['WT_TRY_SALVAGE: database corruption detected']) + errmsg = 'WT_TRY_SALVAGE: database corruption detected' + if self.filename == 'WiredTigerHS.wt': + if self.kind == 'removal': + errmsg = 'hs_exists' + elif self.kind == 'truncate': + errmsg = 'file size=0, alloc size=4096' + self.check_file_contains_one_of(errfile, [errmsg]) def test_corrupt_meta(self): errfile = 'list.err' |