diff options
Diffstat (limited to 'src/third_party/wiredtiger/test')
159 files changed, 2853 insertions, 2122 deletions
diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 9191a9447e8..5222df53fc0 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -435,15 +435,15 @@ functions: "upload artifact": - command: archive.targz_pack params: - target: "wiredtiger.tgz" - source_dir: "wiredtiger" + target: ${upload_filename|wiredtiger.tgz} + source_dir: ${upload_source_dir|wiredtiger} include: - "./**" - command: s3.put params: aws_secret: ${aws_secret} aws_key: ${aws_key} - local_file: wiredtiger.tgz + local_file: ${upload_filename|wiredtiger.tgz} bucket: build_external permissions: public-read content_type: application/tar @@ -2151,18 +2151,31 @@ tasks: script: | set -o errexit set -o verbose - - GCOV=/opt/mongodbtoolchain/v3/bin/gcov gcovr -r .. -e '.*/bt_(debug|dump|misc|salvage|vrfy).*' -e '.*/(log|progress|verify_build|strerror|env_msg|err_file|cur_config|os_abort)\..*' -e '.*_stat\..*' --html -o ../coverage_report.html + mkdir -p ../coverage_report + GCOV=/opt/mongodbtoolchain/v3/bin/gcov gcovr -r .. -f ../src -e '.*/bt_(debug|dump|misc|salvage|vrfy).*' -e '.*/(log|progress|verify_build|strerror|env_msg|err_file|cur_config|os_abort)\..*' -e '.*_stat\..*' -e 'bench' -e 'examples' -e 'test' -e 'ext' -e 'dist' -e 'tools' -j 4 --html-details --html-self-contained -o ../coverage_report/2_coverage_report.html + - command: s3.put + params: + aws_secret: ${aws_secret} + aws_key: ${aws_key} + local_files_include_filter: wiredtiger/coverage_report/* + bucket: build_external + permissions: public-read + content_type: text/html + remote_file: wiredtiger/${build_variant}/${revision}/coverage_report_${build_id}-${execution}/ - command: s3.put params: aws_secret: ${aws_secret} aws_key: ${aws_key} - local_file: wiredtiger/coverage_report.html + local_file: wiredtiger/coverage_report/2_coverage_report.html bucket: build_external permissions: public-read content_type: text/html - display_name: Coverage report - remote_file: wiredtiger/${build_variant}/${revision}/coverage_report/coverage_report_${build_id}-${execution}.html + # Ensure that the first character of the display_name is a space + # This will ensure that it sorts before the per-file report pages which also get a space + # at the start of their display name (why this happens is not yet clear). + display_name: " 1 Coverage report main page" + remote_file: wiredtiger/${build_variant}/${revision}/coverage_report_${build_id}-${execution}/1_coverage_report_main.html + - name: spinlock-gcc-test commands: @@ -2606,8 +2619,6 @@ tasks: script: | set -o errexit set -o verbose - sudo su - ulimit -n 1000000 export "PATH=/opt/mongodbtoolchain/v3/bin:$PATH" virtualenv -p python3 venv source venv/bin/activate @@ -2819,6 +2830,8 @@ buildvariants: --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH + upload_source_dir: mongo-tests/largescale/many-collection/dbpath/diagnostic.data + upload_filename: diagnostic.data.tgz tasks: - name: many-dhandle-stress-test - name: many-collection-test @@ -2908,7 +2921,7 @@ buildvariants: - name: code-statistics display_name: "Code statistics" - batchtime: 1440 # 1 day + batchtime: 10080 # 7 days run_on: - ubuntu1804-test expansions: @@ -2958,6 +2971,7 @@ buildvariants: windows_configure_flags: -vcvars_bat "'C:\Program Files (x86)\Microsoft Visual Studio\2017\Professional\VC\Auxiliary\Build\vcvars64.bat'" tasks: - name: compile + - name: make-check-test - name: unit-test - name: macos-1014 diff --git a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh index 521a9787145..b686b6a90ba 100755 --- a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh +++ b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh @@ -53,44 +53,134 @@ build_branch() } ############################################################# +# get_config_file_name: +# arg1: branch name +############################################################# +get_config_file_name() +{ + local file_name="" + branch_name=$1 + if [ "${wt_standalone}" = true ] || [ $older = true ] ; then + file_name="${branch_name}/test/format/CONFIG_default" + echo $file_name + return + fi + file_name="CONFIG_${branch_name}" + + echo $file_name +} + +############################################################# +# create_configs: +# arg1: branch name +############################################################# +create_configs() +{ + branch_name=$1 + + file_name=$(get_config_file_name $branch_name) + + if [ -f $file_name ] ; then + echo " WARNING - ${file_name} already exists, overwriting it." + fi + + echo "##################################################" > $file_name + echo "runs.type=row" >> $file_name # WT-7379 - Temporarily disable column store tests + echo "btree.prefix=0" >> $file_name # WT-7579 - Prefix testing isn't portable between releases + echo "cache=80" >> $file_name # Medium cache so there's eviction + echo "checksum=on" >> $file_name # WT-7851 Fix illegal checksum configuration + echo "checkpoints=1" >> $file_name # Force periodic writes + echo "compression=snappy" >> $file_name # We only built with snappy, force the choice + echo "data_source=table" >> $file_name + echo "huffman_key=0" >> $file_name # WT-6893 - Not supoprted by newer releases + echo "in_memory=0" >> $file_name # Interested in the on-disk format + echo "leak_memory=1" >> $file_name # Faster runs + echo "logging=1" >> $file_name # Test log compatibility + echo "logging_compression=snappy" >> $file_name # We only built with snappy, force the choice + echo "rows=1000000" >> $file_name + echo "salvage=0" >> $file_name # Faster runs + echo "timer=4" >> $file_name + echo "verify=1" >> $file_name # Faster runs + + # Append older release configs for newer compatibility release test + if [ $newer = true ]; then + for i in "${compatible_upgrade_downgrade_release_branches[@]}" + do + if [ "$i" == "$branch_name" ] ; then + echo "transaction.isolation=snapshot" >> $file_name # WT-7545 - Older releases can't do lower isolation levels + echo "transaction.timestamps=1" >> $file_name # WT-7545 - Older releases can't do non-timestamp transactions + break + fi + done + fi + echo "##################################################" >> $file_name +} + +############################################################# +# create_default_configs: +# This function will create the default configs for older and standalone +# release branches. +############################################################# +create_default_configs() +{ + # Iterate over the release branches and create configuration files + for b in `ls`; do + if [ -d "$b" ]; then + (create_configs $b) + fi + done +} + +############################################################# +# create_configs_for_newer_release_branches: +############################################################# +create_configs_for_newer_release_branches() +{ + # Create configs for all the newer releases + for b in ${newer_release_branches[@]}; do + (create_configs $b) + done + + # Copy per-release configs in the newer release branches + for b in ${newer_release_branches[@]}; do + cp -rf CONFIG* $b/test/format/ + done + + # Delete configs from the top folder + rm -rf CONFIG* +} + +############################################################# # run_format: # arg1: branch name # arg2: access methods list ############################################################# run_format() { + branch_name=$1 echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - echo "Running format in branch: \"$1\"" + echo "Running format in branch: \"$branch_name\"" echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - cd "$1/test/format" - - flags="-1q $(bflag $1)" - - args="" - args+="btree.prefix=0 " # Prefix testing isn't portable between releases - args+="cache=80 " # Medium cache so there's eviction - args+="checkpoints=1 " # Force periodic writes - args+="checksum=on " # Force checksums. - args+="compression=snappy " # We only built with snappy, force the choice - args+="data_source=table " - args+="huffman_key=0 " # Not supoprted by newer releases - args+="in_memory=0 " # Interested in the on-disk format - args+="leak_memory=1 " # Faster runs - args+="logging=1 " # Test log compatibility - args+="logging_compression=snappy " # We only built with snappy, force the choice - args+="rows=1000000 " - args+="runs.type=row " # Temporarily disable column store tests - args+="salvage=0 " # Faster runs - args+="timer=4 " - args+="transaction.isolation=snapshot " # Older releases can't do lower isolation levels - args+="transaction.timestamps=1 " # Older releases can't do non-timestamp transactions - args+="verify=0 " # Faster runs + cd "$branch_name/test/format" + flags="-1q $(bflag $branch_name)" + + config_file="" + + # Compatibility test for newer releases will have CONFIG file for each release + # branches for the upgrade/downgrade testing. + # + # Compatibility test for older and standalone releases will have the default config. + if [ "$newer" = true ]; then + config_file="-c CONFIG_${branch_name}" + else + config_file="-c CONFIG_default" + fi for am in $2; do dir="RUNDIR.$am" echo "./t running $am access method..." - ./t $flags -h $dir "file_type=$am" $args + ./t $flags ${config_file} -h $dir "file_type=$am" # Remove the version string from the base configuration file. (MongoDB does not create # a base configuration file, but format does, so we need to remove its version string @@ -145,6 +235,9 @@ upgrade_downgrade() echo "Upgrade/downgrade testing with \"$1\" and \"$2\"" echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + cfg_file_branch1=$(get_config_file_name $1) + cfg_file_branch2=$(get_config_file_name $2) + # Alternate running each branch format test program on the second branch's build. # Loop twice, that is, run format twice using each branch. top="$PWD" @@ -152,13 +245,13 @@ upgrade_downgrade() for reps in {1..2}; do echo "$1 format running on $2 access method $am..." cd "$top/$1/test/format" - flags="-1qR $(bflag $1)" - ./t $flags -h "$top/$2/test/format/RUNDIR.$am" timer=2 + flags="-1Rq $(bflag $1)" + ./t $flags -c "$top/$2/test/format/${cfg_file_branch1}" -h "$top/$2/test/format/RUNDIR.$am" timer=2 echo "$2 format running on $2 access method $am..." cd "$top/$2/test/format" - flags="-1qR $(bflag $2)" - ./t $flags -h "RUNDIR.$am" timer=2 + flags="-1Rq $(bflag $2)" + ./t $flags -c $cfg_file_branch2 -h "RUNDIR.$am" timer=2 done done } @@ -178,6 +271,11 @@ wt_standalone=false newer_release_branches=(develop mongodb-5.0 mongodb-4.4 mongodb-4.2) older_release_branches=(mongodb-4.2 mongodb-4.0 mongodb-3.6) +# This array is used to generate compatible configuration files between releases, because +# upgrade/downgrade test runs each build's format test program on the second build's +# configuration file. +compatible_upgrade_downgrade_release_branches=(mongodb-4.4 mongodb-4.2) + declare -A scopes scopes[newer]="newer stable release branches" scopes[older]="older stable release branches" @@ -229,6 +327,7 @@ top="test-compatibility-run" rm -rf "$top" && mkdir "$top" cd "$top" + # Build the branches. if [ "$newer" = true ]; then for b in ${newer_release_branches[@]}; do @@ -253,6 +352,13 @@ if [ "${wt_standalone}" = true ]; then (build_branch "$wt2") fi +if [ "$newer" = true ]; then + create_configs_for_newer_release_branches +else + create_default_configs +fi + + # Run format in each branch for supported access methods. if [ "$newer" = true ]; then for b in ${newer_release_branches[@]}; do @@ -267,8 +373,8 @@ if [ "$older" = true ]; then fi if [ "${wt_standalone}" = true ]; then - (run_format "$wt1" "fix row var") - (run_format "$wt2" "fix row var") + (run_format "$wt1" "row") + (run_format "$wt2" "row") fi # Verify backward compatibility for supported access methods. diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 9ea615cb519..81d21111ec3 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -149,6 +149,7 @@ tinfo_init(void) tinfo->session = NULL; tinfo->cursor = NULL; + memset(tinfo->insert_list, 0, sizeof(tinfo->insert_list)); tinfo->insert_list_cnt = 0; tinfo->state = TINFO_RUNNING; @@ -685,7 +686,7 @@ ops(void *arg) WT_SESSION *session; iso_level_t iso_level; thread_op op; - uint64_t reset_op, session_op, truncate_op; + uint64_t max_rows, reset_op, session_op, truncate_op; uint32_t range, rnd; u_int i, j; const char *iso_config; @@ -814,8 +815,12 @@ ops(void *arg) op = UPDATE; } - /* Select a row. */ - tinfo->keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows); + /* + * Select a row. Column-store extends the object, explicitly read the maximum row count and + * then use a local variable so the value won't change inside the loop. + */ + max_rows = (volatile uint64_t)g.rows; + tinfo->keyno = mmrand(&tinfo->rnd, 1, (u_int)max_rows); /* * Inserts, removes and updates can be done following a cursor set-key, or based on a cursor @@ -938,7 +943,7 @@ ops(void *arg) } if (!positioned) - tinfo->keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows); + tinfo->keyno = mmrand(&tinfo->rnd, 1, (u_int)max_rows); /* * Truncate up to 5% of the table. If the range overlaps the beginning/end of the table, @@ -950,7 +955,7 @@ ops(void *arg) * (truncating from lower keys to higher keys or vice-versa). */ greater_than = mmrand(&tinfo->rnd, 0, 1) == 1; - range = g.rows < 20 ? 0 : mmrand(&tinfo->rnd, 0, (u_int)g.rows / 20); + range = max_rows < 20 ? 0 : mmrand(&tinfo->rnd, 0, (u_int)max_rows / 20); tinfo->last = tinfo->keyno; if (greater_than) { if (g.c_reverse) { @@ -960,13 +965,13 @@ ops(void *arg) tinfo->last -= range; } else { tinfo->last += range; - if (tinfo->last > g.rows) + if (tinfo->last > max_rows) tinfo->last = 0; } } else { if (g.c_reverse) { tinfo->keyno += range; - if (tinfo->keyno > g.rows) + if (tinfo->keyno > max_rows) tinfo->keyno = 0; } else { if (tinfo->keyno <= range) @@ -1700,7 +1705,7 @@ col_insert_resolve(TINFO *tinfo) */ do { WT_ORDERED_READ(v, g.rows); - for (i = 0, p = tinfo->insert_list; i < WT_ELEMENTS(tinfo->insert_list); ++i) { + for (i = 0, p = tinfo->insert_list; i < WT_ELEMENTS(tinfo->insert_list); ++i, ++p) { if (*p == v + 1) { testutil_assert(__wt_atomic_casv64(&g.rows, v, v + 1)); *p = 0; @@ -1746,6 +1751,7 @@ col_insert(TINFO *tinfo, WT_CURSOR *cursor) else cursor->set_value(cursor, tinfo->value); + /* Create a record, then add the key to our list of new records for later resolution. */ if ((ret = cursor->insert(cursor)) != 0) return (ret); diff --git a/src/third_party/wiredtiger/test/manydbs/manydbs.c b/src/third_party/wiredtiger/test/manydbs/manydbs.c index 2dc0e5a1f8f..1dec47c5f16 100644 --- a/src/third_party/wiredtiger/test/manydbs/manydbs.c +++ b/src/third_party/wiredtiger/test/manydbs/manydbs.c @@ -54,6 +54,27 @@ static const char *const uri = "table:main"; #define MAX_KV 100 #define MAX_VAL 128 +/* + * Maximum expected condition variable wakeups. POSIX allows arbitrarily many spurious wakeups to + * happen, so we need to be able to adjust this expectation per-platform. There are two cases: when + * completely idle, and when running a light workload. The latter is expressed as a fraction of the + * total number of condition variable sleeps; the former is a constant. + */ +#if defined(__NetBSD__) || defined(_WIN32) +/* + * NetBSD should never generate spurious wakeups, but does: see https://gnats.netbsd.org/56275. + * Windows can also generate spurious wakeups: + * https://docs.microsoft.com/en-us/windows/win32/sync/condition-variables These values allow the + * test to complete in spite of that. + */ +#define CV_RESET_THRESHOLD_IDLE 20 +#define CV_RESET_THRESHOLD_DENOM 10 +#else +/* Default values: should be no wakeups when idle and allow 1/20 otherwise. */ +#define CV_RESET_THRESHOLD_IDLE 0 +#define CV_RESET_THRESHOLD_DENOM 20 +#endif + static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void usage(void) @@ -208,13 +229,13 @@ main(int argc, char *argv[]) * On an idle workload there should be no resets of condition variables during the idle * period. Even with a light workload, resets should not be very common. We look for 5%. */ - if (idle && cond_reset != cond_reset_orig[i]) - testutil_die( - ERANGE, "condition reset on idle connection %d of %" PRIu64, i, cond_reset); - if (!idle && cond_reset > cond_wait / 20) + if (idle && cond_reset > cond_reset_orig[i] + CV_RESET_THRESHOLD_IDLE) + testutil_die(ERANGE, "condition reset on idle connection %d of %" PRIu64 " exceeds %d", + i, cond_reset, CV_RESET_THRESHOLD_IDLE); + if (!idle && cond_reset > cond_wait / CV_RESET_THRESHOLD_DENOM) testutil_die(ERANGE, - "connection %d condition reset %" PRIu64 " exceeds 5%% of %" PRIu64, i, cond_reset, - cond_wait); + "connection %d condition reset %" PRIu64 " exceeds %d%% of %" PRIu64, i, cond_reset, + 100 / CV_RESET_THRESHOLD_DENOM, cond_wait); testutil_check(connections[i]->close(connections[i], NULL)); } diff --git a/src/third_party/wiredtiger/test/suite/test_assert01.py b/src/third_party/wiredtiger/test/suite/test_assert01.py index a91fd1700e0..4793ef26e72 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert01.py +++ b/src/third_party/wiredtiger/test/suite/test_assert01.py @@ -32,9 +32,7 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): base = 'assert01' @@ -43,13 +41,18 @@ class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): uri_def = base_uri + '.def.wt' uri_never = base_uri + '.never.wt' uri_none = base_uri + '.none.wt' - cfg = 'key_format=S,value_format=S,' cfg_always = 'verbose=[write_timestamp],write_timestamp_usage=always,assert=(write_timestamp=on)' cfg_def = '' cfg_never = 'verbose=(write_timestamp=true),write_timestamp_usage=never,assert=(write_timestamp=on)' cfg_none = 'assert=(write_timestamp=off)' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + count = 1 # # Commit a k/v pair making sure that it detects an error if needed, when @@ -57,13 +60,13 @@ class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): # def insert_check(self, uri, use_ts): c = self.session.open_cursor(uri) - key = 'key' + str(self.count) + key = 'key' + str(self.count) if self.usestrings else self.count val = 'value' + str(self.count) # Commit with a timestamp self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(self.count)) + 'commit_timestamp=' + self.timestamp_str(self.count)) c[key] = val # All settings other than never should commit successfully if (use_ts != 'never'): @@ -77,7 +80,7 @@ class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): self.count += 1 # Commit without a timestamp - key = 'key' + str(self.count) + key = 'key' + str(self.count) if self.usestrings else self.count val = 'value' + str(self.count) c = self.session.open_cursor(uri) self.session.begin_transaction() @@ -94,11 +97,13 @@ class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): c.close() def test_commit_timestamp(self): + cfg = 'key_format={},value_format=S,'.format(self.key_format) + # Create a data item at a timestamp - self.session.create(self.uri_always, self.cfg + self.cfg_always) - self.session.create(self.uri_def, self.cfg + self.cfg_def) - self.session.create(self.uri_never, self.cfg + self.cfg_never) - self.session.create(self.uri_none, self.cfg + self.cfg_none) + self.session.create(self.uri_always, cfg + self.cfg_always) + self.session.create(self.uri_def, cfg + self.cfg_def) + self.session.create(self.uri_never, cfg + self.cfg_never) + self.session.create(self.uri_none, cfg + self.cfg_none) # Check inserting into each table self.insert_check(self.uri_always, 'always') diff --git a/src/third_party/wiredtiger/test/suite/test_assert02.py b/src/third_party/wiredtiger/test/suite/test_assert02.py index ba871eb01c9..d3d5e885e87 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert02.py +++ b/src/third_party/wiredtiger/test/suite/test_assert02.py @@ -32,13 +32,17 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert02(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + def test_read_timestamp(self): #if not wiredtiger.diagnostic_build(): # self.skipTest('requires a diagnostic build') @@ -50,29 +54,32 @@ class test_assert02(wttest.WiredTigerTestCase, suite_subprocess): uri_never = base_uri + '.never.wt' uri_none = base_uri + '.none.wt' - cfg = 'key_format=S,value_format=S' + cfg = 'key_format={},value_format=S'.format(self.key_format) cfg_always = cfg + ',write_timestamp_usage=always,assert=(read_timestamp=always)' cfg_def = cfg cfg_never = cfg + ',assert=(read_timestamp=never)' cfg_none = cfg + ',assert=(read_timestamp=none)' - # Create a data item at a timestamp + # Create a data item at a timestamp. self.session.create(uri_always, cfg_always) self.session.create(uri_def, cfg_def) self.session.create(uri_never, cfg_never) self.session.create(uri_none, cfg_none) + # Make a key. + key1 = 'key1' if self.usestrings else 1 + # Insert a data item at timestamp 1. This should work for all. c_always = self.session.open_cursor(uri_always) c_def = self.session.open_cursor(uri_def) c_never = self.session.open_cursor(uri_never) c_none = self.session.open_cursor(uri_none) self.session.begin_transaction() - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(1)) - c_always['key1'] = 'value1' - c_def['key1'] = 'value1' - c_never['key1'] = 'value1' - c_none['key1'] = 'value1' + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(1)) + c_always[key1] = 'value1' + c_def[key1] = 'value1' + c_never[key1] = 'value1' + c_none[key1] = 'value1' self.session.commit_transaction() c_always.close() c_def.close() @@ -86,12 +93,12 @@ class test_assert02(wttest.WiredTigerTestCase, suite_subprocess): c_never = self.session.open_cursor(uri_never) c_none = self.session.open_cursor(uri_none) - c_always.set_key('key1') - c_def.set_key('key1') - c_never.set_key('key1') - c_none.set_key('key1') + c_always.set_key(key1) + c_def.set_key(key1) + c_never.set_key(key1) + c_none.set_key(key1) - self.session.begin_transaction('read_timestamp=' + timestamp_str(1)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(1)) c_always.search() c_def.search() c_none.search() @@ -114,10 +121,10 @@ class test_assert02(wttest.WiredTigerTestCase, suite_subprocess): c_never = self.session.open_cursor(uri_never) c_none = self.session.open_cursor(uri_none) - c_always.set_key('key1') - c_def.set_key('key1') - c_never.set_key('key1') - c_none.set_key('key1') + c_always.set_key(key1) + c_def.set_key(key1) + c_never.set_key(key1) + c_none.set_key(key1) self.session.begin_transaction() c_never.search() diff --git a/src/third_party/wiredtiger/test/suite/test_assert03.py b/src/third_party/wiredtiger/test/suite/test_assert03.py index d1ac3926269..51b219b2fac 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert03.py +++ b/src/third_party/wiredtiger/test/suite/test_assert03.py @@ -32,24 +32,36 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest +from wtscenario import make_scenarios class test_assert03(wttest.WiredTigerTestCase, suite_subprocess): conn_config = 'log=(enabled)' base_uri = 'file:assert03.wt' - cfg = 'key_format=S,value_format=S' always = 'write_timestamp_usage=always,assert=(write_timestamp=on)' never = 'write_timestamp_usage=never,assert=(write_timestamp=on)' none = 'assert=(write_timestamp=off)' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + def test_assert03(self): #if not wiredtiger.diagnostic_build(): # self.skipTest('requires a diagnostic build') + cfg = 'key_format={},value_format=S'.format(self.key_format) + key0 = 'key0' if self.usestrings else 17 + key1 = 'key1' if self.usestrings else 18 + key2 = 'key2' if self.usestrings else 19 + key3 = 'key3' if self.usestrings else 20 + # Create a data item at the default setting - self.session.create(self.base_uri, self.cfg) + self.session.create(self.base_uri, cfg) c = self.session.open_cursor(self.base_uri) self.session.begin_transaction() - c['key0'] = 'value0' + c[key0] = 'value0' self.session.commit_transaction() c.close() @@ -58,7 +70,7 @@ class test_assert03(wttest.WiredTigerTestCase, suite_subprocess): self.session.alter(self.base_uri, self.always) c = self.session.open_cursor(self.base_uri) self.session.begin_transaction() - c['key1'] = 'value1' + c[key1] = 'value1' msg = "/none set on this transaction/" self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda:self.assertEquals(self.session.commit_transaction(), 0), msg) @@ -68,14 +80,14 @@ class test_assert03(wttest.WiredTigerTestCase, suite_subprocess): self.session.alter(self.base_uri, self.never) c = self.session.open_cursor(self.base_uri) self.session.begin_transaction() - c['key2'] = 'value2' + c[key2] = 'value2' self.session.commit_transaction() c.close() self.session.alter(self.base_uri, self.none) c = self.session.open_cursor(self.base_uri) self.session.begin_transaction() - c['key3'] = 'value3' + c[key3] = 'value3' self.session.commit_transaction() c.close() diff --git a/src/third_party/wiredtiger/test/suite/test_assert04.py b/src/third_party/wiredtiger/test/suite/test_assert04.py index 711e1d02b06..6c5d4623d4d 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert04.py +++ b/src/third_party/wiredtiger/test/suite/test_assert04.py @@ -32,13 +32,17 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + def test_timestamp_alter(self): base = 'assert04' uri = 'file:' + base @@ -50,14 +54,17 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # Create the table without the key consistency checking turned on. # Create a few items breaking the rules. Then alter the setting and # verify the inconsistent usage is detected. - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + + key_nots = 'key_nots' if self.usestrings else 5 + key_ts1 = 'key_ts1' if self.usestrings else 16 # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)) - c['key_ts1'] = 'value2' + 'commit_timestamp=' + self.timestamp_str(2)) + c[key_ts1] = 'value2' self.session.commit_transaction() c.close() @@ -65,8 +72,8 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(1)) - c['key_ts1'] = 'value1' + 'commit_timestamp=' + self.timestamp_str(1)) + c[key_ts1] = 'value1' self.session.commit_transaction() c.close() @@ -74,27 +81,27 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # again modify without a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots' + c[key_nots] = 'value_nots' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)) - c['key_nots'] = 'value2' + 'commit_timestamp=' + self.timestamp_str(2)) + c[key_nots] = 'value2' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots2' + c[key_nots] = 'value_nots2' self.session.commit_transaction() c.close() # We must move the oldest timestamp forward in order to alter. # Otherwise alter closing the file will fail with EBUSY. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(2)) # Now alter the setting and make sure we detect incorrect usage. self.session.alter(uri, cfg_on) @@ -103,16 +110,16 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(5)) - c['key_ts1'] = 'value5' + 'commit_timestamp=' + self.timestamp_str(5)) + c[key_ts1] = 'value5' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(4)) - c['key_ts1'] = 'value4' + 'commit_timestamp=' + self.timestamp_str(4)) + c[key_ts1] = 'value4' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() @@ -120,7 +127,7 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # Detect not using a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value_nots3' + c[key_ts1] = 'value_nots3' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() @@ -131,41 +138,41 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # following transaction. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots3' + c[key_nots] = 'value_nots3' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(3)) - c['key_nots'] = 'value3' + 'commit_timestamp=' + self.timestamp_str(3)) + c[key_nots] = 'value3' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts1'], 'value5') - self.assertEquals(c['key_nots'], 'value_nots3') + self.assertEquals(c[key_ts1], 'value5') + self.assertEquals(c[key_nots], 'value_nots3') c.close() # Now alter the setting again and detection is off. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5)) self.session.alter(uri, cfg_off) # Detection is off we can successfully change the same key with and # without a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots4' + c[key_nots] = 'value_nots4' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(6)) - c['key_nots'] = 'value6' + 'commit_timestamp=' + self.timestamp_str(6)) + c[key_nots] = 'value6' self.session.commit_transaction() c.close() @@ -179,14 +186,22 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # That checking will verify any individual key is always or never # used with a timestamp. And if it is used with a timestamp that # the timestamps are in increasing order for that key. - self.session.create(uri, 'key_format=S,value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)') + self.session.create(uri, 'key_format={},value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)'.format(self.key_format)) + + key_nots = 'key_nots' if self.usestrings else 5 + key_ts1 = 'key_ts1' if self.usestrings else 16 + key_ts2 = 'key_ts2' if self.usestrings else 17 + key_ts3 = 'key_ts3' if self.usestrings else 18 + key_ts4 = 'key_ts4' if self.usestrings else 19 + key_ts5 = 'key_ts5' if self.usestrings else 20 + key_ts6 = 'key_ts6' if self.usestrings else 21 # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)) - c['key_ts1'] = 'value2' + 'commit_timestamp=' + self.timestamp_str(2)) + c[key_ts1] = 'value2' self.session.commit_transaction() c.close() @@ -194,8 +209,8 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(1)) - c['key_ts1'] = 'value1' + 'commit_timestamp=' + self.timestamp_str(1)) + c[key_ts1] = 'value1' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() @@ -204,8 +219,8 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(1)) - c['key_ts2'] = 'value1' + 'commit_timestamp=' + self.timestamp_str(1)) + c[key_ts2] = 'value1' self.session.commit_transaction() c.close() @@ -218,28 +233,28 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(10)) - c['key_ts3'] = 'value10' + 'commit_timestamp=' + self.timestamp_str(10)) + c[key_ts3] = 'value10' self.session.commit_transaction() self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(15)) - c['key_ts4'] = 'value15' + 'commit_timestamp=' + self.timestamp_str(15)) + c[key_ts4] = 'value15' self.session.commit_transaction() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(13)) - c['key_ts3'] = 'value13' - c['key_ts4'] = 'value13' + 'commit_timestamp=' + self.timestamp_str(13)) + c[key_ts3] = 'value13' + c[key_ts4] = 'value13' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts3'], 'value10') - self.assertEquals(c['key_ts4'], 'value15') + self.assertEquals(c[key_ts3], 'value10') + self.assertEquals(c[key_ts4], 'value15') c.close() # @@ -249,15 +264,15 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(13)) - c['key_ts3'] = 'value13' + 'commit_timestamp=' + self.timestamp_str(13)) + c[key_ts3] = 'value13' self.session.commit_transaction() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(13)) - c['key_ts4'] = 'value13' + 'commit_timestamp=' + self.timestamp_str(13)) + c[key_ts4] = 'value13' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() @@ -268,43 +283,43 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(14)) - c['key_ts4'] = 'value14' + 'commit_timestamp=' + self.timestamp_str(14)) + c[key_ts4] = 'value14' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value15') + self.assertEquals(c[key_ts4], 'value15') c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(16)) - c['key_ts4'] = 'value16' + 'commit_timestamp=' + self.timestamp_str(16)) + c[key_ts4] = 'value16' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value16') + self.assertEquals(c[key_ts4], 'value16') c.close() # Now try to modify a key previously used with timestamps without # one. We should get the inconsistent usage message. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value_nots' + c[key_ts4] = 'value_nots' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value_nots' + c[key_ts4] = 'value_nots' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value16') + self.assertEquals(c[key_ts4], 'value16') c.close() # Now confirm the other way. Create a key without a timestamp and then @@ -312,82 +327,82 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # makes sense here is the inconsistent usage. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots' + c[key_nots] = 'value_nots' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(16)) - c['key_nots'] = 'value16' + 'commit_timestamp=' + self.timestamp_str(16)) + c[key_nots] = 'value16' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots1' + c[key_nots] = 'value_nots1' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(17)) - c['key_nots'] = 'value17' + 'commit_timestamp=' + self.timestamp_str(17)) + c[key_nots] = 'value17' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_nots'], 'value_nots1') + self.assertEquals(c[key_nots], 'value_nots1') c.close() # Confirm it is okay to set the timestamp in the middle or end of the # transaction. That should set the timestamp for the whole thing. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts5'] = 'value_notsyet' + c[key_ts5] = 'value_notsyet' self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(20)) - c['key_ts5'] = 'value20' + 'commit_timestamp=' + self.timestamp_str(20)) + c[key_ts5] = 'value20' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts5'], 'value20') + self.assertEquals(c[key_ts5], 'value20') c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts6'] = 'value_notsyet' - c['key_ts6'] = 'value21_after' + c[key_ts6] = 'value_notsyet' + c[key_ts6] = 'value21_after' self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(21)) + 'commit_timestamp=' + self.timestamp_str(21)) self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts6'], 'value21_after') + self.assertEquals(c[key_ts6], 'value21_after') c.close() # Confirm it is okay to set the timestamp on the commit call. # That should set the timestamp for the whole thing. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts6'] = 'value_committs1' - c['key_ts6'] = 'value22' + c[key_ts6] = 'value_committs1' + c[key_ts6] = 'value22' self.session.commit_transaction('commit_timestamp=' + - timestamp_str(22)) + self.timestamp_str(22)) c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value23' + c[key_nots] = 'value23' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(23)), msg_usage) + 'commit_timestamp=' + self.timestamp_str(23)), msg_usage) c.close() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_assert05.py b/src/third_party/wiredtiger/test/suite/test_assert05.py index b5fe9205bd9..afa2acbcbfb 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert05.py +++ b/src/third_party/wiredtiger/test/suite/test_assert05.py @@ -32,9 +32,7 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): base = 'assert05' @@ -44,12 +42,17 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): uri_def = base_uri + '.def.wt' uri_never = base_uri + '.never.wt' uri_none = base_uri + '.none.wt' - cfg = 'key_format=S,value_format=S,' cfg_always = 'verbose=(write_timestamp=true),write_timestamp_usage=always,assert=(write_timestamp=on)' cfg_def = '' cfg_never = 'write_timestamp_usage=never,assert=(write_timestamp=on)' cfg_none = 'assert=(write_timestamp=off)' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + count = 1 # # Commit a k/v pair making sure that it detects an error if needed, when @@ -57,18 +60,18 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): # def insert_check(self, uri, use_ts): c = self.session.open_cursor(uri) - key = 'key' + str(self.count) + key = 'key' + str(self.count) if self.usestrings else self.count val = 'value' + str(self.count) # Commit with a timestamp self.session.begin_transaction() c[key] = val self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(self.count)) + 'prepare_timestamp=' + self.timestamp_str(self.count)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(self.count)) + 'commit_timestamp=' + self.timestamp_str(self.count)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(self.count)) + 'durable_timestamp=' + self.timestamp_str(self.count)) # All settings other than never should commit successfully if (use_ts != 'never'): self.session.commit_transaction() @@ -86,17 +89,17 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): self.count += 1 # Commit without a timestamp - key = 'key' + str(self.count) + key = 'key' + str(self.count) if self.usestrings else self.count val = 'value' + str(self.count) c = self.session.open_cursor(uri) self.session.begin_transaction() c[key] = val if (use_ts == 'always'): self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(self.count)) + 'prepare_timestamp=' + self.timestamp_str(self.count)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(self.count)) + 'commit_timestamp=' + self.timestamp_str(self.count)) # All settings other than always should commit successfully if (use_ts != 'always' and use_ts != 'never'): self.session.commit_transaction() @@ -114,11 +117,13 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): c.close() def test_durable_timestamp(self): + cfg = 'key_format={},value_format=S,'.format(self.key_format) + # Create a data item at a timestamp - self.session.create(self.uri_always, self.cfg + self.cfg_always) - self.session.create(self.uri_def, self.cfg + self.cfg_def) - self.session.create(self.uri_never, self.cfg + self.cfg_never) - self.session.create(self.uri_none, self.cfg + self.cfg_none) + self.session.create(self.uri_always, cfg + self.cfg_always) + self.session.create(self.uri_def, cfg + self.cfg_def) + self.session.create(self.uri_never, cfg + self.cfg_never) + self.session.create(self.uri_none, cfg + self.cfg_none) # Check inserting into each table self.insert_check(self.uri_always, 'always') diff --git a/src/third_party/wiredtiger/test/suite/test_assert06.py b/src/third_party/wiredtiger/test/suite/test_assert06.py index 24891445ab0..3ea620e638e 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert06.py +++ b/src/third_party/wiredtiger/test/suite/test_assert06.py @@ -32,20 +32,24 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + def apply_timestamps(self, timestamp): self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(timestamp)) + 'prepare_timestamp=' + self.timestamp_str(timestamp)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(timestamp)) + 'commit_timestamp=' + self.timestamp_str(timestamp)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(timestamp)) + 'durable_timestamp=' + self.timestamp_str(timestamp)) def test_timestamp_alter(self): base = 'assert06' @@ -55,14 +59,17 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): msg_ooo='/out of order/' msg_usage='/used inconsistently/' + key_nots = 'key_nots' if self.usestrings else 5 + key_ts1 = 'key_ts1' if self.usestrings else 16 + # Create the table without the key consistency checking turned on. # Create a few items breaking the rules. Then alter the setting and # verify the inconsistent usage is detected. - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value2' + c[key_ts1] = 'value2' self.apply_timestamps(2) self.session.commit_transaction() c.close() @@ -70,7 +77,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # Modify the data item at timestamp 1. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value1' + c[key_ts1] = 'value1' self.apply_timestamps(1) self.session.commit_transaction() c.close() @@ -79,26 +86,26 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # again modify without a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots' + c[key_nots] = 'value_nots' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value2' + c[key_nots] = 'value2' self.apply_timestamps(2) self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots2' + c[key_nots] = 'value_nots2' self.session.commit_transaction() c.close() # We must move the oldest timestamp forward in order to alter. # Otherwise alter closing the file will fail with EBUSY. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(2)) # Now alter the setting and make sure we detect incorrect usage. self.session.alter(uri, cfg_on) @@ -106,7 +113,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # Detect decreasing timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value5' + c[key_ts1] = 'value5' self.apply_timestamps(5) self.session.commit_transaction() c.close() @@ -116,7 +123,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value4' + c[key_ts1] = 'value4' self.apply_timestamps(4) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) @@ -126,7 +133,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # Detect not using a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value_nots3' + c[key_ts1] = 'value_nots3' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() @@ -137,7 +144,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # following transaction. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots3' + c[key_nots] = 'value_nots3' self.session.commit_transaction() c.close() @@ -146,7 +153,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value3' + c[key_nots] = 'value3' self.apply_timestamps(3) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) @@ -155,26 +162,26 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): ''' c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts1'], 'value5') - self.assertEquals(c['key_nots'], 'value_nots3') + self.assertEquals(c[key_ts1], 'value5') + self.assertEquals(c[key_nots], 'value_nots3') c.close() # Test to make sure that key consistency can be turned off # after turning it on. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5)) self.session.alter(uri, cfg_off) # Detection is off we can successfully change the same key with and # without a timestamp. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots4' + c[key_nots] = 'value_nots4' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value6' + c[key_nots] = 'value6' self.apply_timestamps(6) self.session.commit_transaction() c.close() @@ -185,16 +192,24 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): msg_ooo='/out of order/' msg_usage='/used inconsistently/' + key_nots = 'key_nots' if self.usestrings else 5 + key_ts1 = 'key_ts1' if self.usestrings else 16 + key_ts2 = 'key_ts2' if self.usestrings else 17 + key_ts3 = 'key_ts3' if self.usestrings else 18 + key_ts4 = 'key_ts4' if self.usestrings else 19 + key_ts5 = 'key_ts5' if self.usestrings else 20 + key_ts6 = 'key_ts6' if self.usestrings else 21 + # Create the table with the key consistency checking turned on. # That checking will verify any individual key is always or never # used with a timestamp. And if it is used with a timestamp that # the timestamps are in increasing order for that key. - self.session.create(uri, 'key_format=S,value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)') + self.session.create(uri, 'key_format={},value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)'.format(self.key_format)) # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value2' + c[key_ts1] = 'value2' self.apply_timestamps(2) self.session.commit_transaction() c.close() @@ -205,7 +220,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # Modify the data item at timestamp 1. We should detect it is wrong. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts1'] = 'value1' + c[key_ts1] = 'value1' self.apply_timestamps(1) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) @@ -215,7 +230,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # Make sure we can successfully add a different key at timestamp 1. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts2'] = 'value1' + c[key_ts2] = 'value1' self.apply_timestamps(1) self.session.commit_transaction() c.close() @@ -228,11 +243,11 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts3'] = 'value10' + c[key_ts3] = 'value10' self.apply_timestamps(10) self.session.commit_transaction() self.session.begin_transaction() - c['key_ts4'] = 'value15' + c[key_ts4] = 'value15' self.apply_timestamps(15) self.session.commit_transaction() @@ -241,8 +256,8 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts3'] = 'value13' - c['key_ts4'] = 'value13' + c[key_ts3] = 'value13' + c[key_ts4] = 'value13' self.apply_timestamps(13) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) @@ -250,8 +265,8 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): ''' c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts3'], 'value10') - self.assertEquals(c['key_ts4'], 'value15') + self.assertEquals(c[key_ts3], 'value10') + self.assertEquals(c[key_ts4], 'value15') c.close() # @@ -260,7 +275,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts3'] = 'value13' + c[key_ts3] = 'value13' self.apply_timestamps(13) self.session.commit_transaction() @@ -269,7 +284,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value13' + c[key_ts4] = 'value13' self.apply_timestamps(13) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) @@ -284,43 +299,43 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # in the update chain are not considered for the timestamp check. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value14' + c[key_ts4] = 'value14' self.apply_timestamps(14) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_ooo) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value15') + self.assertEquals(c[key_ts4], 'value15') c.close() ''' c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value16' + c[key_ts4] = 'value16' self.apply_timestamps(16) self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value16') + self.assertEquals(c[key_ts4], 'value16') c.close() # Now try to modify a key previously used with timestamps without # one. We should get the inconsistent usage message. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value_nots' + c[key_ts4] = 'value_nots' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts4'] = 'value_nots' + c[key_ts4] = 'value_nots' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts4'], 'value16') + self.assertEquals(c[key_ts4], 'value16') c.close() # Now confirm the other way. Create a key without a timestamp and then @@ -328,7 +343,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # makes sense here is the inconsistent usage. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots' + c[key_nots] = 'value_nots' self.session.commit_transaction() c.close() @@ -337,7 +352,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value16' + c[key_nots] = 'value16' self.apply_timestamps(16) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) @@ -346,7 +361,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value_nots1' + c[key_nots] = 'value_nots1' self.session.commit_transaction() c.close() @@ -355,7 +370,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value17' + c[key_nots] = 'value17' self.apply_timestamps(17) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg_usage) @@ -363,47 +378,47 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): ''' c = self.session.open_cursor(uri) - self.assertEquals(c['key_nots'], 'value_nots1') + self.assertEquals(c[key_nots], 'value_nots1') c.close() # Confirm it is okay to set the timestamp in the middle or end of the # transaction. That should set the timestamp for the whole thing. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts5'] = 'value_notsyet' - c['key_ts5'] = 'value20' + c[key_ts5] = 'value_notsyet' + c[key_ts5] = 'value20' self.apply_timestamps(20) self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts5'], 'value20') + self.assertEquals(c[key_ts5], 'value20') c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts6'] = 'value_notsyet' - c['key_ts6'] = 'value21_after' + c[key_ts6] = 'value_notsyet' + c[key_ts6] = 'value21_after' self.apply_timestamps(21) self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) - self.assertEquals(c['key_ts6'], 'value21_after') + self.assertEquals(c[key_ts6], 'value21_after') c.close() # Confirm it is okay to set the durable timestamp on the commit call. # That should set the timestamp for the whole thing. c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_ts6'] = 'value_committs1' - c['key_ts6'] = 'value22' + c[key_ts6] = 'value_committs1' + c[key_ts6] = 'value22' self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(22)) + 'prepare_timestamp=' + self.timestamp_str(22)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(22)) + 'commit_timestamp=' + self.timestamp_str(22)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(22)) + 'durable_timestamp=' + self.timestamp_str(22)) self.session.commit_transaction() c.close() @@ -412,14 +427,14 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key_nots'] = 'value23' + c[key_nots] = 'value23' self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(23)) + 'prepare_timestamp=' + self.timestamp_str(23)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(23)) + 'commit_timestamp=' + self.timestamp_str(23)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'durable_timestamp=' + timestamp_str(23)), msg_usage) + 'durable_timestamp=' + self.timestamp_str(23)), msg_usage) c.close() ''' diff --git a/src/third_party/wiredtiger/test/suite/test_assert07.py b/src/third_party/wiredtiger/test/suite/test_assert07.py index e79dd1a27dc..687174b4796 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert07.py +++ b/src/third_party/wiredtiger/test/suite/test_assert07.py @@ -33,95 +33,101 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_assert07(wttest.WiredTigerTestCase, suite_subprocess): + key_format_values = [ + ('column', dict(key_format='r', usestrings=False)), + ('string-row', dict(key_format='S', usestrings=True)) + ] + scenarios = make_scenarios(key_format_values) + def apply_timestamps(self, timestamp): self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(timestamp)) + 'prepare_timestamp=' + self.timestamp_str(timestamp)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(timestamp)) + 'commit_timestamp=' + self.timestamp_str(timestamp)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(timestamp)) + 'durable_timestamp=' + self.timestamp_str(timestamp)) def test_timestamp_alter(self): base = 'assert07' uri = 'file:' + base + key_ts1 = 'key_ts1' if self.usestrings else 1 + # No reserved, single update. - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) c = self.session.open_cursor(uri) self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value1' + c[key_ts1] = 'value1' self.apply_timestamps(1) self.session.commit_transaction() # Reserved at the start of the chain, with one update. self.session.begin_transaction('isolation=snapshot') - c.set_key('key_ts1') + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value2' + c[key_ts1] = 'value2' self.apply_timestamps(2) self.session.commit_transaction() # Reserved at the end of the chain, with one update. self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value3' - c.set_key('key_ts1') + c[key_ts1] = 'value3' + c.set_key(key_ts1) c.reserve() self.apply_timestamps(3) self.session.commit_transaction() # Reserved at the start of the chain, with multiple. self.session.begin_transaction('isolation=snapshot') - c.set_key('key_ts1') + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value4' - c['key_ts1'] = 'value5' + c[key_ts1] = 'value4' + c[key_ts1] = 'value5' self.apply_timestamps(4) self.session.commit_transaction() # Reserved at the end of the chain, with multiple updates. self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value6' - c['key_ts1'] = 'value7' - c.set_key('key_ts1') + c[key_ts1] = 'value6' + c[key_ts1] = 'value7' + c.set_key(key_ts1) c.reserve() self.apply_timestamps(5) self.session.commit_transaction() # Reserved between two updates. self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value8' - c.set_key('key_ts1') + c[key_ts1] = 'value8' + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value9' + c[key_ts1] = 'value9' self.apply_timestamps(6) self.session.commit_transaction() # Reserved update with multiple extra updates. self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value10' - c.set_key('key_ts1') + c[key_ts1] = 'value10' + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value11' - c['key_ts1'] = 'value12' - c['key_ts1'] = 'value13' + c[key_ts1] = 'value11' + c[key_ts1] = 'value12' + c[key_ts1] = 'value13' self.apply_timestamps(7) self.session.commit_transaction() # Reserved updates with multiple extra updates. self.session.begin_transaction('isolation=snapshot') - c['key_ts1'] = 'value14' - c.set_key('key_ts1') + c[key_ts1] = 'value14' + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value15' - c['key_ts1'] = 'value16' - c.set_key('key_ts1') + c[key_ts1] = 'value15' + c[key_ts1] = 'value16' + c.set_key(key_ts1) c.reserve() - c['key_ts1'] = 'value17' + c[key_ts1] = 'value17' self.apply_timestamps(8) self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_backup08.py b/src/third_party/wiredtiger/test/suite/test_backup08.py index 1c674df5a54..95e50971408 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup08.py +++ b/src/third_party/wiredtiger/test/suite/test_backup08.py @@ -34,9 +34,6 @@ import os, shutil import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_backup08(wttest.WiredTigerTestCase): conn_config = 'config_base=false,create,log=(enabled)' session_config = 'isolation=snapshot' @@ -85,12 +82,12 @@ class test_backup08(wttest.WiredTigerTestCase): curs[i] = i self.pr("i: " + str(i)) self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(i)) + 'commit_timestamp=' + self.timestamp_str(i)) # Set the oldest and stable timestamp a bit earlier than the data # we inserted. Take a checkpoint to the stable timestamp. self.pr("stable ts: " + str(ts)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(ts) + - ',stable_timestamp=' + timestamp_str(ts)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(ts) + + ',stable_timestamp=' + self.timestamp_str(ts)) # This forces a different checkpoint timestamp for each table. # Default is to use the stable timestamp. expected = ts @@ -102,7 +99,7 @@ class test_backup08(wttest.WiredTigerTestCase): ckpt_config = 'use_timestamp=true' self.session.checkpoint(ckpt_config) q = self.conn.query_timestamp('get=last_checkpoint') - self.assertTimestampsEqual(q, timestamp_str(expected)) + self.assertTimestampsEqual(q, self.timestamp_str(expected)) return expected def backup_and_recover(self, expected_rec_ts): @@ -123,7 +120,7 @@ class test_backup08(wttest.WiredTigerTestCase): backup_conn = self.wiredtiger_open(self.dir) q = backup_conn.query_timestamp('get=recovery') self.pr("query recovery ts: " + q) - self.assertTimestampsEqual(q, timestamp_str(expected_rec_ts)) + self.assertTimestampsEqual(q, self.timestamp_str(expected_rec_ts)) def test_timestamp_backup(self): # Add some data and checkpoint using the timestamp or not diff --git a/src/third_party/wiredtiger/test/suite/test_backup15.py b/src/third_party/wiredtiger/test/suite/test_backup15.py index 6aa2f854b95..009fdf85e40 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup15.py +++ b/src/third_party/wiredtiger/test/suite/test_backup15.py @@ -25,13 +25,8 @@ # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. - -import wiredtiger -import os, shutil +import os from wtbackup import backup_base -from wtdataset import simple_key -from wtscenario import make_scenarios -import glob # test_backup15.py # Test cursor backup with a block-based incremental cursor. @@ -118,6 +113,3 @@ class test_backup15(backup_base): self.take_full_backup(self.home_full) self.compare_backups(self.uri, self.home_full, self.home_incr, str(self.bkup_id)) self.setup_directories(self.home_incr, self.home_full) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug022.py b/src/third_party/wiredtiger/test/suite/test_bug022.py index fe7c2c32f28..31fb2387662 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug022.py +++ b/src/third_party/wiredtiger/test/suite/test_bug022.py @@ -31,9 +31,6 @@ import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_bug022(wttest.WiredTigerTestCase): uri = 'file:test_bug022' conn_config = 'cache_size=50MB' @@ -41,21 +38,21 @@ class test_bug022(wttest.WiredTigerTestCase): def test_apply_modifies_on_onpage_tombstone(self): self.session.create(self.uri, 'key_format=S,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) value = 'a' * 500 for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Apply tombstones for every key. for i in range(1, 10000): self.session.begin_transaction() cursor.set_key(str(i)) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint03.py b/src/third_party/wiredtiger/test/suite/test_checkpoint03.py index 0f4fba34186..10f6138c2c0 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint03.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint03.py @@ -39,9 +39,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_checkpoint03(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_checkpoint03' conn_config = 'statistics=(all)' diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint04.py b/src/third_party/wiredtiger/test/suite/test_checkpoint04.py index 413f3333428..eda62c33f66 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint04.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint04.py @@ -33,9 +33,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - class test_checkpoint04(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' session_config = 'isolation=snapshot' diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint06.py b/src/third_party/wiredtiger/test/suite/test_checkpoint06.py index edd22434228..06795a3cd27 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint06.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint06.py @@ -29,9 +29,6 @@ import time import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - # test_checkpoint06.py # Verify that we rollback the truncation that is committed after stable # timestamp in the checkpoint. @@ -44,14 +41,14 @@ class test_checkpoint06(wttest.WiredTigerTestCase): self.session.create(self.uri, 'key_format=i,value_format=S') value = "abcdefghijklmnopqrstuvwxyz" * 3 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) self.session.begin_transaction() # Setup: Insert some data for i in range(10000): cursor[i] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(2)) # Flush everything to disk self.reopen_conn() @@ -63,7 +60,7 @@ class test_checkpoint06(wttest.WiredTigerTestCase): end = self.session.open_cursor(self.uri) end.set_key(9995) self.session.truncate(None, start, None, None) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Do a checkpoint self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint07.py b/src/third_party/wiredtiger/test/suite/test_checkpoint07.py index c1c11dbad91..a876d1ac807 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint07.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint07.py @@ -33,9 +33,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - class test_checkpoint07(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' session_config = 'isolation=snapshot' diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint08.py b/src/third_party/wiredtiger/test/suite/test_checkpoint08.py index 5a8e8fe647e..49de9c5f163 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint08.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint08.py @@ -37,9 +37,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - class test_checkpoint08(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' session_config = 'isolation=snapshot' @@ -61,8 +58,8 @@ class test_checkpoint08(wttest.WiredTigerTestCase): self.session.create(self.uri2, 'key_format=i,value_format=i') # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) # Setup: Insert some data and checkpoint it. Then modify only # the data in the first table and checkpoint. Verify the clean skip @@ -73,23 +70,23 @@ class test_checkpoint08(wttest.WiredTigerTestCase): self.session.begin_transaction() c1[1] = 1 c2[1] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() c1[1] = 10 c2[1] = 10 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(3)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(3)) self.session.checkpoint(None) # Modify the both tables and reverify. self.session.begin_transaction() c1[3] = 3 c2[3] = 3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(4)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(4)) self.session.checkpoint(None) val = self.get_stat(self.uri1) @@ -100,13 +97,13 @@ class test_checkpoint08(wttest.WiredTigerTestCase): self.assertNotEqual(hsval, 0) # Modify the both tables and reverify when oldest timestamp moved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(4)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() c1[4] = 4 c2[4] = 4 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) self.session.checkpoint(None) val = self.get_stat(self.uri1) diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot01.py b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot01.py index ce839469ed1..0bc38abb5d2 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot01.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot01.py @@ -42,7 +42,7 @@ from wiredtiger import stat # class test_checkpoint_snapshot01(wttest.WiredTigerTestCase): - conn_config = 'cache_size=50MB,statistics=(fast)' + conn_config = 'cache_size=50MB' # Create a table. uri = "table:test_checkpoint_snapshot01" diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py index 2f93fef82b2..9f796b28089 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py @@ -38,9 +38,6 @@ from wiredtiger import stat # This test is to run checkpoint and eviction in parallel with timing # stress for checkpoint and let eviction write more data than checkpoint. # - -def timestamp_str(t): - return '%x' % t class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): # Create a table. @@ -68,7 +65,7 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): if commit_ts == 0: session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, check_value, uri, nrows, read_ts): @@ -76,7 +73,7 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): if read_ts == 0: session.begin_transaction() else: - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -137,8 +134,8 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): valuea = "aaaaa" * 100 # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) self.large_updates(self.uri, valuea, ds, self.nrows, 20) self.check(valuea, self.uri, self.nrows, 20) @@ -151,10 +148,10 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): cursor1.set_key(ds.key(i)) cursor1.set_value(valuea) self.assertEqual(cursor1.insert(), 0) - session1.timestamp_transaction('commit_timestamp=' + timestamp_str(30)) + session1.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) # Set stable timestamp to 40 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) # Create a checkpoint thread done = threading.Event() @@ -191,8 +188,8 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): valueb = "bbbbb" * 100 # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) session1 = self.conn.open_session() session1.begin_transaction() @@ -208,10 +205,10 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): cursor2.set_key(ds.key(i)) cursor2.set_value(valuea) self.assertEqual(cursor2.insert(), 0) - session1.timestamp_transaction('commit_timestamp=' + timestamp_str(30)) + session1.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) # Set stable timestamp to 40 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) # Create a checkpoint thread done = threading.Event() diff --git a/src/third_party/wiredtiger/test/suite/test_compress02.py b/src/third_party/wiredtiger/test/suite/test_compress02.py index a88b8f54304..e2ce0dd3a33 100644 --- a/src/third_party/wiredtiger/test/suite/test_compress02.py +++ b/src/third_party/wiredtiger/test/suite/test_compress02.py @@ -39,11 +39,7 @@ from wiredtiger import stat # we are using zstd as the block compressor. Tables created before reconfiguration # will still use the previous compression level. # - -def timestamp_str(t): - return '%x' % t class test_compress02(wttest.WiredTigerTestCase): - # Create a table. uri = "table:test_compress02" nrows = 1000 diff --git a/src/third_party/wiredtiger/test/suite/test_config07.py b/src/third_party/wiredtiger/test/suite/test_config07.py index 6539178cb7e..33789d518a0 100644 --- a/src/third_party/wiredtiger/test/suite/test_config07.py +++ b/src/third_party/wiredtiger/test/suite/test_config07.py @@ -83,7 +83,7 @@ class test_config07(wttest.WiredTigerTestCase): msg = '/invalid log extend length/' config = 'log=(enabled,file_max=1M),file_extend=' + self.log_extend_len - configarg = 'create,statistics=(fast)' + ',' + config + configarg = 'create,' + config # Expect an error when an invalid log extend size is provided. if self.expected_log_size is None: diff --git a/src/third_party/wiredtiger/test/suite/test_cursor15.py b/src/third_party/wiredtiger/test/suite/test_cursor15.py index ab136333682..1a950b7fc51 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor15.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor15.py @@ -37,7 +37,7 @@ class test_cursor15(wttest.WiredTigerTestCase): tablename = 'test_read_once' uri = 'table:' + tablename - conn_config = 'cache_size=1M,statistics=(all)' + conn_config = 'cache_size=1M' def test_cursor15(self): # This test is configured to use 1MB of cache. It will insert 20 @@ -51,7 +51,7 @@ class test_cursor15(wttest.WiredTigerTestCase): cursor[key] = '1' * (100 * 1024) cursor.close() - # Restart the database to clear the cache and reset statistics. + # Restart the database to clear the cache. self.reopen_conn() # We don't restart the database between runs to exercise that read_once diff --git a/src/third_party/wiredtiger/test/suite/test_cursor17.py b/src/third_party/wiredtiger/test/suite/test_cursor17.py index f068279f8f1..60cad85a5e2 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor17.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor17.py @@ -30,9 +30,6 @@ import wiredtiger, wttest from wtscenario import make_scenarios from wiredtiger import stat, WT_NOTFOUND -def timestamp_str(t): - return '%x' % t - # test_cursor17.py # Test the cursor traversal optimization for delete heavy workloads. This optimization enables # cursor traversal mechanism to skip pages where all records on the page are deleted with a @@ -60,8 +57,8 @@ class test_cursor17(wttest.WiredTigerTestCase): total_keys = 40000 # Keep the oldest and the stable timestamp pinned. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(2)) cursor = self.session.open_cursor(uri) commit_timestamp = 3 @@ -70,7 +67,7 @@ class test_cursor17(wttest.WiredTigerTestCase): for key in range(total_keys): self.session.begin_transaction() cursor[key] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_timestamp)) commit_timestamp += 1 # Delete everything on the table except for the first and the last KV pair. @@ -78,13 +75,13 @@ class test_cursor17(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(key) self.assertEqual(cursor.remove(),0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_timestamp)) commit_timestamp += 1 # Take a checkpoint to reconcile the pages. self.session.checkpoint() - self.session.begin_transaction('read_timestamp=' + timestamp_str(commit_timestamp)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(commit_timestamp)) # Position the cursor on the first record. cursor.set_key(0) self.assertEqual(cursor.search(), 0) @@ -107,11 +104,11 @@ class test_cursor17(wttest.WiredTigerTestCase): # Update a key in the middle of the table. self.session.begin_transaction() cursor[total_keys // 2] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_timestamp)) commit_timestamp += 1 # Make sure we can reach a the record we updated in the middle of the table. - self.session.begin_transaction('read_timestamp=' + timestamp_str(commit_timestamp)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(commit_timestamp)) # Position the cursor on the first record. cursor.set_key(0) self.assertEqual(cursor.search(), 0) diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode05.py b/src/third_party/wiredtiger/test/suite/test_debug_mode05.py index f322dd98f7a..bb480138997 100644 --- a/src/third_party/wiredtiger/test/suite/test_debug_mode05.py +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode05.py @@ -28,9 +28,6 @@ import wiredtiger, wttest -def timestamp_str(t): - return '%x' %t - # test_debug_mode05.py # As per WT-5046, the debug table logging settings prevent rollback to # stable in the presence of prepared transactions. @@ -44,7 +41,7 @@ class test_debug_mode05(wttest.WiredTigerTestCase): def test_table_logging_rollback_to_stable(self): self.session.create(self.uri, 'key_format=i,value_format=u,log=(enabled=false)') - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100)) self.session.checkpoint() # Try doing a normal prepared txn and then rollback to stable. @@ -53,11 +50,11 @@ class test_debug_mode05(wttest.WiredTigerTestCase): for i in range(1, 50): cursor[i] = b'a' * 100 self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(150)) + 'prepare_timestamp=' + self.timestamp_str(150)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(200)) + 'commit_timestamp=' + self.timestamp_str(200)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(250)) + 'durable_timestamp=' + self.timestamp_str(250)) self.session.commit_transaction() cursor.close() @@ -70,11 +67,11 @@ class test_debug_mode05(wttest.WiredTigerTestCase): # Therefore, we're specifically not doing any operations here. self.session.begin_transaction() self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(300)) + 'prepare_timestamp=' + self.timestamp_str(300)) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(350)) + 'commit_timestamp=' + self.timestamp_str(350)) self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(400)) + 'durable_timestamp=' + self.timestamp_str(400)) self.session.commit_transaction() # The aforementioned bug resulted in a failure in rollback to stable. @@ -88,7 +85,7 @@ class test_debug_mode05(wttest.WiredTigerTestCase): for i in range(1, 50): cursor[i] = b'b' * 100 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(450)) + 'commit_timestamp=' + self.timestamp_str(450)) cursor.close() self.conn.rollback_to_stable() diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode07.py b/src/third_party/wiredtiger/test/suite/test_debug_mode07.py index c957fe57799..6c1432937fe 100644 --- a/src/third_party/wiredtiger/test/suite/test_debug_mode07.py +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode07.py @@ -31,7 +31,7 @@ import wttest, wiredtiger # test_debug_mode07.py # Test the debug mode settings. Test realloc_exact use (from WT-4919). class test_debug_mode07(wttest.WiredTigerTestCase): - conn_config = 'log=(enabled=true),debug_mode=(realloc_exact=true),statistics=(all)' + conn_config = 'log=(enabled=true),debug_mode=(realloc_exact=true)' uri = 'file:test_debug_mode07' # Insert some data to ensure setting/unsetting the flag does not diff --git a/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py b/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py index aa5ed0aae65..dfad0d8869d 100644 --- a/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py @@ -32,9 +32,6 @@ from suite_subprocess import suite_subprocess from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' %t - # test_durable_rollback_to_stable.py # Checking visibility and durability of updates with durable_timestamp and # with rollback to stable. @@ -44,8 +41,7 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces keyfmt = [ ('row-string', dict(keyfmt='S')), ('row-int', dict(keyfmt='i')), - # The commented columnar tests needs to be enabled once rollback to stable for columnar is fixed in (WT-5548). - # ('column-store', dict(keyfmt='r')), + ('column-store', dict(keyfmt='r')), ] types = [ ('file', dict(uri='file', ds=SimpleDataSet)), @@ -78,7 +74,7 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces cursor = session.open_cursor(uri, None) # Set stable timestamp to checkpoint initial data set. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100)) self.session.checkpoint() # Update all values with value 111 i.e. first update value. @@ -89,15 +85,15 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(220)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(200)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(220)) session.commit_transaction() # Check the values read are correct with different timestamps. # Read the initial dataset. self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(150)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(150)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(i)) @@ -106,7 +102,7 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces # Read the first update value with timestamp. self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(200)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(200)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(111)) @@ -123,7 +119,7 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces session.commit_transaction() # Set a stable timestamp so that first update value is durable. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) # Update all values with value 222 i.e. second update value. self.assertEquals(cursor.reset(), 0) @@ -134,12 +130,12 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(200)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(200)) # Commit timestamp is earlier to stable timestamp but durable timestamp # is later than stable timestamp. Hence second update value is not durable. - session.timestamp_transaction('commit_timestamp=' + timestamp_str(240)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(300)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(240)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(300)) session.commit_transaction() # Checkpoint so that first update value will be visible and durable, @@ -160,8 +156,8 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces self.conn.rollback_to_stable() session = self.conn.open_session(self.session_config) cursor = session.open_cursor(uri, None) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(250)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(111)) diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts01.py b/src/third_party/wiredtiger/test/suite/test_durable_ts01.py index 7195aa85569..cd20fbf6b35 100644 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts01.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts01.py @@ -31,9 +31,6 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' %t - # test_durable_ts01.py # Checking visibility and durability of updates with durable_timestamp and # with restart. @@ -43,8 +40,7 @@ class test_durable_ts01(wttest.WiredTigerTestCase): keyfmt = [ ('row-string', dict(keyfmt='S')), ('row-int', dict(keyfmt='i')), - # The commented columnar tests needs to be enabled once rollback to stable for columnar is fixed in (WT-5548). - # ('column-store', dict(keyfmt='r')), + ('column-store', dict(keyfmt='r')), ] types = [ ('file', dict(uri='file', ds=SimpleDataSet)), @@ -77,7 +73,7 @@ class test_durable_ts01(wttest.WiredTigerTestCase): cursor = session.open_cursor(uri, None) # Set stable timestamp to checkpoint initial data set. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100)) self.session.checkpoint() # Update all values with value 111 i.e. first update value. @@ -88,15 +84,15 @@ class test_durable_ts01(wttest.WiredTigerTestCase): self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(220)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(200)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(220)) session.commit_transaction() # Check the values read are correct with different timestamps. # Read the initial dataset. self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(150)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(150)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(i)) @@ -105,7 +101,7 @@ class test_durable_ts01(wttest.WiredTigerTestCase): # Read the first update value with timestamp. self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(200)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(200)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(111)) @@ -122,7 +118,7 @@ class test_durable_ts01(wttest.WiredTigerTestCase): session.commit_transaction() # Set a stable timestamp so that first update value is durable. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) # Update all values with value 222 i.e. second update value. self.assertEquals(cursor.reset(), 0) @@ -133,12 +129,12 @@ class test_durable_ts01(wttest.WiredTigerTestCase): self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(200)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(200)) # Commit timestamp is earlier to stable timestamp but durable timestamp # is later than stable timestamp. Hence second update value is not durable. - session.timestamp_transaction('commit_timestamp=' + timestamp_str(240)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(300)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(240)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(300)) session.commit_transaction() # Checkpoint so that first update value will be visible and durable, @@ -159,8 +155,8 @@ class test_durable_ts01(wttest.WiredTigerTestCase): self.reopen_conn() session = self.conn.open_session(self.session_config) cursor = session.open_cursor(uri, None) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(250)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(111)) diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts02.py b/src/third_party/wiredtiger/test/suite/test_durable_ts02.py index b71743c232c..88dea1291a2 100644 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts02.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts02.py @@ -31,9 +31,6 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' %t - # test_durable_ts03.py # Checking visibility and durability of updates with durable_timestamp class test_durable_ts03(wttest.WiredTigerTestCase): @@ -75,7 +72,7 @@ class test_durable_ts03(wttest.WiredTigerTestCase): cursor = session.open_cursor(uri, None) # Set stable timestamp to checkpoint initial data set. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100)) self.session.checkpoint() ''' @@ -90,15 +87,15 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) msg = "/is less than the commit timestamp/" # Check for error when commit timestamp > durable timestamp. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: session.commit_transaction('commit_timestamp=' +\ - timestamp_str(200) + ',durable_timestamp=' + timestamp_str(180)), msg) + self.timestamp_str(200) + ',durable_timestamp=' + self.timestamp_str(180)), msg) # Set a stable timestamp so that first update value is durable. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) # Scenario: 2 # Check to see durable timestamp < stable timestamp, returns error. @@ -111,13 +108,13 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) - session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) msg = "/is less than the stable timestamp/" # Check that error is returned when durable timestamp < stable timestamp. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: session.commit_transaction('commit_timestamp=' +\ - timestamp_str(200) + ',durable_timestamp=' + timestamp_str(240)), msg) + self.timestamp_str(200) + ',durable_timestamp=' + self.timestamp_str(240)), msg) ''' if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py index 298c607904d..deea2f3dfc5 100755 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py @@ -29,9 +29,6 @@ from helper import copy_wiredtiger_home import wiredtiger, wttest -def timestamp_str(t): - return '%x' %t - # test_durable_ts03.py # Check that the checkpoint honors the durable timestamp of updates. class test_durable_ts03(wttest.WiredTigerTestCase): @@ -48,8 +45,8 @@ class test_durable_ts03(wttest.WiredTigerTestCase): valueC = b"ccccc" * 100 # Start with setting a stable and oldest timestamp. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1) + \ - ',oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1) + \ + ',oldest_timestamp=' + self.timestamp_str(1)) # Load the data into the table. session = self.conn.open_session(self.session_config) @@ -57,12 +54,12 @@ class test_durable_ts03(wttest.WiredTigerTestCase): for i in range(0, nrows): session.begin_transaction() cursor[i] = valueA - session.commit_transaction('commit_timestamp=' + timestamp_str(50)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) cursor.close() # Set the stable and the oldest timestamp to checkpoint initial data. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100) + \ - ',oldest_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100) + \ + ',oldest_timestamp=' + self.timestamp_str(100)) self.session.checkpoint() # Update all the values within transaction. Commit the transaction with @@ -71,9 +68,9 @@ class test_durable_ts03(wttest.WiredTigerTestCase): for i in range(0, nrows): session.begin_transaction() cursor[i] = valueB - session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(220)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(200)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(220)) session.commit_transaction() # Check the checkpoint wrote only the durable updates. @@ -83,14 +80,14 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.assertEqual(value, valueA) self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(150)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(150)) for key, value in cursor: self.assertEqual(value, valueA) session.commit_transaction() # Read the updated data to confirm that it is visible. self.assertEquals(cursor.reset(), 0) - session.begin_transaction('read_timestamp=' + timestamp_str(210)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(210)) for key, value in cursor: self.assertEqual(value, valueB) session.commit_transaction() @@ -102,8 +99,8 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.reopen_conn() session = self.conn.open_session(self.session_config) cursor = session.open_cursor(uri, None) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(210) + \ - ',oldest_timestamp=' + timestamp_str(210)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(210) + \ + ',oldest_timestamp=' + self.timestamp_str(210)) for key, value in cursor: self.assertEqual(value, valueA) @@ -111,12 +108,12 @@ class test_durable_ts03(wttest.WiredTigerTestCase): for i in range(0, nrows): session.begin_transaction() cursor[i] = valueC - session.prepare_transaction('prepare_timestamp=' + timestamp_str(220)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(230)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(240)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(220)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(230)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(240)) session.commit_transaction() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) self.session.checkpoint() cursor.close() session.close() @@ -124,8 +121,8 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.reopen_conn() session = self.conn.open_session(self.session_config) cursor = session.open_cursor(uri, None) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250) + \ - ',oldest_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250) + \ + ',oldest_timestamp=' + self.timestamp_str(250)) for key, value in cursor: self.assertEqual(value, valueC) diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt08.py b/src/third_party/wiredtiger/test/suite/test_encrypt08.py index e15dd851908..5e4e065d3f3 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt08.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt08.py @@ -84,11 +84,10 @@ class test_encrypt08(wttest.WiredTigerTestCase): def test_encrypt(self): sysconfig = 'encryption=(name=sodium,{0}),'.format(self.sys_encrypt) - with self.expectedStdoutPattern('Failed wiredtiger_open'): - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: - self.reopen_conn(config = sysconfig), - self.msg) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: + self.reopen_conn(config = sysconfig), + self.msg) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_gc01.py b/src/third_party/wiredtiger/test/suite/test_gc01.py index 950c1e4ce54..8cec5461881 100755 --- a/src/third_party/wiredtiger/test/suite/test_gc01.py +++ b/src/third_party/wiredtiger/test/suite/test_gc01.py @@ -36,11 +36,7 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet from wiredtiger import stat -def timestamp_str(t): - return '%x' % t - # test_gc01.py - # Shared base class used by gc tests. class test_gc_base(wttest.WiredTigerTestCase): @@ -51,7 +47,7 @@ class test_gc_base(wttest.WiredTigerTestCase): for i in range(0, nrows): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def large_modifies(self, uri, value, ds, location, nbytes, nrows, commit_ts): @@ -63,12 +59,12 @@ class test_gc_base(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify(value, location, nbytes)] self.assertEqual(cursor.modify(mods), 0) - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, check_value, uri, nrows, read_ts): session = self.session - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -100,8 +96,8 @@ class test_gc01(test_gc_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 bigvalue2 = "ddddd" * 100 @@ -119,8 +115,8 @@ class test_gc01(test_gc_base): self.check(bigvalue, uri, nrows, 10) # Pin oldest and stable to timestamp 100. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100) + - ',stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100) + + ',stable_timestamp=' + self.timestamp_str(100)) # Checkpoint to ensure that the history store is cleaned. self.session.checkpoint() @@ -152,8 +148,8 @@ class test_gc01(test_gc_base): self.check(bigvalue2, uri, nrows, 100) # Pin oldest and stable to timestamp 200. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(200) + - ',stable_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(200) + + ',stable_timestamp=' + self.timestamp_str(200)) # Checkpoint to ensure that the history store is cleaned. self.session.checkpoint() @@ -185,8 +181,8 @@ class test_gc01(test_gc_base): self.check(bigvalue, uri, nrows, 200) # Pin oldest and stable to timestamp 300. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(300) + - ',stable_timestamp=' + timestamp_str(300)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(300) + + ',stable_timestamp=' + self.timestamp_str(300)) # Checkpoint to ensure that the history store is cleaned. self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_gc02.py b/src/third_party/wiredtiger/test/suite/test_gc02.py index b42d3e23a47..7af18d02186 100755 --- a/src/third_party/wiredtiger/test/suite/test_gc02.py +++ b/src/third_party/wiredtiger/test/suite/test_gc02.py @@ -30,9 +30,6 @@ from test_gc01 import test_gc_base from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - # test_gc02.py # Test that checkpoint cleans the obsolete history store internal pages. class test_gc02(test_gc_base): @@ -49,8 +46,8 @@ class test_gc02(test_gc_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 bigvalue2 = "ddddd" * 100 @@ -76,8 +73,8 @@ class test_gc02(test_gc_base): c.close() # Pin oldest and stable to timestamp 100. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100) + - ',stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100) + + ',stable_timestamp=' + self.timestamp_str(100)) # Check that the new updates are only seen after the update timestamp. self.check(bigvalue2, uri, nrows, 100) @@ -114,8 +111,8 @@ class test_gc02(test_gc_base): self.check(bigvalue, uri, nrows, 200) # Pin oldest and stable to timestamp 200. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(200) + - ',stable_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(200) + + ',stable_timestamp=' + self.timestamp_str(200)) # Checkpoint to ensure that the history store is cleaned. self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_gc03.py b/src/third_party/wiredtiger/test/suite/test_gc03.py index c605a07963e..386a7bea801 100755 --- a/src/third_party/wiredtiger/test/suite/test_gc03.py +++ b/src/third_party/wiredtiger/test/suite/test_gc03.py @@ -30,9 +30,6 @@ from test_gc01 import test_gc_base from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - # test_gc03.py # Test that checkpoint cleans the obsolete history store pages that are in-memory. class test_gc03(test_gc_base): @@ -61,8 +58,8 @@ class test_gc03(test_gc_base): ds_extra.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 bigvalue2 = "ddddd" * 100 @@ -84,8 +81,8 @@ class test_gc03(test_gc_base): self.assertGreater(self.get_stat(stat.conn.cc_pages_visited), 0) # Pin oldest and stable to timestamp 100. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100) + - ',stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100) + + ',stable_timestamp=' + self.timestamp_str(100)) # Check that the new updates are only seen after the update timestamp. self.check(bigvalue2, uri, nrows, 100) @@ -113,8 +110,8 @@ class test_gc03(test_gc_base): self.check(bigvalue2, uri, nrows, 100) # Pin oldest and stable to timestamp 200. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(200) + - ',stable_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(200) + + ',stable_timestamp=' + self.timestamp_str(200)) # Update on extra table. self.large_updates(uri_extra, bigvalue, ds_extra, 100, 210) @@ -129,8 +126,8 @@ class test_gc03(test_gc_base): self.check(bigvalue, uri, nrows, 200) # Pin oldest and stable to timestamp 300. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(300) + - ',stable_timestamp=' + timestamp_str(300)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(300) + + ',stable_timestamp=' + self.timestamp_str(300)) self.large_updates(uri_extra, bigvalue, ds_extra, 100, 310) self.large_updates(uri_extra, bigvalue2, ds_extra, 100, 320) diff --git a/src/third_party/wiredtiger/test/suite/test_gc04.py b/src/third_party/wiredtiger/test/suite/test_gc04.py index 1db0cceb24f..4504bd3f75e 100755 --- a/src/third_party/wiredtiger/test/suite/test_gc04.py +++ b/src/third_party/wiredtiger/test/suite/test_gc04.py @@ -30,9 +30,6 @@ from test_gc01 import test_gc_base from wiredtiger import stat from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - # test_gc04.py # Test that checkpoint must not clean the pages that are not obsolete. class test_gc04(test_gc_base): @@ -55,8 +52,8 @@ class test_gc04(test_gc_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 bigvalue2 = "ddddd" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_gc05.py b/src/third_party/wiredtiger/test/suite/test_gc05.py index 1d34ce34dc6..75bf3d253e8 100755 --- a/src/third_party/wiredtiger/test/suite/test_gc05.py +++ b/src/third_party/wiredtiger/test/suite/test_gc05.py @@ -29,9 +29,6 @@ from test_gc01 import test_gc_base from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - # test_gc05.py # Verify a locked checkpoint is not removed during garbage collection. class test_gc05(test_gc_base): @@ -52,8 +49,8 @@ class test_gc05(test_gc_base): ds.populate() # Set the oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) # Insert values with varying timestamps. self.large_updates(uri, value_x, ds, nrows, 20) @@ -70,8 +67,8 @@ class test_gc05(test_gc_base): ckpt_cursor = self.session.open_cursor(uri, None, "checkpoint=checkpoint_one") # Move the oldest and stable timestamps to 40. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(40) + - ',stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(40) + + ',stable_timestamp=' + self.timestamp_str(40)) # Insert values with varying timestamps. self.large_updates(uri, value_z, ds, nrows, 50) @@ -79,8 +76,8 @@ class test_gc05(test_gc_base): self.large_updates(uri, value_x, ds, nrows, 70) # Move the oldest and stable timestamps to 70. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(70) + - ',stable_timestamp=' + timestamp_str(70)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(70) + + ',stable_timestamp=' + self.timestamp_str(70)) # Perform a checkpoint. self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_hs01.py b/src/third_party/wiredtiger/test/suite/test_hs01.py index 90a7be1cd0b..52c68cb699d 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs01.py +++ b/src/third_party/wiredtiger/test/suite/test_hs01.py @@ -28,72 +28,70 @@ from helper import copy_wiredtiger_home import wiredtiger, wttest +from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs01.py -# Smoke tests to ensure history store tables are working. +# Test that update and modify operations are durable across crash and recovery. +# Additionally test that checkpoint inserts content into the history store. class test_hs01(wttest.WiredTigerTestCase): - # Force a small cache. - conn_config = 'cache_size=50MB' + conn_config = 'cache_size=200MB,statistics=(all)' session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), - ('string', dict(key_format='S')) + ('row_integer', dict(key_format='i')), + ('row_string', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) + def get_stat(self, stat): + stat_cursor = self.session.open_cursor('statistics:') + val = stat_cursor[stat][2] + stat_cursor.close() + return val + def large_updates(self, session, uri, value, ds, nrows, timestamp=False): - # Update a large number of records, we'll hang if the history store table - # isn't doing its thing. cursor = session.open_cursor(uri) - for i in range(1, 10000): + for i in range(1, nrows): if timestamp == True: session.begin_transaction() - cursor.set_key(ds.key(nrows + i)) + cursor.set_key(ds.key(i)) cursor.set_value(value) self.assertEqual(cursor.update(), 0) if timestamp == True: - session.commit_transaction('commit_timestamp=' + timestamp_str(i + 1)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(i + 1)) cursor.close() def large_modifies(self, session, uri, offset, ds, nrows, timestamp=False): - # Modify a large number of records, we'll hang if the history store table - # isn't doing its thing. cursor = session.open_cursor(uri) - for i in range(1, 10000): - if timestamp == True: - session.begin_transaction() - cursor.set_key(ds.key(nrows + i)) + for i in range(1, nrows): + # Unlike inserts and updates, modify operations do not implicitly start/commit a transaction. + # Hence, we begin/commit transaction manually. + session.begin_transaction() + cursor.set_key(ds.key(i)) mods = [] mod = wiredtiger.Modify('A', offset, 1) mods.append(mod) - self.assertEqual(cursor.modify(mods), 0) + if timestamp == True: - session.commit_transaction('commit_timestamp=' + timestamp_str(i + 1)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(i + 1)) + else: + session.commit_transaction() cursor.close() - def durable_check(self, check_value, uri, ds, nrows): - # Checkpoint and backup so as to simulate recovery - self.session.checkpoint() + def durable_check(self, check_value, uri, ds): + # Simulating recovery. newdir = "BACKUP" copy_wiredtiger_home(self, '.', newdir, True) - conn = self.setUpConnectionOpen(newdir) session = self.setUpSessionOpen(conn) cursor = session.open_cursor(uri, None) - # Skip the initial rows, which were not updated - for i in range(0, nrows+1): - self.assertEqual(cursor.next(), 0) - if check_value != cursor.get_value(): - session.breakpoint() + + cursor.next() self.assertTrue(check_value == cursor.get_value(), - "for key " + str(i) + ", expected " + str(check_value) + + "for key " + str(1) + ", expected " + str(check_value) + ", got " + str(cursor.get_value())) cursor.close() session.close() @@ -102,15 +100,15 @@ class test_hs01(wttest.WiredTigerTestCase): def test_hs(self): # Create a small table. uri = "table:test_hs01" - nrows = 100 - ds = SimpleDataSet(self, uri, nrows, key_format=self.key_format, value_format='u') + ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format='u') ds.populate() - bigvalue = b"aaaaa" * 100 - # Initially load huge data. + # Initially insert a lot of data. + nrows = 10000 + bigvalue = b"aaaaa" * 100 cursor = self.session.open_cursor(uri) - for i in range(1, 10000): - cursor.set_key(ds.key(nrows + i)) + for i in range(1, nrows): + cursor.set_key(ds.key(i)) cursor.set_value(bigvalue) self.assertEqual(cursor.insert(), 0) cursor.close() @@ -119,11 +117,19 @@ class test_hs01(wttest.WiredTigerTestCase): # Scenario: 1 # Check to see if the history store is working with the old reader. bigvalue2 = b"ccccc" * 100 + # Open session 2. session2 = self.conn.open_session() session2.begin_transaction('isolation=snapshot') + # Large updates with session 1. self.large_updates(self.session, uri, bigvalue2, ds, nrows) - # Check to see the value after recovery. - self.durable_check(bigvalue2, uri, ds, nrows) + + # Checkpoint and then assert that the (nrows-1) insertions were moved to history store from data store. + self.session.checkpoint() + hs_writes = self.get_stat(stat.conn.cache_hs_insert) + self.assertEqual(hs_writes, nrows-1) + + # Check to see the latest updated value after recovery. + self.durable_check(bigvalue2, uri, ds) session2.rollback_transaction() session2.close() @@ -131,34 +137,48 @@ class test_hs01(wttest.WiredTigerTestCase): # Check to see the history store working with modify operations. bigvalue3 = b"ccccc" * 100 bigvalue3 = b'AA' + bigvalue3[2:] + # Open session 2. session2 = self.conn.open_session() session2.begin_transaction('isolation=snapshot') - # Apply two modify operations - replacing the first two items with 'A'. - self.session.begin_transaction() + # Apply two modify operations (session1)- replacing the first two items with 'A'. self.large_modifies(self.session, uri, 0, ds, nrows) self.large_modifies(self.session, uri, 1, ds, nrows) - self.session.commit_transaction() - # Check to see the value after recovery. - self.durable_check(bigvalue3, uri, ds, nrows) + + # Checkpoint and then assert if updates (nrows-1) and first large modifies (nrows-1) were moved to history store. + self.session.checkpoint() + hs_writes = self.get_stat(stat.conn.cache_hs_insert) + # The updates in data store: nrows-1 + # The first modifies in cache: nrows-1 + # The stats was already set at: nrows-1 (previous hs stats) + # Total: (nrows-1)*3 + self.assertEqual(hs_writes, (nrows-1) * 3) + + # Check to see the modified value after recovery. + self.durable_check(bigvalue3, uri, ds) session2.rollback_transaction() session2.close() - # FIXME-WT-7120: Rollback to stable support for column store is not implemented, and it - # fails only when it is used with timestamps. - if self.key_format == 'r': - return - # Scenario: 3 # Check to see if the history store is working with the old timestamp. bigvalue4 = b"ddddd" * 100 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) self.large_updates(self.session, uri, bigvalue4, ds, nrows, timestamp=True) - # Check to see data can be see only till the stable_timestamp - self.durable_check(bigvalue3, uri, ds, nrows) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(i + 1)) + self.session.checkpoint() + # Check if the (nrows-1) modifications were moved to history store from data store. + # The stats was already set at: (nrows-1)*3 (previous hs stats) + # Total: (nrows-1)*4 + hs_writes = self.get_stat(stat.conn.cache_hs_insert) + self.assertEqual(hs_writes, (nrows-1) * 4) + + # Check to see data can be see only till the stable_timestamp. + self.durable_check(bigvalue3, uri, ds) + + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(i + 1)) + # No need to check history store stats here as nothing will be moved. + self.session.checkpoint() # Check that the latest data can be seen. - self.durable_check(bigvalue4, uri, ds, nrows) + self.durable_check(bigvalue4, uri, ds) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_hs02.py b/src/third_party/wiredtiger/test/suite/test_hs02.py index 6f92f422161..42cd3b327ce 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs02.py +++ b/src/third_party/wiredtiger/test/suite/test_hs02.py @@ -31,9 +31,6 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs02.py # Test that truncate with history store entries and timestamps gives expected results. class test_hs02(wttest.WiredTigerTestCase): @@ -42,9 +39,8 @@ class test_hs02(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ - ('string', dict(key_format='S')), - # FIXME-WT-7120: Uncomment the column store scenario when rollback to stable support is complete. - # ('column', dict(key_format='r')) + ('string-row', dict(key_format='S')), + ('column', dict(key_format='r')) ] scenarios = make_scenarios(key_format_values) @@ -55,12 +51,12 @@ class test_hs02(wttest.WiredTigerTestCase): for i in range(1, nrows + 1): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, check_value, uri, nrows, read_ts): session = self.session - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -84,8 +80,8 @@ class test_hs02(wttest.WiredTigerTestCase): ds2.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 self.large_updates(uri, bigvalue, ds, nrows // 3, 1) @@ -110,7 +106,7 @@ class test_hs02(wttest.WiredTigerTestCase): end.set_key(ds.key(nrows // 2)) self.session.truncate(None, None, end) end.close() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(200)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(200)) # Check that the truncate is visible after commit self.check(bigvalue2, uri, nrows // 2, 200) diff --git a/src/third_party/wiredtiger/test/suite/test_hs03.py b/src/third_party/wiredtiger/test/suite/test_hs03.py index 6980859ced6..54a54bf1f0f 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs03.py +++ b/src/third_party/wiredtiger/test/suite/test_hs03.py @@ -32,9 +32,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs03.py # Ensure checkpoints don't read too unnecessary history store entries. class test_hs03(wttest.WiredTigerTestCase): @@ -43,8 +40,8 @@ class test_hs03(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), - ('string', dict(key_format='S')) + ('integer-row', dict(key_format='i')), + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -61,7 +58,7 @@ class test_hs03(wttest.WiredTigerTestCase): for i in range(nrows + 1, nrows + nops + 1): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(i)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(i)) cursor.close() def test_checkpoint_hs_reads(self): @@ -81,7 +78,7 @@ class test_hs03(wttest.WiredTigerTestCase): # Check to see the history store working with old timestamp. bigvalue2 = b"ddddd" * 100 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) hs_writes_start = self.get_stat(stat.conn.cache_write_hs) self.large_updates(self.session, uri, bigvalue2, ds, nrows, 10000) @@ -91,7 +88,7 @@ class test_hs03(wttest.WiredTigerTestCase): self.assertGreaterEqual(hs_writes, 0) for ts in range(2, 4): - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(ts)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(ts)) # Now just update one record and checkpoint again. self.large_updates(self.session, uri, bigvalue2, ds, nrows, 1) diff --git a/src/third_party/wiredtiger/test/suite/test_hs05.py b/src/third_party/wiredtiger/test/suite/test_hs05.py index 5a66d04f546..6f869e83aaa 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs05.py +++ b/src/third_party/wiredtiger/test/suite/test_hs05.py @@ -32,9 +32,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs05.py # Verify hs_score reflects cache pressure due to history # even if we're not yet actively pushing into the history store file. @@ -47,8 +44,8 @@ class test_hs05(wttest.WiredTigerTestCase): stable = 1 key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), - ('string', dict(key_format='S')) + ('integer-row', dict(key_format='i')), + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -66,7 +63,7 @@ class test_hs05(wttest.WiredTigerTestCase): for i in range(nrows + 1, nrows + nops + 1): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(self.stable + i)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(self.stable + i)) cursor.close() score_end = self.get_stat(stat.conn.cache_hs_score) score_diff = score_end - score_start @@ -93,7 +90,7 @@ class test_hs05(wttest.WiredTigerTestCase): self.session.checkpoint() # Pin the oldest timestamp so that all history has to stay. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) # Loop a couple times, partly filling the cache but not # overfilling it to see the history store score value change # even if the history store is not yet in use. @@ -103,7 +100,7 @@ class test_hs05(wttest.WiredTigerTestCase): loop_start = self.get_stat(stat.conn.cache_hs_score) for i in range(1, 9): bigvalue2 = valstr[i].encode() * 50 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(self.stable)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(self.stable)) entries_start = self.get_stat(stat.conn.cache_hs_insert) score_start = self.get_stat(stat.conn.cache_hs_score) self.pr("Update iteration: " + str(i) + " Value: " + str(bigvalue2)) @@ -127,14 +124,14 @@ class test_hs05(wttest.WiredTigerTestCase): # By moving the oldest after updating we should see the score drop # to zero. score_start = loop_end - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(self.stable)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(self.stable)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(self.stable)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(self.stable)) for i in range(9, 11): bigvalue2 = valstr[i].encode() * 50 self.pr("Update iteration with oldest: " + str(i) + " Value: " + str(bigvalue2)) self.large_updates(self.session, uri, bigvalue2, ds, nrows, nrows) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(self.stable)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(self.stable)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(self.stable)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(self.stable)) self.stable += nrows score_end = self.get_stat(stat.conn.cache_hs_score) self.assertLess(score_end, score_start) diff --git a/src/third_party/wiredtiger/test/suite/test_hs06.py b/src/third_party/wiredtiger/test/suite/test_hs06.py index a7747e281e4..1c2f18f09f2 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs06.py +++ b/src/third_party/wiredtiger/test/suite/test_hs06.py @@ -30,9 +30,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs06.py # Verify that triggering history store usage does not cause a spike in memory usage # to form an update chain from the history store contents. @@ -48,10 +45,11 @@ class test_hs06(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), - ('string', dict(key_format='S')) + ('integer-row', dict(key_format='i')), + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) + nrows = 2000 def get_stat(self, stat): stat_cursor = self.session.open_cursor('statistics:') @@ -77,21 +75,21 @@ class test_hs06(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Load 1Mb of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load another 1Mb of data with a later timestamp. self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Write a version of the data to disk. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(2)) self.session.checkpoint() # Check the checkpoint wrote the expected values. @@ -102,7 +100,7 @@ class test_hs06(wttest.WiredTigerTestCase): # # cursor2 = self.session.open_cursor(uri, None, 'checkpoint=WiredTigerCheckpoint') cursor2 = self.session.open_cursor(uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) for key, value in cursor2: self.assertEqual(value, value1) self.session.commit_transaction() @@ -113,8 +111,8 @@ class test_hs06(wttest.WiredTigerTestCase): # Whenever we request something out of cache of timestamp 2, we should # be reading it straight from the history store without initialising a full # update chain of every version of the data. - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) - for i in range(1, 2000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) + for i in range(1, self.nrows): self.assertEqual(cursor[self.create_key(i)], value1) self.session.rollback_transaction() @@ -143,34 +141,34 @@ class test_hs06(wttest.WiredTigerTestCase): value2 = 'd' * 500 # Load 1Mb of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load a slight modification with a later timestamp. self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor.set_key(self.create_key(i)) mods = [wiredtiger.Modify('B', 100, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # And another. self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor.set_key(self.create_key(i)) mods = [wiredtiger.Modify('C', 200, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Now write something completely different. self.session.begin_transaction() - for i in range(1, 2000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Now the latest version will get written to the data file. self.session.checkpoint() @@ -187,8 +185,8 @@ class test_hs06(wttest.WiredTigerTestCase): # t4: full update in las # t3: (reverse delta in las) <= We're querying for t4 so we begin here. # t2: value2 (full update in las) - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) - for i in range(1, 2000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) + for i in range(1, self.nrows): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() @@ -203,17 +201,12 @@ class test_hs06(wttest.WiredTigerTestCase): # t4: full update in las <= We're querying for t4 and we return. # t3: (reverse delta in las) # t2: value2 (full update in las) - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) - for i in range(1, 2000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) + for i in range(1, self.nrows): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() def test_hs_prepare_reads(self): - # FIXME-WT-6061: Prepare reads currently not supported with columnar store. - # Remove this once prepare reads is supported. - if self.key_format == 'r': - return - # Create a small table. uri = "table:test_hs06" create_params = 'key_format={},value_format=S'.format(self.key_format) @@ -222,12 +215,12 @@ class test_hs06(wttest.WiredTigerTestCase): value1 = 'a' * 500 value2 = 'b' * 500 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - for i in range(1, 2000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load prepared data and leave it in a prepared state. prepare_session = self.conn.open_session(self.session_config) @@ -236,20 +229,20 @@ class test_hs06(wttest.WiredTigerTestCase): for i in range(1, 11): prepare_cursor[self.create_key(i)] = value2 prepare_session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(3)) + 'prepare_timestamp=' + self.timestamp_str(3)) # Write some more to cause eviction of the prepared data. - for i in range(11, 2000): + for i in range(11, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.checkpoint() # Try to read every key of the prepared data again. # Ensure that we read the history store to find the prepared update and # return a prepare conflict as appropriate. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): cursor.set_key(self.create_key(i)) self.assertRaisesException( @@ -259,9 +252,9 @@ class test_hs06(wttest.WiredTigerTestCase): self.session.rollback_transaction() prepare_session.commit_transaction( - 'commit_timestamp=' + timestamp_str(5) + ',durable_timestamp=' + timestamp_str(6)) + 'commit_timestamp=' + self.timestamp_str(5) + ',durable_timestamp=' + self.timestamp_str(6)) - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) for i in range(1, 11): self.assertEquals(value2, cursor[self.create_key(i)]) self.session.rollback_transaction() @@ -278,12 +271,12 @@ class test_hs06(wttest.WiredTigerTestCase): value4 = 'd' * 500 # Load 1Mb of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - for i in range(1, 2000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Do two different updates to the same key with the same timestamp. # We want to make sure that the second value is the one that is visible even after eviction. @@ -291,16 +284,16 @@ class test_hs06(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor[self.create_key(i)] = value2 cursor[self.create_key(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Write a newer value on top. - for i in range(1, 2000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value4 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Ensure that we see the last of the two updates that got applied. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): self.assertEquals(cursor[self.create_key(i)], value3) self.session.rollback_transaction() @@ -315,12 +308,12 @@ class test_hs06(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Load 1Mb of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - for i in range(1, 2000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Apply three sets of modifies. # They specifically need to be in separate modify calls. @@ -330,7 +323,7 @@ class test_hs06(wttest.WiredTigerTestCase): self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) expected = list(value1) expected[100] = 'B' @@ -339,13 +332,13 @@ class test_hs06(wttest.WiredTigerTestCase): expected = str().join(expected) # Write a newer value on top. - for i in range(1, 2000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Go back and read. We should get the initial value with the 3 modifies applied on top. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() @@ -361,25 +354,25 @@ class test_hs06(wttest.WiredTigerTestCase): # Load 5Mb of data. self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) for i in range(1, 10000): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Apply three sets of modifies. for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Since the stable timestamp is still at 1, there will be no birthmark record. # History store instantiation should choose this update since it is the most recent. @@ -388,7 +381,7 @@ class test_hs06(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Make a bunch of updates to another table to flush everything out of cache. uri2 = 'table:test_hs06_extra' @@ -397,7 +390,7 @@ class test_hs06(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor2[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(6)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(6)) expected = list(value1) expected[100] = 'B' @@ -406,7 +399,7 @@ class test_hs06(wttest.WiredTigerTestCase): expected = str().join(expected) # Go back and read. We should get the initial value with the 3 modifies applied on top. - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) for i in range(1, 11): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() @@ -422,7 +415,7 @@ class test_hs06(wttest.WiredTigerTestCase): # Load 5Mb of data. self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) # The base update is at timestamp 1. # When we history store evict these pages, the base update is the only thing behind @@ -431,26 +424,26 @@ class test_hs06(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) # Apply three sets of modifies. for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Make a bunch of updates to another table to flush everything out of cache. uri2 = 'table:test_hs06_extra' @@ -459,7 +452,7 @@ class test_hs06(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor2[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) expected = list(value1) expected[100] = 'B' @@ -468,7 +461,7 @@ class test_hs06(wttest.WiredTigerTestCase): expected = str().join(expected) # Go back and read. - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) for i in range(1, 11): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() @@ -483,44 +476,44 @@ class test_hs06(wttest.WiredTigerTestCase): value2 = 'b' * 500 self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) # Base update. for i in range(1, 10000): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Apply three sets of modifies. for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # This is the one we want to be selected by the checkpoint. for i in range(1, 11): self.session.begin_transaction() cursor.set_key(self.create_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Apply another update and evict the pages with the modifies out of cache. for i in range(1, 10000): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(6)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(6)) # Checkpoint such that the modifies will be selected. When we grab it from the history # store, we'll need to unflatten it before using it for reconciliation. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) self.session.checkpoint() expected = list(value1) @@ -530,7 +523,7 @@ class test_hs06(wttest.WiredTigerTestCase): expected = str().join(expected) # Check that the correct value is visible after checkpoint. - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) for i in range(1, 11): self.assertEqual(cursor[self.create_key(i)], expected) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs07.py b/src/third_party/wiredtiger/test/suite/test_hs07.py index 58d44374e42..7e2562e5420 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs07.py +++ b/src/third_party/wiredtiger/test/suite/test_hs07.py @@ -32,9 +32,6 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs07.py # Test that the history store sweep cleans the obsolete history store entries and gives expected results. class test_hs07(wttest.WiredTigerTestCase): @@ -45,7 +42,7 @@ class test_hs07(wttest.WiredTigerTestCase): key_format_values = ( ('column', dict(key_format='r')), - ('int', dict(key_format='i')) + ('integer-row', dict(key_format='i')) ) scenarios = make_scenarios(key_format_values) @@ -56,12 +53,12 @@ class test_hs07(wttest.WiredTigerTestCase): for i in range(1, nrows + 1): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, check_value, uri, nrows, read_ts): session = self.session - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -85,8 +82,8 @@ class test_hs07(wttest.WiredTigerTestCase): ds2.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) bigvalue = "aaaaa" * 100 bigvalue2 = "ddddd" * 100 @@ -102,8 +99,8 @@ class test_hs07(wttest.WiredTigerTestCase): self.check(bigvalue, uri, nrows, 100) # Pin oldest and stable to timestamp 100. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100) + - ',stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100) + + ',stable_timestamp=' + self.timestamp_str(100)) # Sleep here to let that sweep server to trigger cleanup of obsolete entries. time.sleep(10) @@ -118,7 +115,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('A', 10, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(110)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(110)) # Load a slight modification with a later timestamp. self.session.begin_transaction() @@ -126,7 +123,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('B', 20, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(120)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(120)) # Load a slight modification with a later timestamp. self.session.begin_transaction() @@ -134,7 +131,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('C', 30, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(130)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(130)) cursor.close() # Second set of update operations with increased timestamp @@ -147,8 +144,8 @@ class test_hs07(wttest.WiredTigerTestCase): self.check(bigvalue2, uri, nrows, 200) # Pin oldest and stable to timestamp 300. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(200) + - ',stable_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(200) + + ',stable_timestamp=' + self.timestamp_str(200)) # Sleep here to let that sweep server to trigger cleanup of obsolete entries. time.sleep(10) @@ -163,7 +160,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('A', 10, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(210)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(210)) # Load a slight modification with a later timestamp. self.session.begin_transaction() @@ -171,7 +168,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('B', 20, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(220)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(220)) # Load a slight modification with a later timestamp. self.session.begin_transaction() @@ -179,7 +176,7 @@ class test_hs07(wttest.WiredTigerTestCase): cursor.set_key(i) mods = [wiredtiger.Modify('C', 30, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(230)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(230)) cursor.close() # Third set of update operations with increased timestamp @@ -192,8 +189,8 @@ class test_hs07(wttest.WiredTigerTestCase): self.check(bigvalue, uri, nrows, 300) # Pin oldest and stable to timestamp 400. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(300) + - ',stable_timestamp=' + timestamp_str(300)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(300) + + ',stable_timestamp=' + self.timestamp_str(300)) # Sleep here to let that sweep server to trigger cleanup of obsolete entries. time.sleep(10) diff --git a/src/third_party/wiredtiger/test/suite/test_hs08.py b/src/third_party/wiredtiger/test/suite/test_hs08.py index 0cb26149687..13769f2f19f 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs08.py +++ b/src/third_party/wiredtiger/test/suite/test_hs08.py @@ -30,9 +30,6 @@ import wiredtiger, wttest, time from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs08.py # Verify modify insert into history store logic. class test_hs08(wttest.WiredTigerTestCase): @@ -40,7 +37,7 @@ class test_hs08(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), + ('integer-row', dict(key_format='i')), ] scenarios = make_scenarios(key_format_values) @@ -57,27 +54,27 @@ class test_hs08(wttest.WiredTigerTestCase): self.session.create(uri, create_params) # Insert a full value. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) self.session.begin_transaction() cursor[1] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Insert 3 modifies in separate transactions. self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('A', 1000, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 1001, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 1002, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Call checkpoint. self.session.checkpoint('use_timestamp=true') @@ -89,15 +86,15 @@ class test_hs08(wttest.WiredTigerTestCase): self.assertEqual(squashed_write, 0) # Validate that we see the correct value at each of the timestamps. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) self.assertEqual(cursor[1], value1 + 'A') self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) self.assertEqual(cursor[1], value1 + 'AB') self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) self.assertEqual(cursor[1], value1 + 'ABC') self.session.commit_transaction() @@ -107,12 +104,12 @@ class test_hs08(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 1000, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(7)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(7)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('E', 1001, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(8)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(8)) # Call checkpoint again. self.session.checkpoint('use_timestamp=true') @@ -126,11 +123,11 @@ class test_hs08(wttest.WiredTigerTestCase): # Validate that we see the expected value on the modifies, this # scenario tests the logic that will retrieve a full value for # a modify previously inserted into the history store. - self.session.begin_transaction('read_timestamp=' + timestamp_str(7)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(7)) self.assertEqual(cursor[1], value1 + 'DBC') self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(8)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(8)) self.assertEqual(cursor[1], value1 + 'DEC') self.session.commit_transaction() @@ -140,7 +137,7 @@ class test_hs08(wttest.WiredTigerTestCase): self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('H', 1004, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(9)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(9)) # Call checkpoint again. self.session.checkpoint('use_timestamp=true') @@ -157,13 +154,13 @@ class test_hs08(wttest.WiredTigerTestCase): cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0) self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(11)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(11)) # Call checkpoint again. self.session.checkpoint('use_timestamp=true') @@ -179,17 +176,17 @@ class test_hs08(wttest.WiredTigerTestCase): # modify to guarantee we squash zero modifies. self.session.begin_transaction() cursor.set_key(1) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(12)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(12)) self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(13)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(13)) self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0) self.session.commit_transaction() self.session.begin_transaction() cursor.set_key(1) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(14)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(14)) self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(15)) self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0) self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs09.py b/src/third_party/wiredtiger/test/suite/test_hs09.py index 4217da2b02d..1b7993239b2 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs09.py +++ b/src/third_party/wiredtiger/test/suite/test_hs09.py @@ -34,23 +34,21 @@ import wiredtiger, wttest from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs09.py # Verify that we write the newest committed version to data store and the # second newest committed version to history store. class test_hs09(wttest.WiredTigerTestCase): # Force a small cache. - conn_config = 'cache_size=20MB,statistics=(fast)' + conn_config = 'cache_size=20MB' session_config = 'isolation=snapshot' uri = "table:test_hs09" key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), - ('string', dict(key_format='S')), + ('integer-row', dict(key_format='i')), + ('string-row', dict(key_format='S')), ] scenarios = make_scenarios(key_format_values) + nrows = 1000 def create_key(self, i): if self.key_format == 'S': @@ -101,18 +99,18 @@ class test_hs09(wttest.WiredTigerTestCase): value3 = 'c' * 500 # Load 500KB of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load another 500KB of data with a later timestamp. self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Uncommitted changes. self.session.begin_transaction() @@ -122,11 +120,6 @@ class test_hs09(wttest.WiredTigerTestCase): self.check_ckpt_hs(value2, value1, 2, 3) def test_prepared_updates_not_written_to_hs(self): - # FIXME-WT-6061: Prepare reads currently not supported with columnar store. - # Remove this once prepare reads is supported. - if self.key_format == 'r': - return - # Create a small table. create_params = 'key_format={},value_format=S'.format(self.key_format) self.session.create(self.uri, create_params) @@ -136,30 +129,30 @@ class test_hs09(wttest.WiredTigerTestCase): value3 = 'c' * 500 # Load 1MB of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) self.session.begin_transaction() for i in range(1, 2000): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load another 1MB of data with a later timestamp. self.session.begin_transaction() for i in range(1, 2000): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Prepare some updates. self.session.begin_transaction() for i in range(1, 11): cursor[self.create_key(i)] = value3 - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(4)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(4)) # We can expect prepared values to show up in data store if the eviction runs between now # and the time when we open a cursor on the user table. self.check_ckpt_hs(value2, value1, 2, 3, True) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5) + - ',durable_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5) + + ',durable_timestamp=' + self.timestamp_str(5)) def test_write_newest_version_to_data_store(self): # Create a small table. @@ -170,18 +163,18 @@ class test_hs09(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Load 500KB of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load another 500KB of data with a later timestamp. self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) self.check_ckpt_hs(value2, value1, 2, 3) @@ -194,26 +187,26 @@ class test_hs09(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Load 500KB of data. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Load another 500KB of data with a later timestamp. self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Delete records. self.session.begin_transaction() - for i in range(1, 1000): + for i in range(1, self.nrows): cursor = self.session.open_cursor(self.uri) cursor.set_key(self.create_key(i)) self.assertEqual(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.check_ckpt_hs(value2, value1, 2, 3) diff --git a/src/third_party/wiredtiger/test/suite/test_hs10.py b/src/third_party/wiredtiger/test/suite/test_hs10.py index 9e3cb4f2699..256ecff4f7b 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs10.py +++ b/src/third_party/wiredtiger/test/suite/test_hs10.py @@ -30,9 +30,6 @@ import wiredtiger, wttest, time from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs10.py # Verify modify read after eviction. class test_hs10(wttest.WiredTigerTestCase): @@ -40,7 +37,7 @@ class test_hs10(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = ( ('column', dict(key_format='r')), - ('int', dict(key_format='i')) + ('integer-row', dict(key_format='i')) ) scenarios = make_scenarios(key_format_values) @@ -61,28 +58,28 @@ class test_hs10(wttest.WiredTigerTestCase): session2.create(uri2, create_params) cursor2 = session2.open_cursor(uri2) # Insert a full value. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) self.session.begin_transaction() cursor[1] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Insert 3 modifies in separate transactions. self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('A', 1000, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 1001, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() cursor.set_key(1) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 1002, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) self.session.checkpoint() # Insert a whole bunch of data into the other table to force wiredtiger to evict data @@ -91,20 +88,20 @@ class test_hs10(wttest.WiredTigerTestCase): cursor2[i] = value2 # Validate that we see the correct value at each of the timestamps. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) cursor.set_key(1) cursor.search() self.assertEqual(cursor[1], value1 + 'A') self.session.commit_transaction() cursor2 = self.session.open_cursor(uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) cursor2.set_key(1) cursor2.search() self.assertEqual(cursor2.get_value(), value1 + 'AB') self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) self.assertEqual(cursor[1], value1 + 'ABC') self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs11.py b/src/third_party/wiredtiger/test/suite/test_hs11.py index 7f0bbfe141a..fd2afc0d977 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs11.py +++ b/src/third_party/wiredtiger/test/suite/test_hs11.py @@ -30,23 +30,22 @@ import wiredtiger, wttest from wtscenario import make_scenarios from wiredtiger import stat -def timestamp_str(t): - return '%x' % t - # test_hs11.py # Ensure that updates without timestamps clear the history store records. class test_hs11(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,statistics=(all)' session_config = 'isolation=snapshot' - key_format_values = ( - ('int', dict(key_format='i')), - ('string', dict(key_format='S')), - ('column', dict(key_format='r')) - ) - scenarios = make_scenarios([ + key_format_values = [ + ('column', dict(key_format='r')), + ('integer-row', dict(key_format='i')), + ('string-row', dict(key_format='S')) + ] + update_type_values = [ ('deletion', dict(update_type='deletion')), - ('update', dict(update_type='update')), - ], key_format_values) + ('update', dict(update_type='update')) + ] + scenarios = make_scenarios(key_format_values, update_type_values) + nrows = 10000 def create_key(self, i): if self.key_format == 'S': @@ -68,19 +67,19 @@ class test_hs11(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Apply a series of updates from timestamps 1-4. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) for ts in range(1, 5): - for i in range(1, 10000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) # Reconcile and flush versions 1-3 to the history store. self.session.checkpoint() # Apply an update without timestamp. - for i in range(1, 10000): + for i in range(1, self.nrows): if i % 2 == 0: if self.update_type == 'deletion': cursor.set_key(self.create_key(i)) @@ -92,20 +91,15 @@ class test_hs11(wttest.WiredTigerTestCase): self.session.checkpoint() # Now apply an update at timestamp 10. - for i in range(1, 10000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) - - # FIXME-WT-7120: Remove for column store until rollback to stable is implemented for column - # store. - if self.key_format == 'r': - return + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Ensure that we blew away history store content. for ts in range(1, 5): - self.session.begin_transaction('read_timestamp=' + timestamp_str(ts)) - for i in range(1, 10000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts)) + for i in range(1, self.nrows): if i % 2 == 0: if self.update_type == 'deletion': cursor.set_key(self.create_key(i)) @@ -129,38 +123,38 @@ class test_hs11(wttest.WiredTigerTestCase): value2 = 'b' * 500 # Apply a series of updates from timestamps 1-4. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) for ts in range(1, 5): - for i in range(1, 10000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) # Reconcile and flush versions 1-3 to the history store. self.session.checkpoint() # Remove the key with timestamp 10. - for i in range(1, 10000): + for i in range(1, self.nrows): if i % 2 == 0: self.session.begin_transaction() cursor.set_key(self.create_key(i)) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Reconcile and remove the obsolete entries. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) self.session.checkpoint() # Now apply an update at timestamp 20. - for i in range(1, 10000): + for i in range(1, self.nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Ensure that we didn't select old history store content even if it is not blew away. - self.session.begin_transaction('read_timestamp=' + timestamp_str(10)) - for i in range(1, 10000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(10)) + for i in range(1, self.nrows): if i % 2 == 0: cursor.set_key(self.create_key(i)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) diff --git a/src/third_party/wiredtiger/test/suite/test_hs12.py b/src/third_party/wiredtiger/test/suite/test_hs12.py index d3be5f40874..e92e338737c 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs12.py +++ b/src/third_party/wiredtiger/test/suite/test_hs12.py @@ -29,19 +29,14 @@ import wiredtiger, wttest, time from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs12.py # Verify we can correctly append modifies to the end of string values class test_hs12(wttest.WiredTigerTestCase): - conn_config = 'cache_size=2MB,statistics=(all),eviction=(threads_max=1)' + conn_config = 'cache_size=2MB,eviction=(threads_max=1)' session_config = 'isolation=snapshot' key_format_values = [ - # FIXME-WT-7120: The commented columnar tests needs to be enabled once columnar - # modify type update is fixed. - # ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), + ('column', dict(key_format='r')), + ('integer-row', dict(key_format='i')), ] scenarios = make_scenarios(key_format_values) @@ -94,12 +89,12 @@ class test_hs12(wttest.WiredTigerTestCase): cursor[1] = value2 self.session.commit_transaction() - # Insert a whole bunch of data into the table to force wiredtiger to evict data - # from the previous table. - self.session.begin_transaction() - for i in range(2, 10000): - cursor[i] = valuebig - self.session.commit_transaction() + # Configure debug behavior on a cursor to evict the positioned page on cursor reset + # and evict the page. + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + evict_cursor.set_key(1) + self.assertEquals(evict_cursor.search(), 0) + evict_cursor.reset() # Try to find the value we saw earlier cursor2.set_key(1) diff --git a/src/third_party/wiredtiger/test/suite/test_hs13.py b/src/third_party/wiredtiger/test/suite/test_hs13.py index ad0992a1656..3781ab200ad 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs13.py +++ b/src/third_party/wiredtiger/test/suite/test_hs13.py @@ -29,19 +29,14 @@ import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs13.py # Verify reverse modify traversal after eviction. class test_hs13(wttest.WiredTigerTestCase): - conn_config = 'cache_size=2MB,statistics=(all),eviction=(threads_max=1)' + conn_config = 'cache_size=2MB,eviction=(threads_max=1)' session_config = 'isolation=snapshot' key_format_values = [ - # FIXME-WT-5550: The commented columnar tests needs to be enabled once columnar modify type - # update is fixed. - # ('column', dict(key_format='r')), - ('integer', dict(key_format='i')) + ('column', dict(key_format='r')), + ('integer-row', dict(key_format='i')) ] scenarios = make_scenarios(key_format_values) @@ -94,12 +89,12 @@ class test_hs13(wttest.WiredTigerTestCase): cursor[1] = value2 self.session.commit_transaction() - # Insert a whole bunch of data into the table to force wiredtiger to evict data - # from the previous table. - self.session.begin_transaction() - for i in range(2, 10000): - cursor[i] = value3 - self.session.commit_transaction() + # Configure debug behavior on a cursor to evict the positioned page on cursor reset + # and evict the page. + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + evict_cursor.set_key(1) + self.assertEquals(evict_cursor.search(), 0) + evict_cursor.reset() # Try to find the value we saw earlier. cursor2.set_key(1) diff --git a/src/third_party/wiredtiger/test/suite/test_hs14.py b/src/third_party/wiredtiger/test/suite/test_hs14.py index 1351dd23e37..7c9d0241d82 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs14.py +++ b/src/third_party/wiredtiger/test/suite/test_hs14.py @@ -29,9 +29,6 @@ import time, wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs14.py # Ensure that point in time reads with few visible history store records don't # damage performance. @@ -40,7 +37,7 @@ class test_hs14(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('string', dict(key_format='S')) + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -52,35 +49,37 @@ class test_hs14(wttest.WiredTigerTestCase): def test_hs14(self): uri = 'table:test_hs14' self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) + nrows = 10000 + value1 = 'a' * 500 value2 = 'b' * 500 value3 = 'c' * 500 value4 = 'd' * 500 value5 = 'e' * 500 - for i in range(1, 10000): + for i in range(1, nrows): self.session.begin_transaction() cursor[self.create_key(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() cursor[self.create_key(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) self.session.begin_transaction() cursor[self.create_key(i)] = value4 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # A checkpoint will ensure that older values are written to the history store. self.session.checkpoint() start = time.time() - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) - for i in range(1, 10000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) + for i in range(1, nrows): self.assertEqual(cursor[self.create_key(i)], value3) self.session.rollback_transaction() end = time.time() @@ -88,21 +87,21 @@ class test_hs14(wttest.WiredTigerTestCase): # The time spent when all history store keys are visible to us. visible_hs_latency = (end - start) - for i in range(1, 10000): + for i in range(1, nrows): self.session.begin_transaction() cursor.set_key(self.create_key(i)) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) self.session.begin_transaction() cursor[self.create_key(i)] = value5 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # A checkpoint will ensure that older values are written to the history store. self.session.checkpoint() start = time.time() - self.session.begin_transaction('read_timestamp=' + timestamp_str(9)) - for i in range(1, 10000): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(9)) + for i in range(1, nrows): cursor.set_key(self.create_key(i)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs15.py b/src/third_party/wiredtiger/test/suite/test_hs15.py index 7d27d166a70..9dded0b2d7d 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs15.py +++ b/src/third_party/wiredtiger/test/suite/test_hs15.py @@ -34,9 +34,6 @@ import time, wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs15.py # Ensure eviction doesn't clear the history store again after checkpoint has done so because of the same update without timestamp. class test_hs15(wttest.WiredTigerTestCase): @@ -44,7 +41,7 @@ class test_hs15(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('string', dict(key_format='S')) + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -71,46 +68,46 @@ class test_hs15(wttest.WiredTigerTestCase): for i in range(2, 1000): self.session.begin_transaction() cursor[self.create_key(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Do a modify and an update with timestamps self.session.begin_transaction() cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('B', 100, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) self.session.begin_transaction() cursor[self.create_key(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Make the modify with timestamp and the update without timestamp obsolete - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) # Do a checkpoint self.session.checkpoint() self.session.begin_transaction() cursor[self.create_key(1)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Insert a bunch of other contents to trigger eviction for i in range(2, 1000): self.session.begin_transaction() cursor[self.create_key(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) expected = list(value1) expected[100] = 'B' expected = str().join(expected) - self.session.begin_transaction('read_timestamp=' + timestamp_str(1)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(1)) self.assertEqual(cursor[self.create_key(1)], expected) self.session.rollback_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) self.assertEqual(cursor[self.create_key(1)], value2) self.session.rollback_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) self.assertEqual(cursor[self.create_key(1)], value3) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs16.py b/src/third_party/wiredtiger/test/suite/test_hs16.py index 11ca79456d0..1a5ed51128c 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs16.py +++ b/src/third_party/wiredtiger/test/suite/test_hs16.py @@ -29,9 +29,6 @@ import time, wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs16.py # Ensure that we don't panic when inserting an update without timestamp to the history store. class test_hs16(wttest.WiredTigerTestCase): @@ -39,7 +36,7 @@ class test_hs16(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = ( ('column', dict(key_format='r')), - ('string', dict(key_format='S')) + ('string-row', dict(key_format='S')) ) scenarios = make_scenarios(key_format_values) @@ -62,7 +59,7 @@ class test_hs16(wttest.WiredTigerTestCase): # Update an update at timestamp 1 self.session.begin_transaction() cursor[self.create_key(1)] = 'b' - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) # Open anther session to make the next update without timestamp non-globally visible session2 = self.setUpSessionOpen(self.conn) @@ -78,7 +75,7 @@ class test_hs16(wttest.WiredTigerTestCase): # Update an update at timestamp 2 self.session.begin_transaction() cursor[self.create_key(1)] = 'd' - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Do a checkpoint, it should not panic self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_hs18.py b/src/third_party/wiredtiger/test/suite/test_hs18.py index 889d2dceccc..bcef53e4d17 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs18.py +++ b/src/third_party/wiredtiger/test/suite/test_hs18.py @@ -29,9 +29,6 @@ import time, wiredtiger, wttest, unittest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs18.py # Test various older reader scenarios class test_hs18(wttest.WiredTigerTestCase): @@ -39,7 +36,7 @@ class test_hs18(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('string', dict(key_format='S')) + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -76,7 +73,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 3 self.session.begin_transaction() cursor[self.create_key(1)] = value0 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Start a long running transaction which could see update 0. session2.begin_transaction() @@ -85,12 +82,12 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Insert another update at timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Insert a bunch of contents to fill the cache for i in range(2000, 10000): @@ -106,7 +103,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 15 self.session.begin_transaction() cursor[self.create_key(1)] = value5 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Check our value is still correct. self.check_value(cursor2, value0) @@ -122,11 +119,6 @@ class test_hs18(wttest.WiredTigerTestCase): # Test that we don't get the wrong value if we read with a timestamp originally. def test_read_timestamp_weirdness(self): - # FIXME-WT-7136: Mixed mode transactions not supported for column store scenario. - # Re-enable once complete. - if self.key_format == 'r': - return - uri = 'table:test_hs18' self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) @@ -144,20 +136,20 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 3 self.session.begin_transaction() cursor[self.create_key(1)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Start a long running transaction which could see update 0. - session2.begin_transaction('read_timestamp=' + timestamp_str(5)) + session2.begin_transaction('read_timestamp=' + self.timestamp_str(5)) # Check our value is still correct. self.check_value(cursor2, value1) # Insert another update at timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Start a long running transaction which could see update 0. - session3.begin_transaction('read_timestamp=' + timestamp_str(5)) + session3.begin_transaction('read_timestamp=' + self.timestamp_str(5)) # Check our value is still correct. self.check_value(cursor3, value1) @@ -175,7 +167,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 15 self.session.begin_transaction() cursor[self.create_key(1)] = value5 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Check our value is still correct. self.check_value(cursor2, value1) @@ -219,12 +211,12 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Insert another update at timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Insert a bunch of other contents to trigger eviction for i in range(2, 10000): @@ -251,11 +243,6 @@ class test_hs18(wttest.WiredTigerTestCase): # Test older readers for each of the updates moved to the history store. def test_multiple_older_readers(self): - # FIXME-WT-7136: Mixed mode transactions not supported for column store scenario. - # Re-enable once complete. - if self.key_format == 'r': - return - uri = 'table:test_multiple_older_readers' self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) @@ -274,7 +261,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 3 self.session.begin_transaction() cursor[self.create_key(1)] = values[0] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Start a transaction that will see update 0. self.start_txn(sessions, cursors, values, 0) @@ -282,7 +269,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = values[1] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Start a transaction that will see update 1. self.start_txn(sessions, cursors, values, 1) @@ -290,7 +277,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert another update at timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = values[2] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Start a transaction that will see update 2. self.start_txn(sessions, cursors, values, 2) @@ -312,7 +299,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 15 self.session.begin_transaction() cursor[self.create_key(1)] = values[4] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Insert a bunch of other contents to trigger eviction for i in range(10001, 20000): @@ -328,11 +315,6 @@ class test_hs18(wttest.WiredTigerTestCase): cursors[i].reset() def test_multiple_older_readers_with_multiple_mixed_mode(self): - # FIXME-WT-7136: Mixed mode transactions not supported for column store scenario. - # Re-enable once complete. - if self.key_format == 'r': - return - uri = 'table:test_multiple_older_readers' self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) @@ -351,7 +333,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 3 self.session.begin_transaction() cursor[self.create_key(1)] = values[0] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Start a transaction that will see update 0. self.start_txn(sessions, cursors, values, 0) @@ -359,7 +341,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = values[1] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Start a transaction that will see update 1. self.start_txn(sessions, cursors, values, 1) @@ -367,7 +349,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert another update at timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = values[2] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Start a transaction that will see update 2. self.start_txn(sessions, cursors, values, 2) @@ -389,7 +371,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = values[4] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Start a transaction that will see update 4. self.start_txn(sessions, cursors, values, 4) @@ -397,7 +379,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 10 self.session.begin_transaction() cursor[self.create_key(1)] = values[5] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Start a transaction that will see update 5. self.start_txn(sessions, cursors, values, 5) @@ -405,7 +387,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 15 self.session.begin_transaction() cursor[self.create_key(1)] = values[6] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Start a transaction that will see update 6. self.start_txn(sessions, cursors, values, 6) @@ -434,7 +416,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Commit an update with timestamp 5 self.session.begin_transaction() cursor[self.create_key(1)] = values[8] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Insert a bunch of other contents to trigger eviction for i in range(10001, 20000): @@ -450,17 +432,14 @@ class test_hs18(wttest.WiredTigerTestCase): cursors[i].reset() def test_modifies(self): - # FIXME-WT-7136: Modify not supported for column store scenario. - # Re-enable once complete. - if self.key_format == 'r': - return - uri = 'table:test_modifies' self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) session_ts_reader = self.setUpSessionOpen(self.conn) cursor_ts_reader = session_ts_reader.open_cursor(uri) + self.skipTest('Skip this part of test_hs18 until WT-7931 is resolved') + # The ID of the session corresponds the value it should see. sessions = [] cursors = [] @@ -480,7 +459,7 @@ class test_hs18(wttest.WiredTigerTestCase): # Insert an update at timestamp 3 self.session.begin_transaction() cursor[self.create_key(1)] = values[0] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Start a long running transaction which could see update 0. self.start_txn(sessions, cursors, values, 0) @@ -489,7 +468,7 @@ class test_hs18(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(self.create_key(1)) cursor.modify([wiredtiger.Modify('a', 0, 0)]) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) session_ts_reader.begin_transaction('read_timestamp=3') self.check_value(cursor_ts_reader, values[0]) @@ -501,7 +480,7 @@ class test_hs18(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(self.create_key(1)) cursor.modify([wiredtiger.Modify('b', 0, 0)]) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Start a long running transaction which could see modify 1. self.start_txn(sessions, cursors, values, 2) @@ -524,7 +503,7 @@ class test_hs18(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(self.create_key(1)) cursor.modify([wiredtiger.Modify('e', 0, 0)]) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Start a long running transaction which could see modify 2. sessions[4].begin_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs19.py b/src/third_party/wiredtiger/test/suite/test_hs19.py index e0549077275..2e3452ae0ca 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs19.py +++ b/src/third_party/wiredtiger/test/suite/test_hs19.py @@ -29,9 +29,6 @@ import time, wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_hs19.py # Ensure eviction doesn't clear the history store again after checkpoint has done so because of the same update without timestamp. class test_hs19(wttest.WiredTigerTestCase): @@ -39,7 +36,7 @@ class test_hs19(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('string', dict(key_format='S')) + ('string-row', dict(key_format='S')) ] scenarios = make_scenarios(key_format_values) @@ -57,7 +54,7 @@ class test_hs19(wttest.WiredTigerTestCase): cursor2 = session2.open_cursor(junk_uri) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) value1 = 'a' * 500 value2 = 'b' * 500 @@ -73,13 +70,13 @@ class test_hs19(wttest.WiredTigerTestCase): cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('B', 100, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) self.session.begin_transaction() cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('C', 101, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Start a transaction to pin back the reconciliation last running value. session2.begin_transaction() @@ -93,14 +90,14 @@ class test_hs19(wttest.WiredTigerTestCase): cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('AAAAAAAAAA', 102, 0)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Insert a modify to get written as the on disk value by checkpoint. self.session.begin_transaction() cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('D', 102, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Checkpoint such that all modifies get written out to the history store and the latest # modify gets written to the on disk value. @@ -112,7 +109,7 @@ class test_hs19(wttest.WiredTigerTestCase): cursor.set_key(self.create_key(1)) mods = [wiredtiger.Modify('E', 103, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # First deposition the first cursor, so the page can be evicted. cursor.reset() @@ -129,7 +126,7 @@ class test_hs19(wttest.WiredTigerTestCase): expected = str().join(expected) # Retrieve the value at timestamp 1. - self.session.begin_transaction('read_timestamp=' + timestamp_str(1)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(1)) cursor.set_key(self.create_key(1)) cursor.search() @@ -143,7 +140,7 @@ class test_hs19(wttest.WiredTigerTestCase): expected = str().join(expected) # Retrieve the value at timestamp 1. - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) cursor.set_key(self.create_key(1)) cursor.search() @@ -159,7 +156,7 @@ class test_hs19(wttest.WiredTigerTestCase): expected = str().join(expected) # Retrieve the value at timestamp 1. - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) cursor.set_key(self.create_key(1)) cursor.search() # Assert that it matches our expected value. diff --git a/src/third_party/wiredtiger/test/suite/test_hs20.py b/src/third_party/wiredtiger/test/suite/test_hs20.py index 53e2f141726..8a8a89ffe4f 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs20.py +++ b/src/third_party/wiredtiger/test/suite/test_hs20.py @@ -27,23 +27,36 @@ # OTHER DEALINGS IN THE SOFTWARE. import time, wiredtiger, wttest +from wtscenario import make_scenarios # test_hs20.py # Ensure we never reconstruct a reverse modify update in the history store based on the onpage overflow value -def timestamp_str(t): - return '%x' % t - class test_hs20(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,eviction=(threads_max=1)' session_config = 'isolation=snapshot' + # Return the k'th (0-based) key. + def make_column_key(k): + return k + 1 + def make_string_key(k): + return str(k) + + key_format_values = [ + ('column', dict(key_format='r', make_key=make_column_key)), + ('string-row', dict(key_format='S', make_key=make_string_key)), + ] + + scenarios = make_scenarios(key_format_values) + def test_hs20(self): uri = 'table:test_hs20' + key_format = 'key_format=' + self.key_format + # Set a very small maximum leaf value to trigger writing overflow values - self.session.create(uri, 'key_format=S,value_format=S,leaf_value_max=10B') + self.session.create(uri, '{},value_format=S,leaf_value_max=10B'.format(key_format)) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) value1 = 'a' * 500 value2 = 'b' * 50 @@ -51,41 +64,41 @@ class test_hs20(wttest.WiredTigerTestCase): # Insert a value that is larger than the maximum leaf value. for i in range(0, 10): self.session.begin_transaction() - cursor[str(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + cursor[self.make_key(i)] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Do 2 modifies. for i in range(0, 10): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.make_key(i)) mods = [wiredtiger.Modify('B', 500, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(0, 10): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.make_key(i)) mods = [wiredtiger.Modify('C', 501, 1)] self.assertEqual(cursor.modify(mods), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Insert more data to trigger eviction. for i in range(10, 100000): self.session.begin_transaction() - cursor[str(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + cursor[self.make_key(i)] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Update the overflow values. for i in range(0, 10): self.session.begin_transaction() - cursor[str(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + cursor[self.make_key(i)] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) # Do a checkpoint to move the overflow values to the history store but keep the current in memory disk image. self.session.checkpoint() # Search the first modifies. for i in range(0, 10): - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) - self.assertEqual(cursor[str(i)], value1 + "B") + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) + self.assertEqual(cursor[self.make_key(i)], value1 + "B") self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs21.py b/src/third_party/wiredtiger/test/suite/test_hs21.py index 920dc41951b..7e8755430ca 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs21.py +++ b/src/third_party/wiredtiger/test/suite/test_hs21.py @@ -30,9 +30,7 @@ import time, re import wiredtiger, wttest from wtdataset import SimpleDataSet from wiredtiger import stat - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_hs21.py # Test we don't lose any data when idle files with an active history are closed/sweeped. @@ -49,6 +47,13 @@ class test_hs21(wttest.WiredTigerTestCase): numfiles = 10 nrows = 1000 + key_format_values = [ + ('column', dict(key_format='r', key1=1, key2=2)), + ('string-row', dict(key_format='S', key1=str(0), key2=str(1))), + ] + + scenarios = make_scenarios(key_format_values) + def large_updates(self, uri, value, ds, nrows, commit_ts): # Update a large number of records, we'll hang if the history store table isn't working. session = self.session @@ -56,13 +61,13 @@ class test_hs21(wttest.WiredTigerTestCase): session.begin_transaction() for i in range(1, nrows + 1): cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, session, check_value, uri, nrows, read_ts=-1): # Validate we read an expected value (optionally at a given read timestamp). if read_ts != -1: - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -70,6 +75,12 @@ class test_hs21(wttest.WiredTigerTestCase): count += 1 if read_ts != -1: session.rollback_transaction() + if count != nrows: + self.prout("Oops") + self.prout("value: " + str(check_value)) + self.prout("count: " + str(count)) + self.prout("nrows: " + str(nrows)) + self.prout("read_ts: " + str(read_ts)) self.assertEqual(count, nrows) cursor.close() @@ -101,7 +112,7 @@ class test_hs21(wttest.WiredTigerTestCase): file_uri = 'file:%s.%d.wt' % (self.file_name, f) # Create a small table. ds = SimpleDataSet( - self, table_uri, 0, key_format='S', value_format='S', config='log=(enabled=false)') + self, table_uri, 0, key_format=self.key_format, value_format='S', config='log=(enabled=false)') ds.populate() # Checkpoint to ensure we write the files metadata checkpoint value. self.session.checkpoint() @@ -110,8 +121,8 @@ class test_hs21(wttest.WiredTigerTestCase): active_files.append((base_write_gen, ds)) # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) # Perform a series of updates over our files at timestamp 2. This being data we can later assert # to ensure the history store is working as intended. @@ -122,7 +133,7 @@ class test_hs21(wttest.WiredTigerTestCase): # We want to create a long running read transaction in a seperate session which we will persist over the closing and # re-opening of handles. We want to ensure the correct data gets read throughout this time period. session_read = self.conn.open_session() - session_read.begin_transaction('read_timestamp=' + timestamp_str(2)) + session_read.begin_transaction('read_timestamp=' + self.timestamp_str(2)) # Check our inital set of updates are seen at the read timestamp. for (_, ds) in active_files: # Check that all updates at timestamp 2 are seen. diff --git a/src/third_party/wiredtiger/test/suite/test_hs22.py b/src/third_party/wiredtiger/test/suite/test_hs22.py index cf30767b8bc..97a65ace10a 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs22.py +++ b/src/third_party/wiredtiger/test/suite/test_hs22.py @@ -27,9 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_hs22.py # Test the case that out of order timestamp @@ -38,43 +36,53 @@ class test_hs22(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', key1=1, key2=2)), + ('string-row', dict(key_format='S', key1=str(0), key2=str(1))), + ] + + scenarios = make_scenarios(key_format_values) + def test_onpage_out_of_order_timestamp_update(self): uri = 'table:test_hs22' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) + + key1 = self.key1 + key2 = self.key2 value1 = 'a' value2 = 'b' # Insert a key. self.session.begin_transaction() - cursor[str(0)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + cursor[key1] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Remove the key. self.session.begin_transaction() - cursor.set_key(str(0)) + cursor.set_key(key1) self.assertEqual(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Do an out of order timestamp # update and write it to the data # store later. self.session.begin_transaction() - cursor[str(0)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + cursor[key1] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Insert another key. self.session.begin_transaction() - cursor[str(1)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + cursor[key2] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Update the key. self.session.begin_transaction() - cursor[str(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + cursor[key2] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) # Do a checkpoint to trigger # history store reconciliation. @@ -83,58 +91,61 @@ class test_hs22(wttest.WiredTigerTestCase): evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") # Search the key to evict it. - self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) - self.assertEqual(evict_cursor[str(0)], value2) + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(15)) + self.assertEqual(evict_cursor[key1], value2) self.assertEqual(evict_cursor.reset(), 0) self.session.rollback_transaction() # Search the key again to verify the data is still as expected. - self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) - self.assertEqual(cursor[str(0)], value2) + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(15)) + self.assertEqual(cursor[key1], value2) self.session.rollback_transaction() def test_out_of_order_timestamp_update_newer_than_tombstone(self): uri = 'table:test_hs22' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) + + key1 = self.key1 + key2 = self.key2 value1 = 'a' value2 = 'b' # Insert a key. self.session.begin_transaction() - cursor[str(0)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + cursor[key1] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Remove a key. self.session.begin_transaction() - cursor.set_key(str(0)) + cursor.set_key(key1) self.assertEqual(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Do an out of order timestamp # update and write it to the # history store later. self.session.begin_transaction() - cursor[str(0)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + cursor[key1] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) # Add another update. self.session.begin_transaction() - cursor[str(0)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + cursor[key1] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Insert another key. self.session.begin_transaction() - cursor[str(1)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + cursor[key2] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Update the key. self.session.begin_transaction() - cursor[str(1)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + cursor[key2] = value2 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) # Do a checkpoint to trigger # history store reconciliation. @@ -143,12 +154,12 @@ class test_hs22(wttest.WiredTigerTestCase): evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") # Search the key to evict it. - self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) - self.assertEqual(evict_cursor[str(0)], value2) + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(15)) + self.assertEqual(evict_cursor[key1], value2) self.assertEqual(evict_cursor.reset(), 0) self.session.rollback_transaction() # Search the key again to verify the data is still as expected. - self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) - self.assertEqual(cursor[str(0)], value2) + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(15)) + self.assertEqual(cursor[key1], value2) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs23.py b/src/third_party/wiredtiger/test/suite/test_hs23.py index cfc7ad70623..d6bc1ec3d70 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs23.py +++ b/src/third_party/wiredtiger/test/suite/test_hs23.py @@ -27,9 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_hs23.py # Test the case that we have update, out of order timestamp @@ -38,12 +36,21 @@ class test_hs23(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', key=1)), + ('string-row', dict(key_format='S', key=str(0))), + ] + + scenarios = make_scenarios(key_format_values) + def test(self): uri = 'table:test_hs23' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) + + key = self.key value1 = 'a' value2 = 'b' @@ -53,34 +60,34 @@ class test_hs23(wttest.WiredTigerTestCase): # Insert a key. self.session.begin_transaction() - cursor[str(0)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + cursor[key] = value1 + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Update at 10, update at 20, update at 15 (out of order), and # update at 20 in the same transaction self.session.begin_transaction() - cursor.set_key(str(0)) + cursor.set_key(key) cursor.set_value(value2) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(10)) + 'commit_timestamp=' + self.timestamp_str(10)) self.assertEquals(cursor.update(), 0) - cursor.set_key(str(0)) + cursor.set_key(key) cursor.set_value(value3) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(20)) + 'commit_timestamp=' + self.timestamp_str(20)) self.assertEquals(cursor.update(), 0) - cursor.set_key(str(0)) + cursor.set_key(key) cursor.set_value(value4) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(15)) + 'commit_timestamp=' + self.timestamp_str(15)) self.assertEquals(cursor.update(), 0) - cursor.set_key(str(0)) + cursor.set_key(key) cursor.set_value(value5) self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(20)) + 'commit_timestamp=' + self.timestamp_str(20)) self.assertEquals(cursor.update(), 0) self.session.commit_transaction() @@ -92,16 +99,16 @@ class test_hs23(wttest.WiredTigerTestCase): # Search the key to evict it. self.session.begin_transaction() - self.assertEqual(evict_cursor[str(0)], value5) + self.assertEqual(evict_cursor[key], value5) self.assertEqual(evict_cursor.reset(), 0) self.session.rollback_transaction() # Search the latest update - self.session.begin_transaction("read_timestamp=" + timestamp_str(20)) - self.assertEqual(cursor[str(0)], value5) + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(20)) + self.assertEqual(cursor[key], value5) self.session.rollback_transaction() - # Serarch the out of order timestamp update - self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) - self.assertEqual(cursor[str(0)], value4) + # Search the out of order timestamp update + self.session.begin_transaction("read_timestamp=" + self.timestamp_str(15)) + self.assertEqual(cursor[key], value4) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs24.py b/src/third_party/wiredtiger/test/suite/test_hs24.py index 4c4ee4a4b94..08d6f303bc5 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs24.py +++ b/src/third_party/wiredtiger/test/suite/test_hs24.py @@ -28,9 +28,7 @@ import wttest, threading, wiredtiger from helper import simulate_crash_restart - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_hs24.py # Test that out of order timestamp fix racing with checkpointing the history store doesn't create inconsistent checkpoint. @@ -38,24 +36,32 @@ class test_hs24(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,timing_stress_for_test=(history_store_checkpoint_delay)' session_config = 'isolation=snapshot' uri = 'table:test_hs24' + numrows = 2000 + + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) value1 = 'a' * 500 value2 = 'b' * 500 value3 = 'c' * 500 value4 = 'd' * 500 def test_zero_ts(self): - self.session.create(self.uri, 'key_format=i,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.session.create(self.uri, 'key_format={},value_format=S'. format(self.key_format)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): self.session.begin_transaction() cursor[i] = self.value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() cursor[i] = self.value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) cursor.close() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) thread = threading.Thread(target=self.zero_ts_deletes) thread.start() self.session.checkpoint() @@ -64,13 +70,13 @@ class test_hs24(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri) session2 = self.conn.open_session(None) cursor2 = session2.open_cursor(self.uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) - session2.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) + session2.begin_transaction('read_timestamp=' + self.timestamp_str(4)) # Check the data store and the history store content is consistent. # If we have a value in the data store, we should see the older # version in the history store as well. newer_data_visible = False - for i in range(0, 2000): + for i in range(1, self.numrows + 1): cursor.set_key(i) cursor2.set_key(i) ret = cursor.search() @@ -88,7 +94,7 @@ class test_hs24(wttest.WiredTigerTestCase): def zero_ts_deletes(self): session = self.setUpSessionOpen(self.conn) cursor = session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): session.begin_transaction() cursor.set_key(i) cursor.remove() @@ -97,29 +103,29 @@ class test_hs24(wttest.WiredTigerTestCase): session.close() def test_zero_commit(self): - self.session.create(self.uri, 'key_format=i,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): self.session.begin_transaction() cursor[i] = self.value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() cursor[i] = self.value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) cursor.close() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(4)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(4)) thread = threading.Thread(target=self.zero_ts_commits) thread.start() self.session.checkpoint() thread.join() simulate_crash_restart(self, '.', "RESTART") cursor = self.session.open_cursor(self.uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) # Check we can only see the version committed by the zero timestamp # commit thread before the checkpoint starts or value1. newer_data_visible = False - for i in range(0, 2000): + for i in range(1, self.numrows + 1): value = cursor[i] if not newer_data_visible: newer_data_visible = value != self.value3 @@ -132,7 +138,7 @@ class test_hs24(wttest.WiredTigerTestCase): def zero_ts_commits(self): session = self.setUpSessionOpen(self.conn) cursor = session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): session.begin_transaction() cursor[i] = self.value3 session.commit_transaction() @@ -140,21 +146,21 @@ class test_hs24(wttest.WiredTigerTestCase): session.close() def test_out_of_order_ts(self): - self.session.create(self.uri, 'key_format=i,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): self.session.begin_transaction() cursor[i] = self.value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() cursor[i] = self.value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(4)) - for i in range(0, 2000): + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(4)) + for i in range(1, self.numrows + 1): self.session.begin_transaction() cursor[i] = self.value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(6)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(6)) cursor.close() thread = threading.Thread(target=self.out_of_order_ts_commits) thread.start() @@ -162,12 +168,12 @@ class test_hs24(wttest.WiredTigerTestCase): thread.join() simulate_crash_restart(self, '.', "RESTART") cursor = self.session.open_cursor(self.uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) # Check we can only see the version at timestamp 4, it's either # committed by the out of order timestamp commit thread before the # checkpoint starts or value1. newer_data_visible = False - for i in range(0, 2000): + for i in range(1, self.numrows + 1): value = cursor[i] if not newer_data_visible: newer_data_visible = value != self.value4 @@ -180,9 +186,9 @@ class test_hs24(wttest.WiredTigerTestCase): def out_of_order_ts_commits(self): session = self.setUpSessionOpen(self.conn) cursor = session.open_cursor(self.uri) - for i in range(0, 2000): + for i in range(1, self.numrows + 1): session.begin_transaction() cursor[i] = self.value4 - session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) cursor.close() session.close() diff --git a/src/third_party/wiredtiger/test/suite/test_hs25.py b/src/third_party/wiredtiger/test/suite/test_hs25.py index e353ce076fe..a1a18928864 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs25.py +++ b/src/third_party/wiredtiger/test/suite/test_hs25.py @@ -27,9 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_hs25.py # Ensure updates structure is correct when processing each key. @@ -38,31 +36,38 @@ class test_hs25(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' uri = 'table:test_hs25' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_insert_updates_hs(self): - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) - self.session.create(self.uri, 'key_format=i,value_format=S') + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) + self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) s = self.conn.open_session() # Update the first key. cursor1 = self.session.open_cursor(self.uri) self.session.begin_transaction() cursor1[1] = 'a' - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Update the second key. self.session.begin_transaction() cursor1[2] = 'a' - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() cursor1[2] = 'b' - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Prepared update on the first key. self.session.begin_transaction() cursor1[1] = 'b' cursor1[1] = 'c' - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(4)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(4)) # Run eviction cursor. s.begin_transaction('ignore_prepare=true') diff --git a/src/third_party/wiredtiger/test/suite/test_hs26.py b/src/third_party/wiredtiger/test/suite/test_hs26.py new file mode 100644 index 00000000000..9aab2322c43 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hs26.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_hs26.py +# Test that changes overlapping variable-length column store RLE groups don't lose or corrupt data. +# (The concern doesn't exist for row stores, so while this test could be run for row stores there's +# little benefit to doing so; thus, no row-store scenarios are generated.) +# +# This works by writing batches of duplicate values, then overwriting them with new batches, +# all with relatively prime counts so the ranges overlap. It also tests the cases where not +# all the old or new value batches exist. +# +# It checks that the immediately written values can be read back, but the interesting part is +# whether they read back correctly after forcing eviction, which will RLE-encode the duplicates +# and read them back. There are many opportunities for the interaction betwee RLE groups and +# history store accesses to go off the rails. +class test_hs26(wttest.WiredTigerTestCase): + conn_config = '' + session_config = 'isolation=snapshot' + + # We control the duplication of values by appending a number computed from the key. + # Because the keys are 1..N (not 0..N-1), to get aligned RLE groups the suffix is + # computed as (key - 1) // value_modulus. This way, if value_modulus is e.g. 7, we + # get the first 7 keys (1-7) using the same suffix. + # + # It would not be _wrong_ to use key // value modulus, but it makes it much more + # difficult to reason about which RLE groups are overlapping where if the first RLE + # group is shorter than the others by 1. So we don't do that. + # + # The cases where the RLE group is mismatched by exactly 1 are particularly likely to + # have issues, so we want to be sure to exercise those cases. Any pair of relatively + # prime integers will generate these cases as long as there are enough keys (e.g. 7 + # and 13 generate one at 14 and the other at 78) but we also want to encounter these + # cases relative to the number of keys written, both the shorter and longer numbers. + # + # Consequently I've picked 103 for one key count (between multiples of 17 and 13) and + # 209 for the other (between multiples of 13 and 7). The ratio of the two key counts + # doesn't signify much but it's close to 2x on general principles. + # + # Other cases of overlapping the key count are still interesting so we still generate + # the full product of the scenarios. + + nrows_values = [ + ('more', dict(nrows_1=103, nrows_2=211)), + ('same', dict(nrows_1=211, nrows_2=211)), + ('less', dict(nrows_1=211, nrows_2=103)), + ] + value_modulus_1_values = [ + ('7', dict(value_modulus_1=7)), + ('13', dict(value_modulus_1=13)), + ('17', dict(value_modulus_1=17)), + ] + value_modulus_2_values = [ + ('7', dict(value_modulus_2=7)), + ('13', dict(value_modulus_2=13)), + ('17', dict(value_modulus_2=17)), + ] + + scenarios = make_scenarios(nrows_values, value_modulus_1_values, value_modulus_2_values) + + value_1 = 'a' * 500 + value_2 = 'd' * 500 + + timestamp_1 = 2 + timestamp_2 = 100 + + # Generate the value for a key. + def make_value(self, key, base_value, value_modulus): + return base_value + str((key - 1) // value_modulus) + + # Write nrows records, using value as the base value string. + def make_updates(self, uri, ds, value, value_modulus, nrows, commit_ts): + session = self.session + cursor = session.open_cursor(uri) + session.begin_transaction() + for i in range(1, nrows + 1): + cursor[ds.key(i)] = self.make_value(i, value, value_modulus) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + cursor.close() + + # Figure the expected value for a key, based on the read time. + # - If the read time is timestamp_1, we should see value_1. + # - If the read time is timestamp_2, we should see value_2, except if nrows_2 < nrows_1, + # in which case we should see value_1 for keys past nrows_1. + def expected_value(self, key, readtime): + if readtime == self.timestamp_1: + return self.make_value(key, self.value_1, self.value_modulus_1) + elif readtime == self.timestamp_2: + if self.nrows_2 < self.nrows_1 and key > self.nrows_2: + return self.make_value(key, self.value_1, self.value_modulus_1) + else: + return self.make_value(key, self.value_2, self.value_modulus_2) + else: + self.prout("expected_value: Unexpected readtime {}".format(readtime)) + self.assertTrue(False) + return None + + # Return the number of keys we expect, based on the read time. + # - If the read time is timestamp_1, we should see nrows_1. + # - If the read time is timestamp_2, we should see max(nrows_1, nrows_2). + def expected_numvalues(self, readtime): + if readtime == self.timestamp_1: + return self.nrows_1 + elif readtime == self.timestamp_2: + return max(self.nrows_1, self.nrows_2) + else: + self.prout("expected_numvalues: Unexpected readtime {}".format(readtime)) + self.assertTrue(False) + return None + + # Check that we got the values we expected. In particular, also make sure that + # we get the expected number of values back. Expect the values that should be + # there at readtime; if explicit_read_ts is set, open a transaction at that + # timestamp. + def check(self, session, uri, readtime, explicit_read_ts=-1): + if explicit_read_ts != -1: + session.begin_transaction('read_timestamp=' + self.timestamp_str(explicit_read_ts)) + cursor = session.open_cursor(uri) + count = 0 + for k, v in cursor: + # Count is key - 1, so pass count + 1 as key. + self.assertEqual(v, self.expected_value(count + 1, readtime)) + count += 1 + if explicit_read_ts != -1: + session.rollback_transaction() + self.assertEqual(count, self.expected_numvalues(readtime)) + cursor.close() + + def test_hs(self): + + # Create a file that contains active history (content newer than the oldest timestamp). + table_uri = 'table:hs26' + ds = SimpleDataSet( + self, table_uri, 0, key_format='r', value_format='S', config='log=(enabled=false)') + ds.populate() + self.session.checkpoint() + + # Pin oldest and stable to timestamp 1. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) + + # Write the first set of values at timestamp_1. + self.make_updates(ds.uri, ds, self.value_1, self.value_modulus_1, self.nrows_1, self.timestamp_1) + + # Create a long running read transaction in a separate session. + # (Is it necessary for it to be separate? Not sure.) + session_read = self.conn.open_session() + session_read.begin_transaction('read_timestamp=' + self.timestamp_str(self.timestamp_1)) + + # Check that the initial writes (at timestamp_1) are seen. + self.check(session_read, ds.uri, self.timestamp_1) + + # Write different values at a later timestamp. + self.make_updates(ds.uri, ds, self.value_2, self.value_modulus_2, self.nrows_2, self.timestamp_2) + + # Check that the new updates are only seen after the update timestamp. + self.check(self.session, ds.uri, self.timestamp_1, self.timestamp_1) + self.check(self.session, ds.uri, self.timestamp_2, self.timestamp_2) + + self.session.breakpoint() + + # Now forcibly evict, so that all the pages are RLE-encoded and then read back in. + # There doesn't seem to be any way to just forcibly evict an entire table, so what + # I'm going to do is assume that each page can hold at least 41 values, and evict + # every 41st key. If this evicts pages repeatedly it won't really hurt anything, + # just waste some time. + + evict_cursor = self.session.open_cursor(ds.uri, None, "debug=(release_evict)") + self.session.begin_transaction() + for k in range(1, max(self.nrows_1, self.nrows_2) + 1, 41): + # Search the key to evict it. + v = evict_cursor[ds.key(k)] + xv = self.expected_value(k, self.timestamp_2) + self.assertEqual(v, xv) + self.assertEqual(evict_cursor.reset(), 0) + self.session.rollback_transaction() + + # Using the long running read transaction, check that the correct data can still be read. + # It should see all the updates at timestamp_1. + self.check(session_read, ds.uri, self.timestamp_1) + session_read.rollback_transaction() + + # Also check that the most recent transaction has the later data. + self.check(self.session, ds.uri, self.timestamp_2, self.timestamp_2) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_hs27.py b/src/third_party/wiredtiger/test/suite/test_hs27.py new file mode 100644 index 00000000000..e685503b8a2 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hs27.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_hs27.py +# Test that variable-length column store doesn't RLE-compact adjacent data with heterogeneous +# timestamps. +# +# (The concern doesn't exist for row stores, so while this test could be run for row stores there's +# little benefit to doing so; thus, no row-store scenarios are generated.) +# +# This works by writing the same value to adjacent keys at different times, evicting them, and +# making sure they read back correctly. (The eviction is necessary to go through the RLE code.) +class test_hs27(wttest.WiredTigerTestCase): + conn_config = '' + session_config = 'isolation=snapshot' + + nrows = 100 + value_1 = 'a' * 119 + value_2 = 'd' * 119 + + # Configure the number of different timestamps to write at. + ntimes_values = [ + ('two', dict(ntimes=2)), + ('three', dict(ntimes=3)), + ('ten', dict(ntimes=10)), + ] + + # Configure the number of keys to write at each timestamp. + nkeys_values = [ + ('one', dict(nkeys=1)), + ('two', dict(nkeys=2)), + ('three', dict(nkeys=3)), + ] + + # Configure whether the keys should be initialized first. + doinit_values = [ + ('init', dict(doinit=True)), + ('noinit', dict(doinit=False)), + ] + + # Configure whether the timestamp groups of keys are written forwards or backwards. + group_forward_values = [ + ('forward', dict(group_forward=True)), + ('backward', dict(group_forward=False)), + ] + + # Configure whether the keys within a timestamp group are written forwards or backwards. + keys_forward_values = [ + ('forward', dict(keys_forward=True)), + ('backward', dict(keys_forward=False)), + ] + + scenarios = make_scenarios(ntimes_values, nkeys_values, doinit_values, + group_forward_values, keys_forward_values) + + # Get the m'th writer timestamp. + def get_writetime(self, m): + return 40 + 2 * m + + # Get the m'th reader timestamp. + def get_readtime(self, m): + return 41 + 2 * m + + # Get the k'th key for the m'th timestamp. + def get_key(self, k, m): + if self.group_forward: + ts_offset = m * self.nkeys + else: + ts_offset = (self.ntimes - m - 1) * self.nkeys + if self.keys_forward: + key_offset = k + else: + key_offset = self.nkeys - k - 1 + return 71 + ts_offset + key_offset + + # Return the key number k and timestamp number m for a key (inverse of get_key). + def invert_key(self, key): + if key < 71 or key >= 71 + self.ntimes * self.nkeys: + return None + key -= 71 + if self.group_forward: + m = key // self.nkeys + else: + m = self.ntimes - (key // self.nkeys) - 1 + if self.keys_forward: + k = key % self.nkeys + else: + k = self.nkeys - (key % self.nkeys) - 1 + return (k, m) + + # Return the timestamp number k for a timestamp (inverse of get_read/writetime). + # Return -1 or self.ntimes for other timestamps as appropriate, for comparison. + def invert_timestamp(self, ts, beforeval, afterval): + if ts < 40: + return beforeval + if ts >= 40 + self.ntimes * 2: + return afterval + ts -= 40 + m = ts // 2 + return m + + # Figure if we should see value_2 for a given key and read time. + def expect_value_2(self, key, readtime): + # Get the key number and timestamp number for the key. + km = self.invert_key(key) + if km is None: + return False + (k, m) = km + + # Get the timestamp number associated with readtime. + m2 = self.invert_timestamp(readtime, -1, self.ntimes) + + # We should see value_2 if we are at or after the time it was written. + return m2 >= m + + # Check each value explicitly to make sure it's what we meant to see. + # Don't bother checking the background values. + def check1(self, session, uri, ds, readtime, make_own_txn): + if make_own_txn: + session.begin_transaction('read_timestamp=' + self.timestamp_str(readtime)) + cursor = session.open_cursor(uri) + for k in range(0, self.nkeys): + for m in range(0, self.ntimes): + key = self.get_key(k, m) + if self.expect_value_2(key, readtime): + self.assertEqual(cursor[ds.key(key)], self.value_2) + elif self.doinit: + self.assertEqual(cursor[ds.key(key)], self.value_1) + else: + self.assertRaisesException(KeyError, lambda: cursor[ds.key(key)]) + if make_own_txn: + session.rollback_transaction() + cursor.close() + + # Scan through the whole table and make sure it's what we expect. + def check2(self, session, uri, readtime, make_own_txn): + if make_own_txn: + session.begin_transaction('read_timestamp=' + self.timestamp_str(readtime)) + cursor = session.open_cursor(uri) + count = 0 + for (key, valueseen) in cursor: + val = self.value_2 if self.expect_value_2(key, readtime) else self.value_1 + self.assertEqual(valueseen, val) + count += 1 + if self.doinit: + # The table was initialized; should always see all rows. + self.assertEqual(count, self.nrows) + else: + # The table was not initialized; should see only what we wrote by this time. + # (If m is 0 we should see one batch of keys.) + m = self.invert_timestamp(readtime, -1, self.ntimes - 1) + self.assertEqual(count, self.nkeys * (m + 1)) + if make_own_txn: + session.rollback_transaction() + cursor.close() + + # Scan through the whole table backward too, since cursor_prev isn't adequately tested. + def check3(self, session, uri, readtime, make_own_txn): + if make_own_txn: + session.begin_transaction('read_timestamp=' + self.timestamp_str(readtime)) + cursor = session.open_cursor(uri) + count = 0 + while cursor.prev() == 0: + key = cursor.get_key() + valueseen = cursor[key] + val = self.value_2 if self.expect_value_2(key, readtime) else self.value_1 + self.assertEqual(valueseen, val) + count += 1 + if self.doinit: + # The table was initialized; should always see all rows. + self.assertEqual(count, self.nrows) + else: + # The table was not initialized; should see only what we wrote by this time. + # (If m is 0 we should see one batch of keys.) + m = self.invert_timestamp(readtime, -1, self.ntimes - 1) + self.assertEqual(count, self.nkeys * (m + 1)) + if make_own_txn: + session.rollback_transaction() + cursor.close() + + # Do all three checks. + def check(self, session, uri, ds, readtime, make_own_txn=True): + self.check1(session, uri, ds, readtime, make_own_txn) + self.check2(session, uri, readtime, make_own_txn) + self.check3(session, uri, readtime, make_own_txn) + + # Scan through the whole table at each relevant timestamp and make sure it's what we expect. + def checkall(self, session, uri, ds): + self.check(session, uri, ds, 2) + for m in range(0, self.ntimes): + self.check(session, uri, ds, self.get_readtime(m)) + self.check(session, uri, ds, 100) + + # Write self.nrows records at timestamp 1, using self.value_1 as the value. + def initialize(self, uri, ds): + session = self.session + cursor = session.open_cursor(uri) + session.begin_transaction() + for i in range(1, self.nrows + 1): + cursor[ds.key(i)] = self.value_1 + session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) + cursor.close() + + # Do the m'th update. + def update(self, uri, ds, m): + writetime = self.get_writetime(m) + + session = self.session + cursor = session.open_cursor(uri) + session.begin_transaction() + if self.keys_forward: + for k in range(0, self.nkeys): + key = self.get_key(k, m) + cursor[ds.key(key)] = self.value_2 + else: + for k in range(self.nkeys - 1, -1, -1): + key = self.get_key(k, m) + cursor[ds.key(key)] = self.value_2 + session.commit_transaction('commit_timestamp=' + self.timestamp_str(writetime)) + cursor.close() + + # Do all the updates. + def updateall(self, uri, ds): + if self.group_forward: + for m in range(0, self.ntimes): + self.update(uri, ds, m) + else: + for m in range(self.ntimes - 1, -1, -1): + self.update(uri, ds, m) + + def test_hs(self): + + # Create a file that contains active history (content newer than the oldest timestamp). + table_uri = 'table:hs27' + ds = SimpleDataSet( + self, table_uri, 0, key_format='r', value_format='S', config='log=(enabled=false)') + ds.populate() + self.session.checkpoint() + + # Pin oldest and stable to timestamp 1. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) + + # Write the initial values, if requested. + if self.doinit: + self.initialize(ds.uri, ds) + + # Create a long running read transaction in a separate session. + # (Is it necessary for it to be separate? Not sure.) + session_read = self.conn.open_session() + session_read.begin_transaction('read_timestamp=' + self.timestamp_str(2)) + + # Check that the initial writes (at timestamp 1) are seen (at timestamp 2). + self.check(session_read, ds.uri, ds, 2, make_own_txn=False) + + # Write more values at assorted timestamps. + self.updateall(ds.uri, ds) + + # Check that the new updates are appropriately visible. + self.checkall(self.session, ds.uri, ds) + + self.session.breakpoint() + + # Now forcibly evict, so that all the pages are RLE-encoded and then read back in. + # There doesn't seem to be any way to just forcibly evict an entire table, so what + # I'm going to do is assume that what we care about is evicting the updates (the + # initial values are not so interesting) and they are on a maximum of two pages, + # so we can evict the first and last key. If this evicts the same page twice, it + # won't really hurt anything. (This also avoids having to worry about whether we + # wrote initial values or not.) + + evict_cursor = self.session.open_cursor(ds.uri, None, "debug=(release_evict)") + self.session.begin_transaction() + firstkey = self.get_key(0, 0) + lastkey = self.get_key(self.nkeys - 1, self.ntimes - 1) + for k in [firstkey, lastkey]: + # Search the key to evict it. + v = evict_cursor[ds.key(k)] + self.assertEqual(v, self.value_2) + self.assertEqual(evict_cursor.reset(), 0) + self.session.rollback_transaction() + + # Check that the long-running read transaction still reads the correct data. + self.check(session_read, ds.uri, ds, 2, make_own_txn=False) + + # Check that our main session reads the correct data. + self.checkall(self.session, ds.uri, ds) + + # Drop the long running read transaction. + session_read.rollback_transaction() + + # Check that our main session can still read the latest data. + self.check(self.session, ds.uri, ds, 100) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_import01.py b/src/third_party/wiredtiger/test/suite/test_import01.py index 9376b805a23..daaae5f95e3 100644 --- a/src/third_party/wiredtiger/test/suite/test_import01.py +++ b/src/third_party/wiredtiger/test/suite/test_import01.py @@ -103,14 +103,10 @@ class test_import_base(wttest.WiredTigerTestCase): "Tmplog" not in file_name and "Preplog" not in file_name: shutil.copy(src_path, dest_dir) - # Convert a WiredTiger timestamp to a string. - def timestamp_str(self, t): - return '%x' % t - # test_import01 class test_import01(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' original_db_file = 'original_db_file' diff --git a/src/third_party/wiredtiger/test/suite/test_import02.py b/src/third_party/wiredtiger/test/suite/test_import02.py index 57f80b769d0..9dbb7b6f66a 100644 --- a/src/third_party/wiredtiger/test/suite/test_import02.py +++ b/src/third_party/wiredtiger/test/suite/test_import02.py @@ -34,7 +34,7 @@ import wiredtiger, wttest from test_import01 import test_import_base class test_import02(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' original_db_file = 'original_db_file' diff --git a/src/third_party/wiredtiger/test/suite/test_import03.py b/src/third_party/wiredtiger/test/suite/test_import03.py index a62486f1bb9..278daafa98e 100644 --- a/src/third_party/wiredtiger/test/suite/test_import03.py +++ b/src/third_party/wiredtiger/test/suite/test_import03.py @@ -34,7 +34,7 @@ from wtscenario import make_scenarios from test_import01 import test_import_base class test_import03(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' ntables = 10 diff --git a/src/third_party/wiredtiger/test/suite/test_import04.py b/src/third_party/wiredtiger/test/suite/test_import04.py index 230fad578e2..f64aefdee70 100644 --- a/src/third_party/wiredtiger/test/suite/test_import04.py +++ b/src/third_party/wiredtiger/test/suite/test_import04.py @@ -54,7 +54,7 @@ from wtscenario import make_scenarios from test_import01 import test_import_base class test_import04(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' ntables = 10 diff --git a/src/third_party/wiredtiger/test/suite/test_import05.py b/src/third_party/wiredtiger/test/suite/test_import05.py index b78a4b81116..0307c6586cb 100644 --- a/src/third_party/wiredtiger/test/suite/test_import05.py +++ b/src/third_party/wiredtiger/test/suite/test_import05.py @@ -35,7 +35,7 @@ from wtscenario import make_scenarios from test_import01 import test_import_base class test_import05(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' ntables = 10 diff --git a/src/third_party/wiredtiger/test/suite/test_import06.py b/src/third_party/wiredtiger/test/suite/test_import06.py index 083fc860a67..7d1360bacba 100644 --- a/src/third_party/wiredtiger/test/suite/test_import06.py +++ b/src/third_party/wiredtiger/test/suite/test_import06.py @@ -87,7 +87,7 @@ class test_import06(test_import_base): extlist.extension('encryptors', self.encryptor) def conn_config(self): - return 'cache_size=50MB,log=(enabled),statistics=(all),encryption=(name={})'.format( + return 'cache_size=50MB,log=(enabled),encryption=(name={})'.format( self.encryptor + self.encryptor_args) def test_import_repair(self): diff --git a/src/third_party/wiredtiger/test/suite/test_import08.py b/src/third_party/wiredtiger/test/suite/test_import08.py index bac4dc8f431..ed5578d02d2 100644 --- a/src/third_party/wiredtiger/test/suite/test_import08.py +++ b/src/third_party/wiredtiger/test/suite/test_import08.py @@ -34,7 +34,7 @@ from test_import01 import test_import_base from wtscenario import make_scenarios class test_import08(test_import_base): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' original_db_file = 'original_db_file' diff --git a/src/third_party/wiredtiger/test/suite/test_import09.py b/src/third_party/wiredtiger/test/suite/test_import09.py index a41aff32571..cc13688d7a1 100644 --- a/src/third_party/wiredtiger/test/suite/test_import09.py +++ b/src/third_party/wiredtiger/test/suite/test_import09.py @@ -100,7 +100,7 @@ class test_import09(test_import_base): extlist.extension('encryptors', self.encryptor) def conn_config(self): - return 'cache_size=50MB,log=(enabled),statistics=(all),encryption=(name={})'.format( + return 'cache_size=50MB,log=(enabled),encryption=(name={})'.format( self.encryptor + self.encryptor_args) def test_import_table_repair(self): diff --git a/src/third_party/wiredtiger/test/suite/test_prepare02.py b/src/third_party/wiredtiger/test/suite/test_prepare02.py index 60e61376c14..9f796d6a50e 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare02.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare02.py @@ -33,9 +33,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_prepare02(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' @@ -91,9 +88,6 @@ class test_prepare02(wttest.WiredTigerTestCase, suite_subprocess): self.assertTimestampsEqual(self.session.query_timestamp('get=prepare'), '2a') self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda:self.session.checkpoint(), msg) - # WT_SESSION.transaction_pinned_range permitted, not supported in the Python API. - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda:self.session.transaction_sync(), msg) self.session.breakpoint() # Commit the transaction. Test that no "not permitted in a prepared transaction" error has diff --git a/src/third_party/wiredtiger/test/suite/test_prepare05.py b/src/third_party/wiredtiger/test/suite/test_prepare05.py index c5971da32fa..c92a079bf6c 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare05.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare05.py @@ -32,33 +32,38 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_prepare05' uri = 'table:' + tablename session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_timestamp_api(self): - self.session.create(self.uri, 'key_format=i,value_format=i') + self.session.create(self.uri, 'key_format={},value_format=i'.format(self.key_format)) c = self.session.open_cursor(self.uri) # It is illegal to set a prepare timestamp older than oldest timestamp. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(1)), + 'prepare_timestamp=' + self.timestamp_str(1)), "/older than the oldest timestamp/") self.session.rollback_transaction() # Check setting the prepare timestamp same as oldest timestamp is valid. self.session.begin_transaction() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(2)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(3)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(3)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(2)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(3)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(3)) self.session.commit_transaction() # In a single transaction it is illegal to set a commit timestamp @@ -67,10 +72,10 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): # prepare itself is illegal. self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(3)) + 'commit_timestamp=' + self.timestamp_str(3)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(2)), + 'prepare_timestamp=' + self.timestamp_str(2)), "/should not have been set before/") self.session.rollback_transaction() @@ -79,11 +84,11 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): # Start a new reader to have an active read timestamp. if wiredtiger.diagnostic_build(): s_reader = self.conn.open_session() - s_reader.begin_transaction('read_timestamp=' + timestamp_str(4)) + s_reader.begin_transaction('read_timestamp=' + self.timestamp_str(4)) self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(4)), + 'prepare_timestamp=' + self.timestamp_str(4)), "/must be greater than the latest active read timestamp/") self.session.rollback_transaction() @@ -92,7 +97,7 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[1] = 1 self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(5)) + 'prepare_timestamp=' + self.timestamp_str(5)) # Resolve the reader transaction started earlier. s_reader.rollback_transaction() self.session.rollback_transaction() @@ -103,10 +108,10 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): # It is illegal to set a commit timestamp older than prepare timestamp of a transaction. self.session.begin_transaction() c[1] = 1 - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(5)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(5)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(4)), + 'commit_timestamp=' + self.timestamp_str(4)), "/less than the prepare timestamp/") ''' @@ -115,9 +120,9 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[1] = 1 self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(5)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(5)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(5)) + 'prepare_timestamp=' + self.timestamp_str(5)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(5)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(5)) self.session.commit_transaction() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_prepare06.py b/src/third_party/wiredtiger/test/suite/test_prepare06.py index 58361bb39a2..b4e65f183b0 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare06.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare06.py @@ -32,36 +32,41 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_prepare06' uri = 'table:' + tablename session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_timestamp_api(self): - self.session.create(self.uri, 'key_format=i,value_format=i') + self.session.create(self.uri, 'key_format={},value_format=i'.format(self.key_format)) c = self.session.open_cursor(self.uri) # It is illegal to set the prepare timestamp older than the oldest # timestamp. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(20)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(10)), + 'prepare_timestamp=' + self.timestamp_str(10)), "/older than the oldest timestamp/") self.session.rollback_transaction() # Check setting a prepared transaction timestamps earlier than the # oldest timestamp is valid with roundup_timestamps settings. self.session.begin_transaction('roundup_timestamps=(prepared=true)') - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(15)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(35)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(15)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(35)) self.session.commit_transaction() ''' @@ -71,11 +76,11 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): # oldest timestamp is invalid, if durable timestamp is less than the # stable timestamp. self.session.begin_transaction('roundup_timestamps=(prepared=true)') - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(15)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(25)), + 'durable_timestamp=' + self.timestamp_str(25)), "/is less than the stable timestamp/") self.session.rollback_transaction() ''' @@ -83,7 +88,7 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): # Check the cases with an active reader. # Start a new reader to have an active read timestamp. s_reader = self.conn.open_session() - s_reader.begin_transaction('read_timestamp=' + timestamp_str(40)) + s_reader.begin_transaction('read_timestamp=' + self.timestamp_str(40)) # It is illegal to set the prepare timestamp as earlier than an active # read timestamp even with roundup_timestamps settings. This is only @@ -92,7 +97,7 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction('roundup_timestamps=(prepared=true)') self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(10)), + 'prepare_timestamp=' + self.timestamp_str(10)), "/must be greater than the latest active read timestamp/") self.session.rollback_transaction() @@ -101,7 +106,7 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction('roundup_timestamps=(prepared=true)') self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(40)), + 'prepare_timestamp=' + self.timestamp_str(40)), "/must be greater than the latest active read timestamp/") self.session.rollback_transaction() @@ -112,10 +117,10 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): # timestamp of a transaction. self.session.begin_transaction() c[1] = 1 - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(45)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(45)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(30)), + 'commit_timestamp=' + self.timestamp_str(30)), "/less than the prepare timestamp/") ''' @@ -127,12 +132,12 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction('roundup_timestamps=(prepared=true)') c[1] = 1 self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(45)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(30)) - #self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(30)) + 'prepare_timestamp=' + self.timestamp_str(45)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) + #self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(30)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'durable_timestamp=' + timestamp_str(30)), + 'durable_timestamp=' + self.timestamp_str(30)), "/is less than the commit timestamp/") self.session.rollback_transaction() ''' diff --git a/src/third_party/wiredtiger/test/suite/test_prepare07.py b/src/third_party/wiredtiger/test/suite/test_prepare07.py index c53f6aaa262..49d3544cf40 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare07.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare07.py @@ -30,9 +30,7 @@ import fnmatch, os, shutil, time from helper import copy_wiredtiger_home import wiredtiger, wttest from wtdataset import SimpleDataSet - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare07.py # Test to ensure prepared updates older than oldest timestamp are not visible. @@ -43,10 +41,17 @@ class test_prepare07(wttest.WiredTigerTestCase): # Force a small cache. conn_config = 'cache_size=50MB' + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(key_format_values) + def older_prepare_updates(self, uri, ds, nrows, value_a): # Commit some updates along with a prepared update, which is not resolved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100)) # Commit some updates. value_b = b"bbbbb" * 100 @@ -55,12 +60,12 @@ class test_prepare07(wttest.WiredTigerTestCase): cursor.set_key(ds.key(nrows + 1)) cursor.set_value(value_b) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(110)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(110)) self.session.begin_transaction('isolation=snapshot') cursor.set_key(ds.key(nrows + 2)) cursor.set_value(value_b) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(120)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(120)) # Prepare a transaction and keep it open. session_p = self.conn.open_session() @@ -69,36 +74,36 @@ class test_prepare07(wttest.WiredTigerTestCase): cursor_p.set_key(ds.key(nrows + 3)) cursor_p.set_value(value_b) self.assertEquals(cursor_p.update(), 0) - session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(130)) + session_p.prepare_transaction('prepare_timestamp=' + self.timestamp_str(130)) # Commit some more updates. self.session.begin_transaction('isolation=snapshot') cursor.set_key(ds.key(nrows + 4)) cursor.set_value(value_b) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(140)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(140)) self.session.begin_transaction('isolation=snapshot') cursor.set_key(ds.key(nrows + 5)) cursor.set_value(value_b) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(150)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(150)) # Move the oldest and the stable timestamp to the latest. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(155)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(155)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(155)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(155)) # Commit an update newer than the stable timestamp. self.session.begin_transaction('isolation=snapshot') cursor.set_key(ds.key(nrows + 6)) cursor.set_value(value_b) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(160)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(160)) # Take a checkpoint here, so that prepared transaction will not be durable.. self.session.checkpoint() # Commit the prepared transaction. - session_p.commit_transaction('commit_timestamp=' + timestamp_str(140) + ',durable_timestamp=' + timestamp_str(160)) + session_p.commit_transaction('commit_timestamp=' + self.timestamp_str(140) + ',durable_timestamp=' + self.timestamp_str(160)) # Take a backup and check the values. self.backup_dir = os.path.join(self.home, "WT_BACKUP") @@ -143,7 +148,7 @@ class test_prepare07(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test" nrows = 100 - ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u') + ds = SimpleDataSet(self, uri, nrows, key_format=self.key_format, value_format='u') ds.populate() value_a = b"aaaaa" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_prepare08.py b/src/third_party/wiredtiger/test/suite/test_prepare08.py index 76a831b0a05..05ad8ab287e 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare08.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare08.py @@ -30,9 +30,7 @@ import fnmatch, os, shutil, time from helper import copy_wiredtiger_home import wiredtiger, wttest from wtdataset import SimpleDataSet - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare08.py # Test to ensure prepared tombstones are properly aborted/committed even when they are written @@ -41,6 +39,13 @@ class test_prepare08(wttest.WiredTigerTestCase): # Force a small cache. conn_config = 'cache_size=10MB,eviction_dirty_trigger=80,eviction_updates_trigger=80' + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(key_format_values) + def updates(self, ds, uri, nrows, value, ts): cursor = self.session.open_cursor(uri) self.session.begin_transaction('isolation=snapshot') @@ -48,7 +53,7 @@ class test_prepare08(wttest.WiredTigerTestCase): cursor.set_key(ds.key(i)) cursor.set_value(value) self.assertEquals(cursor.update(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def removes(self, ds, uri, nrows, ts): @@ -57,7 +62,7 @@ class test_prepare08(wttest.WiredTigerTestCase): for i in range(1, nrows): cursor.set_key(ds.key(i)) self.assertEquals(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def check(self, ds, uri, nrows, value, ts, release_evict): @@ -65,7 +70,7 @@ class test_prepare08(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") else: cursor = self.session.open_cursor(uri) - self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + timestamp_str(ts)) + self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts)) for i in range(1, nrows): cursor.set_key(ds.key(i)) if value == None: @@ -82,11 +87,11 @@ class test_prepare08(wttest.WiredTigerTestCase): # Create a small table. uri_1 = "table:test_prepare08_1" - ds_1 = SimpleDataSet(self, uri_1, 0, key_format="S", value_format='u') + ds_1 = SimpleDataSet(self, uri_1, 0, key_format=self.key_format, value_format='u') ds_1.populate() uri_2 = "table:test_prepare08_2" - ds_2 = SimpleDataSet(self, uri_2, 0, key_format="S", value_format='u') + ds_2 = SimpleDataSet(self, uri_2, 0, key_format=self.key_format, value_format='u') ds_2.populate() value_a = b"aaaaa" * 100 @@ -96,8 +101,8 @@ class test_prepare08(wttest.WiredTigerTestCase): value_e = b"eeeee" * 100 # Commit some updates along with a prepared update, which is not resolved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Initially load huge data self.updates(ds_1, uri_1, nrows, value_a, 20) @@ -124,7 +129,7 @@ class test_prepare08(wttest.WiredTigerTestCase): for i in range(1, nrows): cursor_p.set_key(ds_1.key(i)) self.assertEquals(cursor_p.remove(), 0) - session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) + session_p.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) # Adding more updates to other table should trigger eviction on uri_1 self.updates(ds_2, uri_2, nrows, value_c, 40) @@ -150,12 +155,12 @@ class test_prepare08(wttest.WiredTigerTestCase): # Create a small table. uri_1 = "table:test_prepare10_1" - ds_1 = SimpleDataSet(self, uri_1, 0, key_format="S", value_format='u') + ds_1 = SimpleDataSet(self, uri_1, 0, key_format=self.key_format, value_format='u') ds_1.populate() # Create another small table. uri_2 = "table:test_prepare10_2" - ds_2 = SimpleDataSet(self, uri_2, 0, key_format="S", value_format='u') + ds_2 = SimpleDataSet(self, uri_2, 0, key_format=self.key_format, value_format='u') ds_2.populate() value_a = b"aaaaa" * 100 @@ -165,8 +170,8 @@ class test_prepare08(wttest.WiredTigerTestCase): value_e = b"eeeee" * 100 # Commit some updates along with a prepared update, which is not resolved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Initially load huge data self.updates(ds_1, uri_1, nrows, value_a, 20) @@ -196,7 +201,7 @@ class test_prepare08(wttest.WiredTigerTestCase): self.assertEquals(cursor_p.update(), 0) cursor_p.set_key(ds_1.key(i)) self.assertEquals(cursor_p.remove(), 0) - session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) + session_p.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) # Adding more updates to other table should trigger eviction on uri_1 self.updates(ds_2, uri_2, nrows, value_c, 40) @@ -208,7 +213,7 @@ class test_prepare08(wttest.WiredTigerTestCase): self.check(ds_1, uri_1, nrows, value_b, 50, True) # Commit the prepared session - session_p.commit_transaction('commit_timestamp=' + timestamp_str(50) + ',durable_timestamp=' + timestamp_str(60)) + session_p.commit_transaction('commit_timestamp=' + self.timestamp_str(50) + ',durable_timestamp=' + self.timestamp_str(60)) self.check(ds_1, uri_1, nrows, value_a, 20, False) self.check(ds_1, uri_1, nrows, value_b, 30, False) @@ -224,12 +229,12 @@ class test_prepare08(wttest.WiredTigerTestCase): # Create a small table. uri_1 = "table:test_prepare10_1" - ds_1 = SimpleDataSet(self, uri_1, 0, key_format="S", value_format='u') + ds_1 = SimpleDataSet(self, uri_1, 0, key_format=self.key_format, value_format='u') ds_1.populate() # Create another small table. uri_2 = "table:test_prepare10_2" - ds_2 = SimpleDataSet(self, uri_2, 0, key_format="S", value_format='u') + ds_2 = SimpleDataSet(self, uri_2, 0, key_format=self.key_format, value_format='u') ds_2.populate() value_a = b"aaaaa" * 100 @@ -239,8 +244,8 @@ class test_prepare08(wttest.WiredTigerTestCase): value_e = b"eeeee" * 100 # Commit some updates along with a prepared update, which is not resolved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Initially load huge data self.updates(ds_1, uri_1, nrows, value_a, 20) @@ -266,7 +271,7 @@ class test_prepare08(wttest.WiredTigerTestCase): self.assertEquals(cursor_p.update(), 0) cursor_p.set_key(ds_1.key(i)) self.assertEquals(cursor_p.remove(), 0) - session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) + session_p.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) # Adding more updates to other table should trigger eviction on uri_1 self.updates(ds_2, uri_2, nrows, value_c, 40) @@ -278,7 +283,7 @@ class test_prepare08(wttest.WiredTigerTestCase): self.check(ds_1, uri_1, 0, None, 50, True) # Commit the prepared session - session_p.commit_transaction('commit_timestamp=' + timestamp_str(50) + ',durable_timestamp=' + timestamp_str(60)) + session_p.commit_transaction('commit_timestamp=' + self.timestamp_str(50) + ',durable_timestamp=' + self.timestamp_str(60)) self.check(ds_1, uri_1, nrows, value_a, 20, False) self.check(ds_1, uri_1, 0, None, 30, False) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare09.py b/src/third_party/wiredtiger/test/suite/test_prepare09.py index 3c0aa546d08..8aaacf1c3fb 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare09.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare09.py @@ -31,24 +31,30 @@ # [END_TAGS] import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare09.py # Validate scenarios involving inserting tombstones when rolling back prepares class test_prepare09(wttest.WiredTigerTestCase): - conn_config = 'cache_size=2MB,statistics=(all)' + conn_config = 'cache_size=2MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_prepared_update_is_aborted_correctly_with_on_disk_value(self): uri = "table:test_prepare09" - create_params = 'value_format=S,key_format=i' + create_params = 'value_format=S,key_format={}'.format(self.key_format) value1 = 'a' * 10000 value2 = 'b' * 10000 value3 = 'c' * 10000 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) self.session.create(uri, create_params) cursor = self.session.open_cursor(uri) @@ -58,7 +64,7 @@ class test_prepare09(wttest.WiredTigerTestCase): # Insert a new value. self.session.begin_transaction() cursor[1] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Get the previous update onto disk. for i in range(2, 10000): @@ -68,7 +74,7 @@ class test_prepare09(wttest.WiredTigerTestCase): # Prepare a full value and roll it back. self.session.begin_transaction() cursor[1] = value3 - self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(3)), 0) + self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(3)), 0) self.session.rollback_transaction() # Get the previous update onto disk. @@ -89,8 +95,8 @@ class test_prepare09(wttest.WiredTigerTestCase): value2 = 'b' * 10000 value3 = 'e' * 10000 value4 = 'd' * 10000 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) self.session.create(uri, create_params) cursor = self.session.open_cursor(uri) @@ -102,7 +108,7 @@ class test_prepare09(wttest.WiredTigerTestCase): cursor[1] = value1 cursor[2] = value2 cursor[3] = value3 - self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(3)), 0) + self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(3)), 0) # Get the prepare onto disk. for i in range(4, 10000): diff --git a/src/third_party/wiredtiger/test/suite/test_prepare10.py b/src/third_party/wiredtiger/test/suite/test_prepare10.py index a9ae73848bd..a4eec9bfcc4 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare10.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare10.py @@ -30,9 +30,7 @@ import fnmatch, os, shutil, time from helper import copy_wiredtiger_home import wiredtiger, wttest from wtdataset import SimpleDataSet - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare10.py # Test to ensure prepared tombstones are properly aborted even when they are written @@ -42,6 +40,13 @@ class test_prepare10(wttest.WiredTigerTestCase): conn_config = 'cache_size=10MB,eviction_dirty_trigger=80,eviction_updates_trigger=80' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(key_format_values) + def updates(self, ds, uri, nrows, value, ts): cursor = self.session.open_cursor(uri) self.session.begin_transaction() @@ -49,7 +54,7 @@ class test_prepare10(wttest.WiredTigerTestCase): cursor.set_key(ds.key(i)) cursor.set_value(value) self.assertEquals(cursor.insert(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def removes(self, ds, uri, nrows, ts): @@ -58,12 +63,12 @@ class test_prepare10(wttest.WiredTigerTestCase): for i in range(1, nrows): cursor.set_key(ds.key(i)) self.assertEquals(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def check(self, ds, uri, nrows, value, ts): cursor = self.session.open_cursor(uri) - self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + timestamp_str(ts)) + self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts)) for i in range(1, nrows): cursor.set_key(ds.key(i)) self.assertEquals(cursor.search(), 0) @@ -73,7 +78,7 @@ class test_prepare10(wttest.WiredTigerTestCase): def check_not_found(self, ds, uri, nrows, ts): cursor = self.session.open_cursor(uri) - self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + timestamp_str(ts)) + self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts)) for i in range(1, nrows): cursor.set_key(ds.key(i)) self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) @@ -84,7 +89,7 @@ class test_prepare10(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test_prepare10" nrows = 1000 - ds = SimpleDataSet(self, uri, 0, key_format="S", value_format='u') + ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format='u') ds.populate() value_a = b"aaaaa" * 100 @@ -92,8 +97,8 @@ class test_prepare10(wttest.WiredTigerTestCase): value_c = b"ccccc" * 100 # Commit some updates along with a prepared update, which is not resolved. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Initially load huge data self.updates(ds, uri, nrows, value_a, 20) @@ -141,7 +146,7 @@ class test_prepare10(wttest.WiredTigerTestCase): cursor_p.set_key(ds.key(i)) cursor_p.set_value(value_c) self.assertEquals(cursor_p.insert(), 0) - session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(50)) + session_p.prepare_transaction('prepare_timestamp=' + self.timestamp_str(50)) self.check(ds, uri, nrows, value_a, 20) self.check(ds, uri, nrows, value_b, 35) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare11.py b/src/third_party/wiredtiger/test/suite/test_prepare11.py index 4b35a173576..5770d9ad2f2 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare11.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare11.py @@ -29,38 +29,40 @@ import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_prepare11.py # Test prepare rollback with a reserved update between updates. class test_prepare11(wttest.WiredTigerTestCase): - conn_config = 'cache_size=2MB,statistics=(all)' + conn_config = 'cache_size=2MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', key1=17)), + ('string-row', dict(key_format='S', key1='key1')), + ] + commit_values = [ ('commit', dict(commit=True)), ('rollback', dict(commit=False)), ] - scenarios = make_scenarios(commit_values) + scenarios = make_scenarios(key_format_values, commit_values) def test_prepare_update_rollback(self): uri = "table:test_prepare11" - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) self.session.begin_transaction("isolation=snapshot") # In the scenario where we have a reserved update in between two updates, the key repeated # flag won't get set and we'll call resolve prepared op on both prepared updates. c = self.session.open_cursor(uri, None) - c['key1'] = 'xxxx' - c.set_key('key1') + c[self.key1] = 'xxxx' + c.set_key(self.key1) c.reserve() - c['key1'] = 'yyyy' + c[self.key1] = 'yyyy' self.session.prepare_transaction('prepare_timestamp=10') if self.commit: - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(20)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(30)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(20)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(30)) self.session.commit_transaction() else: self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare12.py b/src/third_party/wiredtiger/test/suite/test_prepare12.py index 55d6c69c2d7..f559bde9c1b 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare12.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare12.py @@ -31,42 +31,48 @@ # [END_TAGS] import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare12.py # Test update restore of a page with prepared update. class test_prepare12(wttest.WiredTigerTestCase): - conn_config = 'cache_size=2MB,statistics=(all)' + conn_config = 'cache_size=2MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_prepare_update_restore(self): uri = "table:test_prepare12" - self.session.create(uri, 'key_format=i,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) # Prepare a transaction cursor = self.session.open_cursor(uri, None) self.session.begin_transaction() - cursor[0] = 'a' - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(1)) + cursor[1] = 'a' + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(1)) # Insert an uncommitted key session2 = self.conn.open_session(None) cursor2 = session2.open_cursor(uri, None) session2.begin_transaction() - cursor2[1] = 'b' + cursor2[2] = 'b' # Insert a bunch of other content to fill the database to trigger eviction. session3 = self.conn.open_session(None) cursor3 = session3.open_cursor(uri, None) - for i in range(2, 100): + for i in range(3, 101): session3.begin_transaction() cursor3[i] = 'a' * 500 session3.commit_transaction() # Commit the prepared update - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1) + ',durable_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1) + ',durable_timestamp=' + self.timestamp_str(2)) # Read the prepared update - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) - self.assertEqual(cursor[0], 'a') + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) + self.assertEqual(cursor[1], 'a') diff --git a/src/third_party/wiredtiger/test/suite/test_prepare13.py b/src/third_party/wiredtiger/test/suite/test_prepare13.py index 8542a182328..fbb735ce4f1 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare13.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare13.py @@ -34,22 +34,29 @@ # Fast-truncate fails when a page contains prepared updates. import wiredtiger, wttest from wtdataset import simple_key, simple_value +from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t class test_prepare13(wttest.WiredTigerTestCase): # Force a small cache. - conn_config = 'cache_size=10MB,statistics=(all),statistics_log=(json,on_close,wait=1)' + conn_config = 'cache_size=10MB' + + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(key_format_values) def test_prepare(self): nrows = 20000 # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) # Create a large table with lots of pages. uri = "table:test_prepare13" - config = 'allocation_size=512,leaf_page_max=512,key_format=S,value_format=S' + key_format_str = "key_format=" + self.key_format + config = 'allocation_size=512,leaf_page_max=512,{},value_format=S'.format(key_format_str) self.session.create(uri, config) cursor = self.session.open_cursor(uri) for i in range(1, nrows): @@ -61,12 +68,12 @@ class test_prepare13(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri) cursor[simple_key(cursor, 1000)] = "replacement_value" cursor.close() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)) try: # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) # Open a separate session and cursor and perform updates to let prepared update to evict. s = self.conn.open_session() @@ -74,7 +81,7 @@ class test_prepare13(wttest.WiredTigerTestCase): for i in range(2000, nrows): s.begin_transaction() cursor[simple_key(cursor, i)] = simple_value(cursor, i) - s.commit_transaction('commit_timestamp=' + timestamp_str(20)) + s.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) cursor.close() # Truncate the middle chunk and expect a conflict. @@ -90,8 +97,8 @@ class test_prepare13(wttest.WiredTigerTestCase): s.rollback_transaction() finally: - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(50)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(50)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(50)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(50)) self.session.commit_transaction() s.close() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare14.py b/src/third_party/wiredtiger/test/suite/test_prepare14.py index fb32aefc713..71737907526 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare14.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare14.py @@ -28,11 +28,9 @@ import wttest from wiredtiger import WT_NOTFOUND +from wtdataset import simple_key from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_prepare14.py # Test that the transaction visibility of an on-disk update # that has both the start and the stop time points from the @@ -61,37 +59,34 @@ class test_prepare14(wttest.WiredTigerTestCase): return config def test_prepare14(self): - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:prepare14" - create_config = 'allocation_size=512,key_format=S,value_format=S' + create_config = 'allocation_size=512,key_format={},value_format=S'.format(self.key_format) self.session.create(uri, create_config) # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value = 'a' - # Perform several updates and removes. + # Perform an update and a remove. s = self.conn.open_session() cursor = s.open_cursor(uri) s.begin_transaction() - cursor[str(0)] = value - cursor.set_key(str(0)) + key = simple_key(cursor, 1) + cursor[key] = value + cursor.set_key(key) cursor.remove() cursor.close() - s.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") # Search for the key so we position our cursor on the page that we want to evict. self.session.begin_transaction("ignore_prepare = true") - evict_cursor.set_key(str(0)) + evict_cursor.set_key(key) self.assertEquals(evict_cursor.search(), WT_NOTFOUND) evict_cursor.reset() evict_cursor.close() @@ -99,6 +94,6 @@ class test_prepare14(wttest.WiredTigerTestCase): self.session.begin_transaction("ignore_prepare = true") cursor2 = self.session.open_cursor(uri) - cursor2.set_key(str(0)) + cursor2.set_key(key) self.assertEquals(cursor2.search(), WT_NOTFOUND) self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare15.py b/src/third_party/wiredtiger/test/suite/test_prepare15.py index 4c4ba49a182..b4db822caf6 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare15.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare15.py @@ -30,9 +30,6 @@ import wttest from wiredtiger import WT_NOTFOUND from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_prepare15.py # Test that the prepare transaction rollback removes the on-disk key # or replace it with history store and commit retains the changes when @@ -64,18 +61,14 @@ class test_prepare15(wttest.WiredTigerTestCase): return config def test_prepare_restore_hs_update(self): - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:prepare15" create_config = 'allocation_size=512,key_format=S,value_format=S' self.session.create(uri, create_config) # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = 'a' valueb = 'a' @@ -84,12 +77,12 @@ class test_prepare15(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri) self.session.begin_transaction() cursor[str(0)] = valuea - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) self.session.begin_transaction() cursor.set_key(str(0)) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) cursor.close() # Perform an update and remove. @@ -100,7 +93,7 @@ class test_prepare15(wttest.WiredTigerTestCase): cursor.set_key(str(0)) cursor.remove() cursor.close() - s.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) + s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") @@ -115,8 +108,8 @@ class test_prepare15(wttest.WiredTigerTestCase): if self.commit: # Commit the prepared transaction - s.timestamp_transaction('commit_timestamp=' + timestamp_str(50)) - s.timestamp_transaction('durable_timestamp=' + timestamp_str(60)) + s.timestamp_transaction('commit_timestamp=' + self.timestamp_str(50)) + s.timestamp_transaction('durable_timestamp=' + self.timestamp_str(60)) s.commit_transaction() else: # Rollback the prepared transaction @@ -133,7 +126,7 @@ class test_prepare15(wttest.WiredTigerTestCase): evict_cursor.close() self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(20)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) cursor2 = self.session.open_cursor(uri) cursor2.set_key(str(0)) self.assertEquals(cursor2.search(), 0) @@ -141,18 +134,14 @@ class test_prepare15(wttest.WiredTigerTestCase): self.session.commit_transaction() def test_prepare_not_found(self): - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:prepare15" create_config = 'allocation_size=512,key_format=S,value_format=S' self.session.create(uri, create_config) # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value = 'a' @@ -164,7 +153,7 @@ class test_prepare15(wttest.WiredTigerTestCase): cursor.set_key(str(0)) cursor.remove() cursor.close() - s.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") @@ -179,8 +168,8 @@ class test_prepare15(wttest.WiredTigerTestCase): if self.commit: # Commit the prepared transaction - s.timestamp_transaction('commit_timestamp=' + timestamp_str(30)) - s.timestamp_transaction('durable_timestamp=' + timestamp_str(40)) + s.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) + s.timestamp_transaction('durable_timestamp=' + self.timestamp_str(40)) s.commit_transaction() else: # Rollback the prepared transaction diff --git a/src/third_party/wiredtiger/test/suite/test_prepare16.py b/src/third_party/wiredtiger/test/suite/test_prepare16.py index 23eb85bf444..f6e05aadfa9 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare16.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare16.py @@ -30,9 +30,6 @@ import wttest from wiredtiger import WT_NOTFOUND from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_prepare16.py # Test that the prepare transaction rollback/commit multiple keys # and each key can occupy a leaf page. @@ -65,18 +62,14 @@ class test_prepare16(wttest.WiredTigerTestCase): def test_prepare(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:prepare16" create_config = 'allocation_size=512,key_format=S,value_format=S,leaf_page_max=512,leaf_value_max=64MB' self.session.create(uri, create_config) # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = 'a' * 400 @@ -87,7 +80,7 @@ class test_prepare16(wttest.WiredTigerTestCase): cursor.reset() cursor.close() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)) s = self.conn.open_session() s.begin_transaction('ignore_prepare = true') @@ -100,17 +93,17 @@ class test_prepare16(wttest.WiredTigerTestCase): evict_cursor.reset() if self.commit: - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(20)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(30)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(20)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(30)) self.session.commit_transaction() else: self.session.rollback_transaction() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) if not self.in_memory: self.session.checkpoint() - self.session.begin_transaction('read_timestamp=' + timestamp_str(20)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) cursor = self.session.open_cursor(uri) for i in range(1, nrows + 1): cursor.set_key(str(i)) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py b/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py index 3f298e63b9b..d62ee17b2a2 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py @@ -32,15 +32,21 @@ import wiredtiger, wttest from wtdataset import simple_key, simple_value - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_prepare_conflict(wttest.WiredTigerTestCase): + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_prepare(self): # Create a large table with lots of pages. uri = "table:test_prepare_conflict" - config = 'allocation_size=512,leaf_page_max=512,key_format=S,value_format=S' + key_format = 'key_format=' + self.key_format + config = 'allocation_size=512,leaf_page_max=512,{},value_format=S'.format(key_format) self.session.create(uri, config) cursor = self.session.open_cursor(uri) for i in range(1, 80000): @@ -68,9 +74,9 @@ class test_prepare_conflict(wttest.WiredTigerTestCase): cursor.close() # Prepare and commit the transaction. - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(20)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(20)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(20)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(20)) self.session.commit_transaction() # WT-6325 reports WT_PREPARE_CONFLICT while iterating the cursor. diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py b/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py index aa2e45bec32..579927bd38e 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py @@ -35,9 +35,6 @@ from wtdataset import SimpleDataSet, SimpleIndexDataSet from wtdataset import SimpleLSMDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' %t - # test_prepare_cursor01.py # WT_CURSOR navigation (next/prev) tests with prepared transactions class test_prepare_cursor01(wttest.WiredTigerTestCase): @@ -103,18 +100,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_cursor.set_key(ds.key(51)) prep_cursor.set_value(ds.value(51)) prep_cursor.insert() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(100)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(100)) # Point all cursors to key 50. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(50)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(50)) before_ts_c.set_key(ds.key(50)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(150)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(150)) between_ts_c.set_key(ds.key(50)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(250)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(250)) after_ts_c.set_key(ds.key(50)) self.assertEquals(after_ts_c.search(), 0) @@ -138,8 +135,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.next()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(200)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(200)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(200)) prep_session.commit_transaction() before_ts_s.commit_transaction() @@ -160,18 +157,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_cursor.set_key(ds.key(1)) prep_cursor.set_value(ds.value(1)) prep_cursor.insert() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(100)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(100)) # Point all cursors to key 2. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(50)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(50)) before_ts_c.set_key(ds.key(2)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(150)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(150)) between_ts_c.set_key(ds.key(2)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(250)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(250)) after_ts_c.set_key(ds.key(2)) self.assertEquals(after_ts_c.search(), 0) @@ -196,8 +193,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.prev()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(200)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(200)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(200)) prep_session.commit_transaction() # As read is between(i.e before commit), prev is not found. @@ -223,18 +220,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_cursor.set_key(ds.key(51)) prep_cursor.set_value(ds.value(151)) prep_cursor.update() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(300)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(300)) # Point all cursors to key 51. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(250)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(250)) before_ts_c.set_key(ds.key(50)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(350)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(350)) between_ts_c.set_key(ds.key(50)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(450)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(450)) after_ts_c.set_key(ds.key(50)) self.assertEquals(after_ts_c.search(), 0) @@ -256,8 +253,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.next()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(400)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(400)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(400)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(400)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. @@ -287,18 +284,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_cursor.set_key(ds.key(1)) prep_cursor.set_value(ds.value(111)) prep_cursor.update() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(300)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(300)) # Point all cursors to key 2. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(250)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(250)) before_ts_c.set_key(ds.key(2)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(350)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(350)) between_ts_c.set_key(ds.key(2)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(450)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(450)) after_ts_c.set_key(ds.key(2)) self.assertEquals(after_ts_c.search(), 0) @@ -320,8 +317,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.prev()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(400)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(400)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(400)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(400)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. @@ -355,18 +352,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_session.begin_transaction() prep_cursor.set_key(ds.key(51)) prep_cursor.remove() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(500)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(500)) # Point all cursors to key 51. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(450)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(450)) before_ts_c.set_key(ds.key(50)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(550)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(550)) between_ts_c.set_key(ds.key(50)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(650)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(650)) after_ts_c.set_key(ds.key(50)) self.assertEquals(after_ts_c.search(), 0) @@ -387,8 +384,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.next()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(600)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(600)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(600)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(600)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. @@ -411,18 +408,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_session.begin_transaction() prep_cursor.set_key(ds.key(1)) prep_cursor.remove() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(500)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(500)) # Point all cursors to key 2. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(450)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(450)) before_ts_c.set_key(ds.key(2)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(550)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(550)) between_ts_c.set_key(ds.key(2)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(650)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(650)) after_ts_c.set_key(ds.key(2)) self.assertEquals(after_ts_c.search(), 0) @@ -443,8 +440,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.prev()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(600)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(600)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(600)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(600)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. @@ -474,18 +471,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_session.begin_transaction() prep_cursor.set_key(ds.key(49)) prep_cursor.remove() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(700)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(700)) # Point all cursors to key 48. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(650)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(650)) before_ts_c.set_key(ds.key(48)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(750)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(750)) between_ts_c.set_key(ds.key(48)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(850)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(850)) after_ts_c.set_key(ds.key(48)) self.assertEquals(after_ts_c.search(), 0) @@ -506,8 +503,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.next()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(800)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(800)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(800)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(800)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. @@ -532,18 +529,18 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): prep_session.begin_transaction() prep_cursor.set_key(ds.key(3)) prep_cursor.remove() - prep_session.prepare_transaction('prepare_timestamp=' + timestamp_str(700)) + prep_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(700)) # Point all cursors to key 4. - before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(650)) + before_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(650)) before_ts_c.set_key(ds.key(4)) self.assertEquals(before_ts_c.search(), 0) - between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(750)) + between_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(750)) between_ts_c.set_key(ds.key(4)) self.assertEquals(between_ts_c.search(), 0) - after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + timestamp_str(850)) + after_ts_s.begin_transaction('isolation=' + self.isolation + ',read_timestamp=' + self.timestamp_str(850)) after_ts_c.set_key(ds.key(4)) self.assertEquals(after_ts_c.search(), 0) @@ -564,8 +561,8 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.prev()) # Commit the prepared transaction. - prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(800)) - prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(800)) + prep_session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(800)) + prep_session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(800)) prep_session.commit_transaction() # Check to see before cursor still gets the old value. diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_cursor02.py b/src/third_party/wiredtiger/test/suite/test_prepare_cursor02.py index 65df05fd239..4353ffabf70 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_cursor02.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_cursor02.py @@ -31,9 +31,6 @@ from wtdataset import SimpleDataSet, SimpleIndexDataSet from wtdataset import SimpleLSMDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' %t - # test_prepare_cursor02.py # WT_CURSOR navigation (next/prev) tests with prepared transactions class test_prepare_cursor02(wttest.WiredTigerTestCase): @@ -69,7 +66,7 @@ class test_prepare_cursor02(wttest.WiredTigerTestCase): cursor.set_key(ds.key(1)) cursor.set_value(ds.value(1)) cursor.insert() - session.prepare_transaction('prepare_timestamp=' + timestamp_str(100)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(100)) prep_session = self.conn.open_session(self.session_config) prep_cursor = prep_session.open_cursor(uri, None) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py index c7810e0f719..909f534982e 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py @@ -29,9 +29,7 @@ from helper import copy_wiredtiger_home import wiredtiger, wttest from wtdataset import SimpleDataSet - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios # test_prepare_hs01.py # test to ensure history store eviction is working for prepared transactions. @@ -39,9 +37,16 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): # Force a small cache. conn_config = 'cache_size=50MB,eviction_updates_trigger=95,eviction_updates_target=80' + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(key_format_values) + def check(self, uri, ds, nrows, nsessions, nkeys, read_ts, expected_value, not_expected_value): cursor = self.session.open_cursor(uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) for i in range(1, nsessions * nkeys): cursor.set_key(ds.key(nrows + i)) self.assertEquals(cursor.search(), 0) @@ -67,7 +72,7 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): # memory. Hence testing if we can read prepared updates from the history store. # Start with setting a stable timestamp to pin history in cache - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) # Commit some updates to get eviction and history store fired up bigvalue1 = b"bbbbb" * 100 @@ -77,7 +82,7 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): cursor.set_key(ds.key(nrows + i)) cursor.set_value(bigvalue1) self.assertEquals(cursor.insert(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) # Have prepared updates in multiple sessions. This should ensure writing # prepared updates to the history store @@ -95,7 +100,7 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): cursors[j].set_key(ds.key(nrows + i)) cursors[j].set_value(bigvalue2) self.assertEquals(cursors[j].insert(), 0) - sessions[j].prepare_transaction('prepare_timestamp=' + timestamp_str(2)) + sessions[j].prepare_transaction('prepare_timestamp=' + self.timestamp_str(2)) # Re-read the original versions of all the data. This ensures reading # original versions from the history store @@ -116,7 +121,7 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test_prepare_hs01" nrows = 100 - ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u') + ds = SimpleDataSet(self, uri, nrows, key_format=self.key_format, value_format='u') ds.populate() bigvalue = b"aaaaa" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs02.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs02.py index 5ba68e6a614..6fbd416e94b 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs02.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs02.py @@ -36,9 +36,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_prepare_cursor' uri = 'table:' + tablename @@ -72,16 +69,16 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction(self.txn_config) c[1] = 1 # update the value with in this transaction - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(100)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(100)) if self.txn_commit == True: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(101) + ',durable_timestamp=' + timestamp_str(101)) + 'commit_timestamp=' + self.timestamp_str(101) + ',durable_timestamp=' + self.timestamp_str(101)) else: self.session.rollback_transaction() # Trigger a checkpoint, which could trigger reconciliation - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(150)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(150)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(150)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(150)) self.session.checkpoint() # Scenario: 2 @@ -94,16 +91,16 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): # update a uncommitted value, insert and update a key. c[2] = 1 c[2] = 2 - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(200)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(200)) if self.txn_commit == True: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(201) + ',durable_timestamp=' + timestamp_str(201)) + 'commit_timestamp=' + self.timestamp_str(201) + ',durable_timestamp=' + self.timestamp_str(201)) else: self.session.rollback_transaction() # Trigger a checkpoint, which could trigger reconciliation - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(250)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(250)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(250)) self.session.checkpoint() # Scenario: 3 @@ -121,10 +118,10 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): c[3] = 2 c.set_key(3) c.remove() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(300)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(300)) if self.txn_commit == True: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(301) + ',durable_timestamp=' + timestamp_str(301)) + 'commit_timestamp=' + self.timestamp_str(301) + ',durable_timestamp=' + self.timestamp_str(301)) else: self.session.rollback_transaction() @@ -133,11 +130,11 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): c[1] = 1 c[2] = 1 c[3] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(302)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(302)) # Trigger a checkpoint, which could trigger reconciliation - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(350)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(350)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(350)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(350)) self.session.checkpoint() #Scenario: 4 @@ -145,8 +142,8 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): # creating the modify update_chain for key instead of insert update # chain. self.reopen_conn() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(350)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(350)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(350)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(350)) self.session.create(self.uri, self.s_config) cur = self.session.open_cursor(self.uri) @@ -159,16 +156,16 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): # Remove a updated key cur.set_key(3) cur.remove() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(400)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(400)) if self.txn_commit == True: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(401) + ',durable_timestamp=' + timestamp_str(401)) + 'commit_timestamp=' + self.timestamp_str(401) + ',durable_timestamp=' + self.timestamp_str(401)) else: self.session.rollback_transaction() # Trigger a checkpoint, which could trigger reconciliation - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(450)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(450)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(450)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(450)) self.session.checkpoint() cur.close() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs03.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs03.py index 502a7ba9c8e..c2d6ca66d4e 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs03.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs03.py @@ -38,9 +38,6 @@ import os, shutil from wtscenario import make_scenarios from wiredtiger import stat -def timestamp_str(t): - return '%x' % t - # test_prepare_hs03.py # test to ensure salvage, verify & simulating crash are working for prepared transactions. class test_prepare_hs03(wttest.WiredTigerTestCase): @@ -51,10 +48,17 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test_prepare_hs03" - scenarios = make_scenarios([ + corrupt_values = [ ('corrupt_table', dict(corrupt=True)), ('dont_corrupt_table', dict(corrupt=False)) - ]) + ] + + key_format_values = [ + ('column', dict(key_format='r')), + ('string-row', dict(key_format='S')), + ] + + scenarios = make_scenarios(corrupt_values, key_format_values) def corrupt_table(self): tablename="test_prepare_hs03.wt" @@ -96,12 +100,12 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): cursor.set_key(ds.key(nrows + i)) cursor.set_value(commit_value) self.assertEquals(cursor.insert(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) cursor.close() # Set the stable/oldest timstamps. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) # Corrupt the table, call salvage to recover data from the corrupted table and call verify self.corrupt_salvage_verify() @@ -124,7 +128,7 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): cursors[j].set_key(ds.key(nrows + i)) cursors[j].set_value(prepare_value) self.assertEquals(cursors[j].insert(), 0) - sessions[j].prepare_transaction('prepare_timestamp=' + timestamp_str(4)) + sessions[j].prepare_transaction('prepare_timestamp=' + self.timestamp_str(4)) hs_writes = self.get_stat(stat.conn.cache_write_hs) - hs_writes_start @@ -133,7 +137,7 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): # Test if we can read prepared updates from the history store. cursor = self.session.open_cursor(self.uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(3)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) for i in range(1, nsessions * nkeys): cursor.set_key(ds.key(nrows + i)) # The search should pass. @@ -156,7 +160,7 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): # Finally, search for the keys inserted with commit timestamp cursor = self.session.open_cursor(self.uri) self.pr('Read Keys') - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) for i in range(1, nkeys): cursor.set_key(ds.key(nrows + i)) # The search should pass @@ -177,7 +181,7 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri) # Search the keys inserted with commit timestamp after crash - self.session.begin_transaction('read_timestamp=' + timestamp_str(4)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4)) for i in range(1, nkeys): cursor.set_key(ds.key(nrows + i)) # The search should pass @@ -195,7 +199,7 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): def test_prepare_hs(self): nrows = 100 - ds = SimpleDataSet(self, self.uri, nrows, key_format="S", value_format='u') + ds = SimpleDataSet(self, self.uri, nrows, key_format=self.key_format, value_format='u') ds.populate() bigvalue = b"aaaaa" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py index 40a80ed0b0e..fa74584ab0c 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py @@ -32,9 +32,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from wiredtiger import stat -def timestamp_str(t): - return '%x' % t - # test_prepare_hs04.py # Read prepared updates from on-disk with ignore_prepare. # Committing or aborting a prepared update when there exists a tombstone for that key already. @@ -50,10 +47,18 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): nkeys = 40 nrows = 100 - scenarios = make_scenarios([ + commit_values = [ ('commit_transaction', dict(commit=True)), ('rollback_transaction', dict(commit=False)) - ]) + ] + + key_format_values = [ + # Note: commit_key must exceed nrows to give behavior comparable to the row case. + ('column', dict(key_format='r', commit_key=1000)), + ('string-row', dict(key_format='S', commit_key='C')), + ] + + scenarios = make_scenarios(commit_values, key_format_values) def get_stat(self, stat): stat_cursor = self.session.open_cursor('statistics:') @@ -64,7 +69,7 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): def search_keys_timestamp_and_ignore(self, ds, txn_config, expected_value, conflict=False): cursor = self.session.open_cursor(self.uri) - commit_key = "C" + commit_key = self.commit_key self.session.begin_transaction(txn_config) for i in range(1, self.nsessions * self.nkeys): key = commit_key + ds.key(self.nrows + i) @@ -82,12 +87,12 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): def prepare_updates(self, ds): # Set oldest and stable timestamp for the database. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) # Commit some updates to get eviction and history store fired up. # Insert a key at timestamp 1. - commit_key = "C" + commit_key = self.commit_key commit_value = b"bbbbb" * 100 cursor = self.session.open_cursor(self.uri) for i in range(1, self.nsessions * self.nkeys): @@ -96,7 +101,7 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): cursor.set_key(key) cursor.set_value(commit_value) self.assertEquals(cursor.insert(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) cursor.close() # Call checkpoint. @@ -108,11 +113,11 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): key = commit_key + ds.key(self.nrows + i) cursor.set_key(key) self.assertEquals(cursor.remove(), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) cursor.close() # Move the stable timestamp to match the timestamp for the last update. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) hs_writes_start = self.get_stat(stat.conn.cache_write_hs) # Have prepared updates in multiple sessions. This should ensure writing prepared updates to @@ -131,22 +136,22 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): cursors[j].set_key(commit_key + ds.key(self.nrows + i)) cursors[j].set_value(prepare_value) self.assertEquals(cursors[j].insert(), 0) - sessions[j].prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + sessions[j].prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) hs_writes = self.get_stat(stat.conn.cache_write_hs) - hs_writes_start # Assert if not writing anything to the history store. self.assertGreaterEqual(hs_writes, 0) - txn_config = 'read_timestamp=' + timestamp_str(5) + ',ignore_prepare=true' + txn_config = 'read_timestamp=' + self.timestamp_str(5) + ',ignore_prepare=true' # Search keys with timestamp 5, ignore_prepare=true and expect the cursor search to return 0 (key found) self.search_keys_timestamp_and_ignore(ds, txn_config, commit_value) - txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=true' + txn_config = 'read_timestamp=' + self.timestamp_str(20) + ',ignore_prepare=true' # Search keys with timestamp 20, ignore_prepare=true, expect the cursor to return wiredtiger.WT_NOTFOUND self.search_keys_timestamp_and_ignore(ds, txn_config, None) prepare_conflict_msg = '/conflict with a prepared update/' - txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=false' + txn_config = 'read_timestamp=' + self.timestamp_str(20) + ',ignore_prepare=false' # Search keys with timestamp 20, ignore_prepare=false and expect the cursor the cursor search to return prepare conflict message self.search_keys_timestamp_and_ignore(ds, txn_config, prepare_conflict_msg, True) @@ -156,9 +161,9 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): # Commit the prepared_transactions with timestamp 30. for j in range (0, self.nsessions): sessions[j].commit_transaction( - 'commit_timestamp=' + timestamp_str(30) + ',durable_timestamp=' + timestamp_str(30)) + 'commit_timestamp=' + self.timestamp_str(30) + ',durable_timestamp=' + self.timestamp_str(30)) # Move the stable timestamp to match the durable timestamp for prepared updates. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) self.session.checkpoint() @@ -174,44 +179,44 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): # Search keys with timestamp 5, ignore_prepare=true and expect the cursor search to return # value committed before prepared update. - txn_config = 'read_timestamp=' + timestamp_str(5) + ',ignore_prepare=false' + txn_config = 'read_timestamp=' + self.timestamp_str(5) + ',ignore_prepare=false' self.search_keys_timestamp_and_ignore(ds, txn_config, commit_value) # Search keys with timestamp 20, ignore_prepare=true and expect the cursor search to return # WT_NOTFOUND. - txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=true' + txn_config = 'read_timestamp=' + self.timestamp_str(20) + ',ignore_prepare=true' self.search_keys_timestamp_and_ignore(ds, txn_config, None) # Search keys with timestamp 20, ignore_prepare=false and expect the cursor search to return WT_NOTFOUND. - txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=false' + txn_config = 'read_timestamp=' + self.timestamp_str(20) + ',ignore_prepare=false' self.search_keys_timestamp_and_ignore(ds, txn_config, None) # If commit is true then the commit_tramsactions was called and we will expect prepare_value. if self.commit == True: - txn_config = 'read_timestamp=' + timestamp_str(30) + ',ignore_prepare=true' + txn_config = 'read_timestamp=' + self.timestamp_str(30) + ',ignore_prepare=true' # Search keys with timestamp 30, ignore_prepare=true and expect the cursor value to be prepare_value. self.search_keys_timestamp_and_ignore(ds, txn_config, prepare_value) else: # Commit is false and we simulated a crash/restart which would have rolled-back the transactions, therefore we expect the # cursor search to return WT_NOTFOUND. - txn_config = 'read_timestamp=' + timestamp_str(30) + ',ignore_prepare=true' + txn_config = 'read_timestamp=' + self.timestamp_str(30) + ',ignore_prepare=true' # Search keys with timestamp 30, ignore_prepare=true and expect the cursor value to return WT_NOTFOUND. self.search_keys_timestamp_and_ignore(ds, txn_config, None) if self.commit == True: - txn_config = 'read_timestamp=' + timestamp_str(30) + ',ignore_prepare=false' + txn_config = 'read_timestamp=' + self.timestamp_str(30) + ',ignore_prepare=false' # Search keys with timestamp 30, ignore_prepare=false and expect the cursor value to be prepare_value. self.search_keys_timestamp_and_ignore(ds, txn_config, prepare_value) else: # Commit is false and we simulated a crash/restart which would have rolled-back the transactions, therefore we expect the # cursor search to return WT_NOTFOUND. - txn_config = 'read_timestamp=' + timestamp_str(30) + ',ignore_prepare=false' + txn_config = 'read_timestamp=' + self.timestamp_str(30) + ',ignore_prepare=false' # Search keys with timestamp 30, ignore_prepare=false and expect the cursor value to return WT_NOTFOUND. self.search_keys_timestamp_and_ignore(ds, txn_config, None) def test_prepare_hs(self): - ds = SimpleDataSet(self, self.uri, self.nrows, key_format="S", value_format='u') + ds = SimpleDataSet(self, self.uri, self.nrows, key_format=self.key_format, value_format='u') ds.populate() bigvalue = b"aaaaa" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py index ae79bd247f5..cc6f4a2f05d 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py @@ -30,52 +30,56 @@ import wiredtiger, wttest from wtscenario import make_scenarios from wiredtiger import stat, WT_NOTFOUND -def timestamp_str(t): - return '%x' % t - # test_prepare_hs05.py # Test that after aborting prepare transaction, correct update from the history store is restored. class test_prepare_hs05(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r', key=1)), + ('string-row', dict(key_format='S', key=str(1))), + ] + + scenarios = make_scenarios(key_format_values) + def test_check_prepare_abort_hs_restore(self): uri = 'table:test_prepare_hs05' - create_params = 'key_format=S,value_format=S' + create_params = 'key_format={},value_format=S'.format(self.key_format) self.session.create(uri, create_params) value1 = 'a' * 5 value2 = 'b' * 5 value3 = 'c' * 5 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - key = 1 + key = self.key self.session.begin_transaction() - cursor[str(key)] = value1 - cursor.set_key(str(key)) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + cursor[key] = value1 + cursor.set_key(key) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Commit update and remove operation in the same transaction. self.session.begin_transaction() - cursor[str(key)] = value2 - cursor.set_key(str(key)) + cursor[key] = value2 + cursor.set_key(key) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Add a prepared update for the key. self.session.begin_transaction() - cursor[str(key)] = value3 - self.session.prepare_transaction('prepare_timestamp='+ timestamp_str(4)) + cursor[key] = value3 + self.session.prepare_transaction('prepare_timestamp='+ self.timestamp_str(4)) # Try to evict the page with prepared update. This will ensure that prepared update is # written as the on-disk version and the older versions are moved to the history store. session2 = self.conn.open_session() session2.begin_transaction('ignore_prepare=true') cursor2 = session2.open_cursor(uri, None, "debug=(release_evict=true)") - cursor2.set_key(str(key)) + cursor2.set_key(key) self.assertEquals(cursor2.search(), WT_NOTFOUND) cursor2.reset() @@ -85,14 +89,14 @@ class test_prepare_hs05(wttest.WiredTigerTestCase): self.session.checkpoint() # We should be able to read the older version of the key from the history store. - self.session.begin_transaction('read_timestamp='+timestamp_str(2)) - cursor.set_key(str(key)) + self.session.begin_transaction('read_timestamp='+self.timestamp_str(2)) + cursor.set_key(key) self.assertEqual(cursor.search(), 0) self.assertEqual(cursor.get_value(), value1) self.session.rollback_transaction() # The latest version should be marked deleted. self.session.begin_transaction() - cursor.set_key(str(key)) + cursor.set_key(key) self.assertEqual(cursor.search(), WT_NOTFOUND) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_readonly02.py b/src/third_party/wiredtiger/test/suite/test_readonly02.py index 4adaf476a0b..62270a52302 100644 --- a/src/third_party/wiredtiger/test/suite/test_readonly02.py +++ b/src/third_party/wiredtiger/test/suite/test_readonly02.py @@ -42,15 +42,15 @@ class test_readonly02(wttest.WiredTigerTestCase, suite_subprocess): entries = 10 conn_params = \ - 'create,statistics=(fast),' + \ + 'create,' + \ 'log=(enabled,file_max=100K,zero_fill=true),' + \ 'operation_tracking=(enabled=false),' conn_params_rd = \ - 'create,readonly=true,statistics=(fast),' + \ + 'create,readonly=true,' + \ 'log=(enabled,zero_fill=false),' + \ 'operation_tracking=(enabled=false),' conn_params_rdcfg = \ - 'create,readonly=true,statistics=(fast),log=(enabled),' + \ + 'create,readonly=true,log=(enabled),' + \ 'operation_tracking=(enabled=false),' # diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py index 7613e169fb1..e85de366970 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py @@ -34,9 +34,6 @@ from wiredtiger import stat, wiredtiger_strerror, WiredTigerError, WT_ROLLBACK from wtscenario import make_scenarios from time import sleep -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable01.py # Shared base class used by rollback to stable tests. class test_rollback_to_stable_base(wttest.WiredTigerTestCase): @@ -77,12 +74,12 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): if commit_ts == 0: session.commit_transaction() elif prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() except WiredTigerError as e: rollback_str = wiredtiger_strerror(WT_ROLLBACK) @@ -104,12 +101,12 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): if commit_ts == 0: session.commit_transaction() elif prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() except WiredTigerError as e: rollback_str = wiredtiger_strerror(WT_ROLLBACK) @@ -129,12 +126,12 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): if commit_ts == 0: session.commit_transaction() elif prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() except WiredTigerError as e: rollback_str = wiredtiger_strerror(WT_ROLLBACK) @@ -147,7 +144,7 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): if read_ts == 0: session.begin_transaction() else: - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -189,10 +186,6 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 10000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable01" ds = SimpleDataSet( @@ -200,8 +193,8 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) valuea = "aaaaa" * 100 self.large_updates(uri, valuea, ds, nrows, self.prepare, 10) @@ -215,9 +208,9 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): # Pin stable to timestamp 20 if prepare otherwise 10. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Checkpoint to ensure that all the updates are flushed to disk. if not self.in_memory: self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py index 20b5d7d6a41..8c27f5395e7 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py @@ -34,9 +34,6 @@ from wiredtiger import stat from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable02.py # Test that rollback to stable brings back the history value to replace on-disk value. class test_rollback_to_stable02(test_rollback_to_stable_base): @@ -70,10 +67,6 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 10000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable02" ds = SimpleDataSet( @@ -81,8 +74,8 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) valuea = "aaaaa" * 100 valueb = "bbbbb" * 100 @@ -106,9 +99,9 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): # Pin stable to timestamp 30 if prepare otherwise 20. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) # Checkpoint to ensure that all the data is flushed. if not self.in_memory: self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py index a0e2d4337b2..c7260db1166 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py @@ -34,9 +34,6 @@ from wiredtiger import stat from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable03.py # Test that rollback to stable clears the history store updates from reconciled pages. class test_rollback_to_stable01(test_rollback_to_stable_base): @@ -70,10 +67,6 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable03" ds = SimpleDataSet( @@ -81,8 +74,8 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) valuea = "aaaaa" * 100 valueb = "bbbbb" * 100 @@ -101,9 +94,9 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): # Pin stable to timestamp 30 if prepare otherwise 20. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) # Checkpoint to ensure that all the updates are flushed to disk. if not self.in_memory: self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py index f865a3975f7..2342163a327 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py @@ -31,9 +31,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - def mod_val(value, char, location, nbytes=1): return value[0:location] + char + value[location+nbytes:] @@ -71,10 +68,6 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable04" ds = SimpleDataSet( @@ -82,8 +75,8 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -131,9 +124,9 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): # Pin stable to timestamp 40 if prepare otherwise 30. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) # Checkpoint to ensure the data is flushed, then rollback to the stable timestamp. if not self.in_memory: diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py index 5604196241c..4a32b589b18 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py @@ -34,9 +34,6 @@ from wiredtiger import stat from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable05.py # Test that rollback to stable cleans history store for non-timestamp tables. class test_rollback_to_stable05(test_rollback_to_stable_base): @@ -70,10 +67,6 @@ class test_rollback_to_stable05(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create two tables without logging. uri_1 = "table:rollback_to_stable05_1" ds_1 = SimpleDataSet( diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py index cd2c35783ce..7b702ca7b5e 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py @@ -31,9 +31,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable06.py # Test that rollback to stable removes all keys when the stable timestamp is earlier than # all commit timestamps. @@ -68,10 +65,6 @@ class test_rollback_to_stable06(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable06" ds = SimpleDataSet( @@ -79,8 +72,8 @@ class test_rollback_to_stable06(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py index 9c9e39d16b9..f745d0a3592 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py @@ -33,9 +33,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable07.py # Test the rollback to stable operation performs as expected following a server crash and # recovery. Verify that @@ -63,10 +60,6 @@ class test_rollback_to_stable07(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable07" ds = SimpleDataSet( @@ -74,8 +67,8 @@ class test_rollback_to_stable07(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -96,9 +89,9 @@ class test_rollback_to_stable07(test_rollback_to_stable_base): # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) # Perform additional updates. self.large_updates(uri, value_b, ds, nrows, self.prepare, 60) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py index 05153034362..e5c3c5bc9ae 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py @@ -31,9 +31,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable08.py # Test that rollback to stable does not abort updates when the stable timestamp is # set to the latest commit. @@ -68,10 +65,6 @@ class test_rollback_to_stable08(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 10000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable08" ds = SimpleDataSet( @@ -79,8 +72,8 @@ class test_rollback_to_stable08(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -101,9 +94,9 @@ class test_rollback_to_stable08(test_rollback_to_stable_base): # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(60)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(60)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Checkpoint to ensure the data is flushed, then rollback to the stable timestamp. if not self.in_memory: diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py index d0a45f0522d..70664a3d776 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py @@ -32,15 +32,17 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable09.py # Test that rollback to stable does not abort schema operations that are done # as they don't have transaction support class test_rollback_to_stable09(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + colstore_values = [ + ('column', dict(use_columns=True)), + ('row', dict(use_columns=False)), + ] + in_memory_values = [ ('no_inmem', dict(in_memory=False)), ('inmem', dict(in_memory=True)) @@ -55,10 +57,10 @@ class test_rollback_to_stable09(test_rollback_to_stable_base): uri = "table:" + tablename index_uri = "index:test_rollback_stable09:country" - scenarios = make_scenarios(in_memory_values, prepare_values) + scenarios = make_scenarios(colstore_values, in_memory_values, prepare_values) def conn_config(self): - config = 'cache_size=250MB,statistics=(all)' + config = 'cache_size=250MB' if self.in_memory: config += ',in_memory=true' else: @@ -69,16 +71,20 @@ class test_rollback_to_stable09(test_rollback_to_stable_base): self.pr('create table') session = self.session session.begin_transaction() - session.create(self.uri, 'key_format=5s,value_format=HQ,columns=(country,year,population)') + if self.use_columns: + config = 'key_format=r,value_format=5sHQ,columns=(id,country,year,population)' + else: + config = 'key_format=5s,value_format=HQ,columns=(country,year,population)' + session.create(self.uri, config) if commit_ts == 0: session.commit_transaction() elif self.prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) def drop_table(self, commit_ts): self.pr('drop table') @@ -88,12 +94,12 @@ class test_rollback_to_stable09(test_rollback_to_stable_base): if commit_ts == 0: session.commit_transaction() elif self.prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) def create_index(self, commit_ts): session = self.session @@ -102,17 +108,17 @@ class test_rollback_to_stable09(test_rollback_to_stable_base): if commit_ts == 0: session.commit_transaction() elif self.prepare: - session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) session.commit_transaction() else: - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) def test_rollback_to_stable(self): # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) # Create table and index at a later timestamp self.create_table(20) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py index cfd0c2f3c46..25aaae7ff67 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py @@ -34,9 +34,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from wtthread import checkpoint_thread, op_thread -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable10.py # Test the rollback to stable operation performs sweeping history store. class test_rollback_to_stable10(test_rollback_to_stable_base): @@ -61,10 +58,6 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. self.pr("create/populate tables") uri_1 = "table:rollback_to_stable10_1" @@ -79,8 +72,8 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): ds_2.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -114,9 +107,9 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(60)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(60)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Create a checkpoint thread done = threading.Event() @@ -180,10 +173,6 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): def test_rollback_to_stable_prepare(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. self.pr("create/populate tables") uri_1 = "table:rollback_to_stable10_1" @@ -198,8 +187,8 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): ds_2.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -235,9 +224,9 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(60)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(60)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Here's the update operations we'll perform, encapsulated so we can easily retry # it if we get a rollback. Rollbacks may occur when checkpoint is running. @@ -265,7 +254,7 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): self.retry_rollback('update ds1', session_p1, lambda: prepare_range_updates( session_p1, cursor_p1, ds_1, value_e, nrows, - 'prepare_timestamp=' + timestamp_str(69))) + 'prepare_timestamp=' + self.timestamp_str(69))) # Perform several updates in parallel with checkpoint. session_p2 = self.conn.open_session() @@ -274,7 +263,7 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): self.retry_rollback('update ds2', session_p2, lambda: prepare_range_updates( session_p2, cursor_p2, ds_2, value_e, nrows, - 'prepare_timestamp=' + timestamp_str(69))) + 'prepare_timestamp=' + self.timestamp_str(69))) finally: done.set() ckpt.join() @@ -290,8 +279,8 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): copy_wiredtiger_home(self, ".", "RESTART") # Commit the prepared transaction. - session_p1.commit_transaction('commit_timestamp=' + timestamp_str(70) + ',durable_timestamp=' + timestamp_str(71)) - session_p2.commit_transaction('commit_timestamp=' + timestamp_str(70) + ',durable_timestamp=' + timestamp_str(71)) + session_p1.commit_transaction('commit_timestamp=' + self.timestamp_str(70) + ',durable_timestamp=' + self.timestamp_str(71)) + session_p2.commit_transaction('commit_timestamp=' + self.timestamp_str(70) + ',durable_timestamp=' + self.timestamp_str(71)) session_p1.close() session_p2.close() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py index 59014d5aac9..f88d1ec0082 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py @@ -33,9 +33,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable11.py # Test the rollback to stable is retrieving the proper history store update. class test_rollback_to_stable11(test_rollback_to_stable_base): @@ -60,10 +57,6 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable11" ds = SimpleDataSet( @@ -71,8 +64,8 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -90,9 +83,9 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): # Pin stable to timestamp 30 if prepare otherwise 20. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) # Checkpoint to ensure that all the updates are flushed to disk. self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py index cd0211f2d12..0ab66b1d991 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py @@ -33,9 +33,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable12.py # Test the rollback to stable operation skipping subtrees in during tree walk. class test_rollback_to_stable12(test_rollback_to_stable_base): @@ -60,10 +57,6 @@ class test_rollback_to_stable12(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable12" ds = SimpleDataSet( @@ -71,8 +64,8 @@ class test_rollback_to_stable12(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -85,9 +78,9 @@ class test_rollback_to_stable12(test_rollback_to_stable_base): # Pin stable to timestamp 30 if prepare otherwise 20. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) # Load a single row modification to be removed. commit_ts = 30 @@ -95,12 +88,12 @@ class test_rollback_to_stable12(test_rollback_to_stable_base): self.session.begin_transaction() cursor[ds.key(1)] = value_b if self.prepare: - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) - self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1)) self.session.commit_transaction() else: - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py index 73f23b1f615..dddbd02c260 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py @@ -31,9 +31,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable13.py # Test the rollback to stable should retain/restore the tombstone from # the update list or from the history store for on-disk database. @@ -59,10 +56,6 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable13" ds = SimpleDataSet( @@ -70,8 +63,8 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -92,9 +85,9 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) self.session.checkpoint() # Simulate a server crash and restart. @@ -113,10 +106,6 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): def test_rollback_to_stable_with_aborted_updates(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable13" ds = SimpleDataSet( @@ -124,8 +113,8 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -164,9 +153,9 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) self.session.checkpoint() # Simulate a server crash and restart. @@ -185,10 +174,6 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): def test_rollback_to_stable_with_history_tombstone(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable13" ds = SimpleDataSet( @@ -196,8 +181,8 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 @@ -216,14 +201,14 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): cursor[ds.key(i)] = value_b cursor.set_key(ds.key(i)) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(40)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) cursor.close() # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) self.session.checkpoint() @@ -251,17 +236,14 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): def test_rollback_to_stable_with_stable_remove(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') # Create a table without logging. uri = "table:rollback_to_stable13" ds = SimpleDataSet( self, uri, 0, key_format=self.key_format, value_format="S", config='split_pct=50,log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 value_c = "ccccc" * 100 @@ -273,9 +255,9 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): self.large_removes(uri, ds, nrows, self.prepare, 40) # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) # Perform several updates and checkpoint. self.large_updates(uri, value_c, ds, nrows, self.prepare, 60) self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py index e7a5229555d..25b625e1cfa 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py @@ -34,9 +34,6 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios from wtthread import checkpoint_thread, op_thread -def timestamp_str(t): - return '%x' % t - def mod_val(value, char, location, nbytes=1): return value[0:location] + char + value[location+nbytes:] @@ -67,10 +64,6 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1500 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. self.pr("create/populate table") uri = "table:rollback_to_stable14" @@ -79,8 +72,8 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 @@ -106,9 +99,9 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(60)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(60)) else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Create a checkpoint thread done = threading.Event() @@ -172,10 +165,6 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): def test_rollback_to_stable_same_ts(self): nrows = 1500 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. self.pr("create/populate table") uri = "table:rollback_to_stable14" @@ -184,8 +173,8 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 @@ -213,7 +202,7 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.check(value_modQ, uri, nrows, 30) self.check(value_modT, uri, nrows, 60) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Create a checkpoint thread done = threading.Event() @@ -275,10 +264,6 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): def test_rollback_to_stable_same_ts_append(self): nrows = 1500 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. self.pr("create/populate table") uri = "table:rollback_to_stable14" @@ -287,8 +272,8 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 @@ -316,7 +301,7 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.check(value_modQ, uri, nrows, 30) self.check(value_modT, uri, nrows, 60) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Create a checkpoint thread done = threading.Event() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable15.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable15.py index b91691b99cd..080fcddfe9a 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable15.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable15.py @@ -32,9 +32,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable15.py # Test that roll back to stable handles updates present in the # update-list for both fixed length and variable length column store. @@ -43,7 +40,7 @@ class test_rollback_to_stable15(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), - ('integer', dict(key_format='i')), + ('integer-row', dict(key_format='i')), ] value_format_values = [ # Fixed length @@ -70,7 +67,7 @@ class test_rollback_to_stable15(wttest.WiredTigerTestCase): if read_ts == 0: session.begin_transaction() else: - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: @@ -89,8 +86,8 @@ class test_rollback_to_stable15(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri) # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) value20 = 0x20 value30 = 0x30 @@ -101,17 +98,17 @@ class test_rollback_to_stable15(wttest.WiredTigerTestCase): for i in range(1, nrows): self.session.begin_transaction() cursor[i] = value20 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) #First Update to value 30 at timestamp 5 for i in range(1, nrows): self.session.begin_transaction() cursor[i] = value30 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) cursor.close() #Set stable timestamp to 2 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(2)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(2)) self.conn.rollback_to_stable() # Check that only value20 is available self.check(value20, uri, nrows - 1, 2) @@ -121,17 +118,17 @@ class test_rollback_to_stable15(wttest.WiredTigerTestCase): for i in range(1, nrows): self.session.begin_transaction() cursor[i] = value30 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(7)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(7)) #Third Update to value40 at timestamp 9 for i in range(1, nrows): self.session.begin_transaction() cursor[i] = value40 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(9)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(9)) cursor.close() #Set stable timestamp to 7 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(7)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(7)) self.conn.rollback_to_stable() #Check that only value30 is available self.check(value30, uri, nrows - 1, 7) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py index 5ca08c18baf..4c52e52f50c 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py @@ -37,9 +37,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable16.py # Test that rollback to stable removes updates present on disk for column store. class test_rollback_to_stable16(wttest.WiredTigerTestCase): @@ -81,7 +78,7 @@ class test_rollback_to_stable16(wttest.WiredTigerTestCase): cursor[i] = value + str(i) else: cursor[i] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(timestamp)) cursor.close() def check(self, check_value, uri, nrows, start_row, read_ts): @@ -89,7 +86,7 @@ class test_rollback_to_stable16(wttest.WiredTigerTestCase): if read_ts == 0: session.begin_transaction() else: - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 @@ -128,14 +125,14 @@ class test_rollback_to_stable16(wttest.WiredTigerTestCase): self.session.create(uri, create_params) # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) for i in range(len(values)): self.insert_update_data(uri, values[i], start_row, nrows, ts[i]) start_row += nrows - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) if not self.in_memory: # Checkpoint to ensure that all the updates are flushed to disk. diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py index 829e5d13680..1697cd50c8b 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py @@ -33,9 +33,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable17.py # Test that rollback to stable handles updates present on history store and data store for variable # length column store. @@ -67,12 +64,12 @@ class test_rollback_to_stable17(wttest.WiredTigerTestCase): for i in range(start_row, end_row): self.session.begin_transaction() cursor[i] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(timestamp)) cursor.close() def check(self, check_value, uri, nrows, read_ts): session = self.session - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 @@ -96,15 +93,15 @@ class test_rollback_to_stable17(wttest.WiredTigerTestCase): self.session.create(uri, create_params) # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) # Make a series of updates for the same keys with different values at different timestamps. for i in range(len(values)): self.insert_update_data(uri, values[i], start_row, nrows, ts[i]) # Set the stable timestamp to 5. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) if not self.in_memory: # Checkpoint to ensure that all the updates are flushed to disk. diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py index a9725841b8b..d34e4ffb08b 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py @@ -38,9 +38,6 @@ from wiredtiger import stat from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable18.py # Test the rollback to stable shouldn't skip any pages that don't have aggregated time window. class test_rollback_to_stable18(test_rollback_to_stable_base): @@ -65,10 +62,6 @@ class test_rollback_to_stable18(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 10000 - # Prepare transactions for column store table is not yet supported. - if self.prepare and self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable18" ds = SimpleDataSet( @@ -76,8 +69,8 @@ class test_rollback_to_stable18(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 @@ -102,11 +95,11 @@ class test_rollback_to_stable18(test_rollback_to_stable_base): # Pin stable and oldest to timestamp 30 if prepare otherwise 20. if self.prepare: - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(30) + - ',stable_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(30) + + ',stable_timestamp=' + self.timestamp_str(30)) else: - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(20) + - ',stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(20) + + ',stable_timestamp=' + self.timestamp_str(20)) # Perform rollback to stable. self.conn.rollback_to_stable() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py index dd98263ae41..3f10b186b47 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py @@ -33,9 +33,6 @@ from wiredtiger import stat, WT_NOTFOUND from wtdataset import SimpleDataSet from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable19.py # Test that rollback to stable aborts both insert and remove updates from a single prepared transaction class test_rollback_to_stable19(test_rollback_to_stable_base): @@ -52,8 +49,8 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): ] restart_options = [ - ('shutdown', dict(crash='false')), - ('crash', dict(crash='true')), + ('shutdown', dict(crash=False)), + ('crash', dict(crash=True)), ] scenarios = make_scenarios(in_memory_values, key_format_values, restart_options) @@ -69,10 +66,6 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): def test_rollback_to_stable_no_history(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable19" ds = SimpleDataSet( @@ -80,8 +73,8 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = "aaaaa" * 100 @@ -94,7 +87,7 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): cursor.set_key(i) cursor.remove() cursor.close() - s.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") @@ -116,7 +109,7 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): cursor2.close() # Pin stable timestamp to 20. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) if not self.in_memory: self.session.checkpoint() @@ -137,17 +130,22 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): stat_cursor = self.session.open_cursor('statistics:', None, None) upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] - self.assertGreater(upd_aborted, 0) - self.assertGreater(keys_removed, 0) + + # After restart (not crash) the stats for the aborted updates will be 0, as the updates + # will be aborted during shutdown, and on startup there will be no updates to be aborted. + # This is similar case with keys removed. + if not self.in_memory and not self.crash: + self.assertEqual(upd_aborted, 0) + self.assertEqual(keys_removed, 0) + else: + self.assertGreater(upd_aborted, 0) + self.assertGreater(keys_removed, 0) + stat_cursor.close() def test_rollback_to_stable_with_history(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable19" ds = SimpleDataSet( @@ -155,8 +153,8 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = "aaaaa" * 100 valueb = "bbbbb" * 100 @@ -176,7 +174,7 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): cursor.set_key(i) cursor.remove() cursor.close() - s.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) + s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") @@ -198,7 +196,7 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): cursor2.close() # Pin stable timestamp to 40. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) if not self.in_memory: self.session.checkpoint() @@ -220,6 +218,15 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): stat_cursor = self.session.open_cursor('statistics:', None, None) upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] - self.assertGreater(upd_aborted, 0) + + # After restart (not crash) the stats for the aborted updates and history store removed will be 0, + # as the updates aborted and history store removed will occur during shutdown, and on startup there + # will be no updates to be removed. if not self.in_memory: - self.assertGreater(hs_removed, 0) + if self.crash: + self.assertGreater(hs_removed, 0) + else: + self.assertEqual(hs_removed, 0) + self.assertEqual(upd_aborted, 0) + else: + self.assertGreater(upd_aborted, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable20.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable20.py index a2c301ec362..8a2f44c5a36 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable20.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable20.py @@ -30,17 +30,22 @@ import time from helper import copy_wiredtiger_home import wiredtiger, wttest from wtdataset import SimpleDataSet +from wtscenario import make_scenarios from wiredtiger import stat from helper import simulate_crash_restart from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # Test that rollback to stable does not open any dhandles that don't have unstable updates. class test_rollback_to_stable20(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def conn_config(self): config = 'cache_size=50MB,statistics=(all)' return config @@ -51,12 +56,12 @@ class test_rollback_to_stable20(test_rollback_to_stable_base): create_params = 'key_format=i,value_format=S' uri = "table:rollback_to_stable20" ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds.populate() # Pin oldest and stable timestamp to 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) valuea = "aaaaa" * 100 @@ -65,7 +70,7 @@ class test_rollback_to_stable20(test_rollback_to_stable_base): self.session.create(uri, create_params) self.large_updates(uri, valuea, ds, nrows, 0, 10) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py index 4e44b9c2d97..3a348d00dc0 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py @@ -37,9 +37,6 @@ from helper import simulate_crash_restart from wtdataset import SimpleDataSet from test_rollback_to_stable01 import test_rollback_to_stable_base -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable21.py # Test rollback to stable when an out of order prepared transaction is written to disk class test_rollback_to_stable21(test_rollback_to_stable_base): @@ -57,10 +54,6 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable21" ds = SimpleDataSet( @@ -68,8 +61,8 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = 'a' * 400 valueb = 'b' * 400 @@ -79,7 +72,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): for i in range(1, nrows + 1): cursor[i] = valuea - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) self.session.begin_transaction() for i in range(1, nrows + 1): @@ -87,7 +80,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): cursor.reset() cursor.close() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) s = self.conn.open_session() s.begin_transaction('ignore_prepare = true') @@ -101,7 +94,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): evict_cursor.reset() s.rollback_transaction() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) s.checkpoint() # Rollback the prepared transaction @@ -123,10 +116,6 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): def test_rollback_to_stable_with_different_tombstone(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable21" ds = SimpleDataSet( @@ -134,8 +123,8 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = 'a' * 400 valueb = 'b' * 400 @@ -144,13 +133,13 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): self.session.begin_transaction() for i in range(1, nrows + 1): cursor[i] = valuea - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) self.session.begin_transaction() for i in range(1, nrows + 1): cursor.set_key(i) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(40)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) self.session.begin_transaction() for i in range(1, nrows + 1): @@ -158,10 +147,10 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): cursor.reset() cursor.close() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) s = self.conn.open_session() - s.begin_transaction('ignore_prepare = true, read_timestamp = ' + timestamp_str(30)) + s.begin_transaction('ignore_prepare = true, read_timestamp = ' + self.timestamp_str(30)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = s.open_cursor(uri, None, "debug=(release_evict)") @@ -172,7 +161,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): evict_cursor.reset() s.rollback_transaction() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) s.checkpoint() # Rollback the prepared transaction @@ -197,10 +186,6 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): def test_rollback_to_stable_with_same_tombstone(self): nrows = 1000 - # Prepare transactions for column store table is not yet supported. - if self.key_format == 'r': - self.skipTest('Prepare transactions for column store table is not yet supported') - # Create a table without logging. uri = "table:rollback_to_stable21" ds = SimpleDataSet( @@ -208,8 +193,8 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): ds.populate() # Pin oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) valuea = 'a' * 400 valueb = 'b' * 400 @@ -221,7 +206,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): cursor.set_key(i) cursor.remove() - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) self.session.begin_transaction() for i in range(1, nrows + 1): @@ -229,7 +214,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): cursor.reset() cursor.close() - self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) s = self.conn.open_session() s.begin_transaction('ignore_prepare = true') @@ -242,7 +227,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): evict_cursor.reset() s.rollback_transaction() - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) s.checkpoint() # Rollback the prepared transaction diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py index ac123fe9ccd..f6ef52cc388 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py @@ -29,16 +29,13 @@ from test_rollback_to_stable01 import test_rollback_to_stable_base from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - # test_rollback_to_stable22 # Test history store operations conflicting with rollback to stable. We're trying to trigger a # history store eviction concurrently to a rollback to stable call. We'll do this by limiting # the cache size to 100MB and performing 100MB worth of inserts while periodically calling rollback # to stable. class test_rollback_to_stable22(test_rollback_to_stable_base): - conn_config = 'cache_size=100MB,statistics=(fast),statistics_log=(wait=1,json)' + conn_config = 'cache_size=100MB' session_config = 'isolation=snapshot' prepare = False @@ -82,5 +79,5 @@ class test_rollback_to_stable22(test_rollback_to_stable_base): if i % 100 == 0: # Put the timestamp backwards so we can rollback the updates we just did. stable_ts = (i - 1) * 10 - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(stable_ts)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(stable_ts)) self.conn.rollback_to_stable() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable23.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable23.py new file mode 100644 index 00000000000..e4ff9e4dfb4 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable23.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from helper import simulate_crash_restart +from test_rollback_to_stable01 import test_rollback_to_stable_base +from wiredtiger import stat +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +def mod_val(value, char, location, nbytes=1): + return value[0:location] + char + value[location+nbytes:] + +# test_rollback_to_stable23.py +# Test to verify that search operation uses proper base update while returning modifies from +# the history store after the on-disk update is removed by the rollback to stable. +class test_rollback_to_stable23(test_rollback_to_stable_base): + session_config = 'isolation=snapshot' + + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + prepare_values = [ + ('no_prepare', dict(prepare=False)), + ('prepare', dict(prepare=True)) + ] + + scenarios = make_scenarios(key_format_values, prepare_values) + + def conn_config(self): + config = 'cache_size=50MB,statistics=(all),statistics_log=(json,on_close,wait=1)' + return config + + def check_with_set_key(self, ds, check_value, uri, nrows, read_ts): + cursor = self.session.open_cursor(uri) + self.session.begin_transaction("read_timestamp = " + self.timestamp_str(read_ts)) + for i in range(1, nrows + 1): + cursor.set_key(ds.key(i)) + self.assertEquals(cursor.search(), 0) + self.assertEquals(cursor.get_value(), check_value) + cursor.close() + self.session.commit_transaction() + + def test_rollback_to_stable(self): + nrows = 1000 + + # Create a table without logging. + uri = "table:rollback_to_stable23" + ds = SimpleDataSet( + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + ds.populate() + + # Pin oldest and stable to timestamp 10. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) + + value_a = "aaaaa" * 100 + + value_modQ = mod_val(value_a, 'Q', 0) + value_modR = mod_val(value_modQ, 'R', 1) + value_modS = mod_val(value_modR, 'S', 2) + value_modT = mod_val(value_modS, 'T', 3) + + # Perform a combination of modifies and updates. + self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) + self.large_modifies(uri, 'Q', ds, 0, 1, nrows, self.prepare, 30) + self.large_modifies(uri, 'R', ds, 1, 1, nrows, self.prepare, 40) + self.large_modifies(uri, 'S', ds, 2, 1, nrows, self.prepare, 50) + self.large_modifies(uri, 'T', ds, 3, 1, nrows, self.prepare, 60) + + # Verify data is visible and correct. + self.check(value_a, uri, nrows, 20) + self.check(value_modQ, uri, nrows, 30) + self.check(value_modR, uri, nrows, 40) + self.check(value_modS, uri, nrows, 50) + self.check(value_modT, uri, nrows, 60) + + # Pin stable to timestamp 60 if prepare otherwise 50. + if self.prepare: + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(60)) + else: + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) + + # Checkpoint the database. + self.session.checkpoint() + + # Simulate a server crash and restart. + simulate_crash_restart(self, ".", "RESTART") + + # Check that the correct data is seen at and after the stable timestamp. + self.check_with_set_key(ds, value_a, uri, nrows, 20) + self.check_with_set_key(ds, value_modQ, uri, nrows, 30) + self.check_with_set_key(ds, value_modR, uri, nrows, 40) + self.check_with_set_key(ds, value_modS, uri, nrows, 50) + + stat_cursor = self.session.open_cursor('statistics:', None, None) + hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] + hs_restore_updates = stat_cursor[stat.conn.txn_rts_hs_restore_updates][2] + upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] + stat_cursor.close() + + self.assertEqual(hs_restore_updates, nrows) + if self.prepare: + self.assertGreaterEqual(upd_aborted, 0) + else: + self.assertEqual(upd_aborted, 0) + self.assertGreaterEqual(hs_removed, nrows) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_search_near01.py b/src/third_party/wiredtiger/test/suite/test_search_near01.py index 80f50d2f97a..412cbd894f0 100755 --- a/src/third_party/wiredtiger/test/suite/test_search_near01.py +++ b/src/third_party/wiredtiger/test/suite/test_search_near01.py @@ -30,9 +30,6 @@ import time, wiredtiger, wttest, unittest from wiredtiger import stat -def timestamp_str(t): - return '%x' % t - # test_search_near01.py # Test various prefix search near scenarios. class test_search_near01(wttest.WiredTigerTestCase): diff --git a/src/third_party/wiredtiger/test/suite/test_stat09.py b/src/third_party/wiredtiger/test/suite/test_stat09.py index 9f775f7f949..5345449581e 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat09.py +++ b/src/third_party/wiredtiger/test/suite/test_stat09.py @@ -31,10 +31,6 @@ import wiredtiger, wttest # test_stat09.py # Check oldest active read timestamp statistic - -def timestamp_str(t): - return '%x' % t - class test_stat09(wttest.WiredTigerTestCase): tablename = 'test_stat09' uri = 'table:' + tablename @@ -88,7 +84,7 @@ class test_stat09(wttest.WiredTigerTestCase): for k in keys: self.session.begin_transaction() c[k] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k)) # Create a cursor on statistics that we can use repeatedly allstat_cursor = self.session.open_cursor('statistics:', None, None) @@ -98,15 +94,15 @@ class test_stat09(wttest.WiredTigerTestCase): # Introduce multiple transactions with varying read_timestamp s1 = self.conn.open_session() - s1.begin_transaction('read_timestamp=' + timestamp_str(10)) + s1.begin_transaction('read_timestamp=' + self.timestamp_str(10)) s2 = self.conn.open_session() - s2.begin_transaction('read_timestamp=' + timestamp_str(20)) + s2.begin_transaction('read_timestamp=' + self.timestamp_str(20)) s3 = self.conn.open_session() - s3.begin_transaction('read_timestamp=' + timestamp_str(30)) + s3.begin_transaction('read_timestamp=' + self.timestamp_str(30)) s4 = self.conn.open_session() - s4.begin_transaction('read_timestamp=' + timestamp_str(40)) + s4.begin_transaction('read_timestamp=' + self.timestamp_str(40)) s5 = self.conn.open_session() - s5.begin_transaction('read_timestamp=' + timestamp_str(50)) + s5.begin_transaction('read_timestamp=' + self.timestamp_str(50)) # Check oldest reader self.check_stat_oldest_read(allstat_cursor, 10, commit_range) @@ -117,18 +113,18 @@ class test_stat09(wttest.WiredTigerTestCase): # Set and advance the oldest timestamp, it should be ignored for # determining the oldest active read. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5)) self.check_stat_oldest_read(allstat_cursor, 20, commit_range) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(30)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(30)) self.check_stat_oldest_read(allstat_cursor, 20, commit_range) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(150)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(150)) self.check_stat_oldest_read(allstat_cursor, 20, commit_range) # Move the commit timestamp and check again commit_range = 200 - s2.commit_transaction('commit_timestamp=' + timestamp_str(commit_range)) + s2.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_range)) self.check_stat_oldest_read(allstat_cursor, 30, commit_range) # Close all the readers and check the oldest reader, it should be back to 0 diff --git a/src/third_party/wiredtiger/test/suite/test_tiered03.py b/src/third_party/wiredtiger/test/suite/test_tiered03.py index 61e12c08e28..f605342fa5a 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered03.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered03.py @@ -56,7 +56,6 @@ class test_tiered03(wttest.WiredTigerTestCase): if not os.path.exists(self.bucket): os.mkdir(self.bucket) return \ - 'statistics=(all),' + \ 'tiered_storage=(auth_token=%s,' % self.auth_token + \ 'bucket=%s,' % self.bucket + \ 'bucket_prefix=%s,' % self.bucket_prefix + \ diff --git a/src/third_party/wiredtiger/test/suite/test_tiered05.py b/src/third_party/wiredtiger/test/suite/test_tiered05.py index 2af464ba64b..1d199bad74b 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered05.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered05.py @@ -50,7 +50,6 @@ class test_tiered05(wttest.WiredTigerTestCase): def conn_config(self): os.mkdir(self.bucket) return \ - 'statistics=(fast),' + \ 'tiered_manager=(wait=%d),' % self.wait + \ 'tiered_storage=(auth_token=%s,' % self.auth_token + \ 'bucket=%s,' % self.bucket + \ diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp01.py b/src/third_party/wiredtiger/test/suite/test_timestamp01.py index 433347bcec4..f8c768eb2f8 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp01.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp01.py @@ -33,9 +33,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_timestamp01(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' @@ -43,28 +40,28 @@ class test_timestamp01(wttest.WiredTigerTestCase, suite_subprocess): # Cannot set a timestamp on a non-running transaction self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(1 << 5000)), + 'commit_timestamp=' + self.timestamp_str(1 << 5000)), '/only permitted in a running/') # Zero is not permitted self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(0)), + 'commit_timestamp=' + self.timestamp_str(0)), '/zero not permitted/') # Too big is also not permitted self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(1 << 5000)), + 'commit_timestamp=' + self.timestamp_str(1 << 5000)), '/too long/') # Anything other than lower case hexadecimal characters is not permitted self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(-1)), + 'commit_timestamp=' + self.timestamp_str(-1)), '/Failed to parse commit timestamp/') self.session.begin_transaction() @@ -88,13 +85,13 @@ class test_timestamp01(wttest.WiredTigerTestCase, suite_subprocess): # One is okay, as is upper case hex and 2**64 - 1 self.session.begin_transaction() self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(1)) + 'commit_timestamp=' + self.timestamp_str(1)) self.session.begin_transaction() self.session.commit_transaction( 'commit_timestamp=0A78F') self.session.begin_transaction() self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(1 << 64 - 1)) + 'commit_timestamp=' + self.timestamp_str(1 << 64 - 1)) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp02.py b/src/third_party/wiredtiger/test/suite/test_timestamp02.py index b77f409a182..c0f45f82dc5 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp02.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp02.py @@ -35,9 +35,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp02(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp02' uri = 'table:' + tablename @@ -79,58 +76,58 @@ class test_timestamp02(wttest.WiredTigerTestCase, suite_subprocess): for k in keys: self.session.begin_transaction() c[k] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k)) # Don't set a stable timestamp yet. Make sure we can read with # a timestamp before the stable timestamp has been set. # Now check that we see the expected state when reading at each # timestamp for i, t in enumerate(orig_keys): - self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t), dict((k, 1) for k in orig_keys[:i+1])) # Everything up to and including timestamp 100 has been committed. - self.assertTimestampsEqual(self.conn.query_timestamp(), timestamp_str(100)) + self.assertTimestampsEqual(self.conn.query_timestamp(), self.timestamp_str(100)) # Bump the oldest timestamp, we're not going back... - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100)) # Update them and retry. random.shuffle(keys) for k in keys: self.session.begin_transaction() c[k] = 2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k + 100)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k + 100)) # Everything up to and including timestamp 200 has been committed. - self.assertTimestampsEqual(self.conn.query_timestamp(), timestamp_str(200)) + self.assertTimestampsEqual(self.conn.query_timestamp(), self.timestamp_str(200)) # Test that we can manually move the commit timestamp back - self.conn.set_timestamp('commit_timestamp=' + timestamp_str(150)) - self.assertTimestampsEqual(self.conn.query_timestamp(), timestamp_str(150)) - self.conn.set_timestamp('commit_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('commit_timestamp=' + self.timestamp_str(150)) + self.assertTimestampsEqual(self.conn.query_timestamp(), self.timestamp_str(150)) + self.conn.set_timestamp('commit_timestamp=' + self.timestamp_str(200)) # Now the stable timestamp before we read. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(200)) for i, t in enumerate(orig_keys): - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 100), dict((k, (2 if j <= i else 1)) for j, k in enumerate(orig_keys))) # Bump the oldest timestamp, we're not going back... - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(200)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(200)) # Remove them and retry random.shuffle(keys) for k in keys: self.session.begin_transaction() del c[k] - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k + 200)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k + 200)) # We have to continue to advance the stable timestamp before reading. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(300)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(300)) for i, t in enumerate(orig_keys): - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 200), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 200), dict((k, 2) for k in orig_keys[i+1:])) def test_read_your_writes(self): diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp03.py b/src/third_party/wiredtiger/test/suite/test_timestamp03.py index 862ed241b17..a18d3fd86b0 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp03.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp03.py @@ -36,9 +36,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): table_ts_log = 'ts03_ts_logged' table_ts_nolog = 'ts03_ts_nologged' @@ -46,11 +43,13 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): table_nots_nolog = 'ts03_nots_nologged' types = [ - ('file', dict(uri='file:', use_cg=False, use_index=False)), - ('lsm', dict(uri='lsm:', use_cg=False, use_index=False)), - ('table-cg', dict(uri='table:', use_cg=True, use_index=False)), - ('table-index', dict(uri='table:', use_cg=False, use_index=True)), - ('table-simple', dict(uri='table:', use_cg=False, use_index=False)), + ('file-row', dict(uri='file:', key_format='i', use_cg=False, use_index=False)), + ('file-col', dict(uri='file:', key_format='r', use_cg=False, use_index=False)), + ('lsm', dict(uri='lsm:', key_format='i', use_cg=False, use_index=False)), + ('table-row', dict(uri='table:', key_format='i', use_cg=False, use_index=False)), + ('table-row-index', dict(uri='table:', key_format='i', use_cg=False, use_index=True)), + ('table-col', dict(uri='table:', key_format='r', use_cg=False, use_index=False)), + ('table-col-cg', dict(uri='table:', key_format='r', use_cg=True, use_index=False)), ] ckpt = [ @@ -166,13 +165,14 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): # 3. Table is logged and does not use timestamps. # 4. Table is not logged and does not use timestamps. # - self.session.create(uri_ts_log, 'key_format=i,value_format=S') + format = 'key_format={},value_format=S'.format(self.key_format) + self.session.create(uri_ts_log, format) cur_ts_log = self.session.open_cursor(uri_ts_log) - self.session.create(uri_ts_nolog, 'key_format=i,value_format=S,log=(enabled=false)') + self.session.create(uri_ts_nolog, format + ',log=(enabled=false)') cur_ts_nolog = self.session.open_cursor(uri_ts_nolog) - self.session.create(uri_nots_log, 'key_format=i,value_format=S') + self.session.create(uri_nots_log, format) cur_nots_log = self.session.open_cursor(uri_nots_log) - self.session.create(uri_nots_nolog, 'key_format=i,value_format=S, log=(enabled=false)') + self.session.create(uri_nots_nolog, format + ',log=(enabled=false)') cur_nots_nolog = self.session.open_cursor(uri_nots_nolog) # Insert keys 1..100 each with timestamp=key, in some order @@ -187,7 +187,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() cur_ts_log[k] = self.value cur_ts_nolog[k] = self.value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k)) # Scenario: 1 # Check that we see all the inserted values as per transaction @@ -207,19 +207,19 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): for i, t in enumerate(orig_keys): # Tables using the timestamps should see the values as per the # given read_timestamp - self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t), self.table_ts_log, dict((k, self.value) for k in orig_keys[:i+1])) - self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t), self.table_ts_nolog, dict((k, self.value) for k in orig_keys[:i+1])) # Tables not using the timestamps should see all the values. - self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t), self.table_nots_log, dict((k, self.value) for k in orig_keys)) - self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t), self.table_nots_nolog, dict((k, self.value) for k in orig_keys)) # Bump the oldest_timestamp, we're not going back... - self.assertTimestampsEqual(self.conn.query_timestamp(), timestamp_str(100)) - old_ts = timestamp_str(100) + self.assertTimestampsEqual(self.conn.query_timestamp(), self.timestamp_str(100)) + old_ts = self.timestamp_str(100) self.conn.set_timestamp('oldest_timestamp=' + old_ts) self.conn.set_timestamp('stable_timestamp=' + old_ts) @@ -248,7 +248,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() cur_ts_log[k] = self.value2 cur_ts_nolog[k] = self.value2 - ts = timestamp_str(k + 100) + ts = self.timestamp_str(k + 100) self.session.commit_transaction('commit_timestamp=' + ts) count += 1 @@ -270,7 +270,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): # This scenario is same as earlier one with read_timestamp earlier than # oldest_timestamp and using the option of rounding read_timestamp to # the oldest_timestamp - earlier_ts = timestamp_str(90) + earlier_ts = self.timestamp_str(90) self.check(self.session, 'read_timestamp=' + earlier_ts +',roundup_timestamps=(read=true)', self.table_ts_log, dict((k, self.value) for k in orig_keys)) @@ -294,15 +294,15 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): expected_dict[i+1] = self.value2 # Tables using the timestamps should see the updated values as per # the given read_timestamp - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 100), self.table_ts_log, expected_dict) - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 100), self.table_ts_nolog, expected_dict) # Tables not using the timestamps should see all the data values as # updated values (i.e. value2). - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 100), self.table_nots_log, dict((k, self.value2) for k in orig_keys)) - self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 100), self.table_nots_nolog, dict((k, self.value2) for k in orig_keys)) # Take a checkpoint using the given configuration. Then verify @@ -359,7 +359,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): # Update the stable_timestamp to the latest, but not the # oldest_timestamp and make sure we can see the data. Once the # stable_timestamp is moved we should see all keys with value2. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100+nkeys)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100+nkeys)) self.ckpt_backup(self.value2, nkeys, nkeys, nkeys, nkeys) # If we're not using the log we're done. @@ -378,7 +378,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() cur_ts_log[k] = self.value3 cur_ts_nolog[k] = self.value3 - ts = timestamp_str(k + 200) + ts = self.timestamp_str(k + 200) self.session.commit_transaction('commit_timestamp=' + ts) count += 1 diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp04.py b/src/third_party/wiredtiger/test/suite/test_timestamp04.py index 9aff0351e2d..e2aa7609fa7 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp04.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp04.py @@ -35,9 +35,6 @@ import wiredtiger, wttest from wiredtiger import stat from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): table_ts_log = 'table:ts04_ts_logged' table_ts_nolog = 'table:ts04_ts_nologged' @@ -56,12 +53,13 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Minimum cache_size requirement of lsm is 31MB. types = [ - # The commented columnar tests needs to be enabled once rollback to stable for columnar is fixed in (WT-5548). - # ('col_fix', dict(empty=1, cacheSize='cache_size=20MB', extra_config=',key_format=r,value_format=8t')), - # ('col_var', dict(empty=0, cacheSize='cache_size=20MB', extra_config=',key_format=r')), + # FLCS does not yet work in a timestamp world. + #('col_fix', dict(empty=1, \ + # cacheSize='cache_size=20MB', extra_config=',key_format=r,value_format=8t')), ('lsm', dict(empty=0, cacheSize='cache_size=31MB', extra_config=',type=lsm')), ('row', dict(empty=0, cacheSize='cache_size=20MB', extra_config='',)), ('row-smallcache', dict(empty=0, cacheSize='cache_size=2MB', extra_config='',)), + ('var', dict(empty=0, cacheSize='cache_size=20MB', extra_config=',key_format=r')), ] scenarios = make_scenarios(conncfg, types) @@ -146,10 +144,10 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() cur_ts_log[k] = 1 cur_ts_nolog[k] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k)) # Setup an oldest timestamp to ensure state remains in cache. if k == 1: - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cur_ts_log.close() cur_ts_nolog.close() cur_nots_log.close() @@ -157,7 +155,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Scenario: 1 # Check that we see all the inserted values(i.e 1) in all tables - latest_ts = timestamp_str(key_range) + latest_ts = self.timestamp_str(key_range) self.check(self.session, 'read_timestamp=' + latest_ts, self.table_nots_log, dict((k, 1) for k in keys[:])) self.check(self.session, 'read_timestamp=' + latest_ts, @@ -169,7 +167,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Scenario: 2 # Roll back half timestamps. - stable_ts = timestamp_str(key_range // 2) + stable_ts = self.timestamp_str(key_range // 2) self.conn.set_timestamp('stable_timestamp=' + stable_ts) # We're about to test rollback-to-stable which requires a checkpoint to which we can roll back. @@ -231,7 +229,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() cur_ts_log[k] = 2 cur_ts_nolog[k] = 2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k + key_range)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k + key_range)) cur_ts_log.close() cur_ts_nolog.close() cur_nots_log.close() @@ -239,7 +237,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Scenario: 3 # Check that we see all values updated (i.e 2) in all tables. - latest_ts = timestamp_str(2 * key_range) + latest_ts = self.timestamp_str(2 * key_range) self.check(self.session, 'read_timestamp=' + latest_ts, self.table_nots_log, dict((k, 2) for k in keys[:])) self.check(self.session, 'read_timestamp=' + latest_ts, @@ -253,7 +251,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Advance the stable_timestamp by a quarter range and rollback. # Three-fourths of the later timestamps will be rolled back. rolled_range = key_range + key_range // 4 - stable_ts = timestamp_str(rolled_range) + stable_ts = self.timestamp_str(rolled_range) self.conn.set_timestamp('stable_timestamp=' + stable_ts) self.conn.rollback_to_stable() stat_cursor = self.session.open_cursor('statistics:', None, None) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp05.py b/src/third_party/wiredtiger/test/suite/test_timestamp05.py index e7be33de000..852823231c7 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp05.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp05.py @@ -35,13 +35,16 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp05(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:ts05' session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def test_create(self): s = self.session conn = self.conn @@ -51,8 +54,8 @@ class test_timestamp05(wttest.WiredTigerTestCase, suite_subprocess): # Commit at 100 s.begin_transaction() - s.create(self.uri, 'key_format=i,value_format=S') - s.commit_transaction('commit_timestamp=' + timestamp_str(100)) + s.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) + s.commit_transaction('commit_timestamp=' + self.timestamp_str(100)) # Make sure the tree is dirty c = s.open_cursor(self.uri) @@ -65,7 +68,7 @@ class test_timestamp05(wttest.WiredTigerTestCase, suite_subprocess): s = self.session conn = self.conn - s.create(self.uri, 'key_format=i,value_format=S') + s.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) c = s.open_cursor(self.uri, None, 'bulk') # Insert keys 1..100 each with timestamp=key, in some order @@ -79,7 +82,7 @@ class test_timestamp05(wttest.WiredTigerTestCase, suite_subprocess): # Commit at 100 s.begin_transaction() c.close() - s.commit_transaction('commit_timestamp=' + timestamp_str(100)) + s.commit_transaction('commit_timestamp=' + self.timestamp_str(100)) # Make sure the tree is dirty c = s.open_cursor(self.uri) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp06.py b/src/third_party/wiredtiger/test/suite/test_timestamp06.py index d958a3ba3ed..27e32309fdc 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp06.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp06.py @@ -36,15 +36,13 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp06(wttest.WiredTigerTestCase, suite_subprocess): table_ts_log = 'table:ts06_ts_logged' table_ts_nolog = 'table:ts06_ts_nologged' types = [ - ('col_fix', dict(empty=1, extra_config=',key_format=r,value_format=8t')), + # FLCS does not yet work in a timestamp world. + #('col_fix', dict(empty=1, extra_config=',key_format=r,value_format=8t')), ('col_var', dict(empty=0, extra_config=',key_format=r')), ('lsm', dict(empty=0, extra_config=',type=lsm')), ('row', dict(empty=0, extra_config='',)), @@ -125,9 +123,9 @@ class test_timestamp06(wttest.WiredTigerTestCase, suite_subprocess): # Open two timestamp tables: # 1. Table is logged and uses timestamps. # 2. Table is not logged and uses timestamps. - self.session.create(self.table_ts_log, 'key_format=i,value_format=i') + self.session.create(self.table_ts_log, 'key_format=i,value_format=i' + self.extra_config) cur_ts_log = self.session.open_cursor(self.table_ts_log) - self.session.create(self.table_ts_nolog, 'key_format=i,value_format=i,log=(enabled=false)') + self.session.create(self.table_ts_nolog, 'key_format=i,value_format=i,log=(enabled=false)' + self.extra_config) cur_ts_nolog = self.session.open_cursor(self.table_ts_nolog) # Insert keys 1..100 @@ -138,22 +136,22 @@ class test_timestamp06(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() # Make three updates with different timestamps. - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(1)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(1)) for k in keys: cur_ts_log[k] = 1 cur_ts_nolog[k] = 1 - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(101)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(101)) for k in keys: cur_ts_log[k] = 2 cur_ts_nolog[k] = 2 - self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(201)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(201)) for k in keys: cur_ts_log[k] = 3 cur_ts_nolog[k] = 3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(301)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(301)) cur_ts_log.close() cur_ts_nolog.close() @@ -170,9 +168,9 @@ class test_timestamp06(wttest.WiredTigerTestCase, suite_subprocess): # Check that we see the values till correctly from checkpointed data # files in case of multistep transactions. # Set oldest and stable timestamps - old_ts = timestamp_str(100) + old_ts = self.timestamp_str(100) # Set the stable timestamp such that last update is beyond it. - stable_ts = timestamp_str(200) + stable_ts = self.timestamp_str(200) self.conn.set_timestamp('oldest_timestamp=' + old_ts) self.conn.set_timestamp('stable_timestamp=' + stable_ts) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp07.py b/src/third_party/wiredtiger/test/suite/test_timestamp07.py index e84f9ea5f34..5e25afe1675 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp07.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp07.py @@ -36,14 +36,16 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'ts07_ts_nologged' tablename2 = 'ts07_nots_logged' tablename3 = 'ts07_ts_logged' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + types = [ ('file', dict(uri='file:', use_cg=False, use_index=False)), ('table-cg', dict(uri='table:', use_cg=True, use_index=False)), @@ -61,7 +63,7 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): ('1000keys', dict(nkeys=1000)), ] - scenarios = make_scenarios(types, conncfg, nkeys) + scenarios = make_scenarios(key_format_values, types, conncfg, nkeys) # Binary values. value = u'\u0001\u0002abcd\u0007\u0004' @@ -183,11 +185,11 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): # 2. Table is logged and does not use timestamps. # 3. Table is logged and uses timestamps. # - self.session.create(uri, 'key_format=i,value_format=S,log=(enabled=false)') + self.session.create(uri, 'key_format={},value_format=S,log=(enabled=false)'.format(self.key_format)) c = self.session.open_cursor(uri) - self.session.create(uri2, 'key_format=i,value_format=S') + self.session.create(uri2, 'key_format={},value_format=S'.format(self.key_format)) c2 = self.session.open_cursor(uri2) - self.session.create(uri3, 'key_format=i,value_format=S') + self.session.create(uri3, 'key_format={},value_format=S'.format(self.key_format)) c3 = self.session.open_cursor(uri3) # print "tables created" @@ -201,23 +203,23 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[k] = self.value c3[k] = self.value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k)) # print "value inserted in all tables, reading..." # Now check that we see the expected state when reading at each # timestamp. for k in orig_keys: - self.check(self.session, 'read_timestamp=' + timestamp_str(k), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(k), k, self.value) - self.check(self.session, 'read_timestamp=' + timestamp_str(k), + self.check(self.session, 'read_timestamp=' + self.timestamp_str(k), k + 1, None) # print "all values read, updating timestamps" # Bump the oldest timestamp, we're not going back... - self.assertTimestampsEqual(self.conn.query_timestamp(), timestamp_str(self.nkeys)) - self.oldts = self.stablets = timestamp_str(self.nkeys) + self.assertTimestampsEqual(self.conn.query_timestamp(), self.timestamp_str(self.nkeys)) + self.oldts = self.stablets = self.timestamp_str(self.nkeys) self.conn.set_timestamp('oldest_timestamp=' + self.oldts) self.conn.set_timestamp('stable_timestamp=' + self.stablets) # print "Oldest " + self.oldts @@ -237,7 +239,7 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[k] = self.value2 c3[k] = self.value2 - ts = timestamp_str(k + self.nkeys) + ts = self.timestamp_str(k + self.nkeys) self.session.commit_transaction('commit_timestamp=' + ts) # print "Commit key " + str(k) + " ts " + ts count += 1 @@ -252,7 +254,7 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): # Update the stable timestamp to the latest, but not the oldest # timestamp and make sure we can see the data. Once the stable # timestamp is moved we should see all keys with value2. - self.stablets = timestamp_str(self.nkeys*2) + self.stablets = self.timestamp_str(self.nkeys*2) self.conn.set_timestamp('stable_timestamp=' + self.stablets) # print "check_stable 2" self.check_stable(self.value2, self.nkeys, self.nkeys, self.nkeys) @@ -274,7 +276,7 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[k] = self.value3 c3[k] = self.value3 - ts = timestamp_str(k + self.nkeys*2) + ts = self.timestamp_str(k + self.nkeys*2) self.session.commit_transaction('commit_timestamp=' + ts) # print "Commit key " + str(k) + " ts " + ts count += 1 diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp09.py b/src/third_party/wiredtiger/test/suite/test_timestamp09.py index 91c6435e7d2..2188b919c43 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp09.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp09.py @@ -33,9 +33,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp09' uri = 'table:' + tablename @@ -48,7 +45,7 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): # Begin by adding some data. self.session.begin_transaction() self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(1)) + 'commit_timestamp=' + self.timestamp_str(1)) c[1] = 1 # In a single transaction it is illegal to set a commit timestamp @@ -56,21 +53,21 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): # Check both timestamp_transaction and commit_transaction APIs. self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(3)) + 'commit_timestamp=' + self.timestamp_str(3)) c[3] = 3 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)), + 'commit_timestamp=' + self.timestamp_str(2)), '/older than the first commit timestamp/') self.session.rollback_transaction() self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(4)) + 'commit_timestamp=' + self.timestamp_str(4)) c[4] = 4 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(3)), + 'commit_timestamp=' + self.timestamp_str(3)), '/older than the first commit timestamp/') # Commit timestamp >= Oldest timestamp @@ -78,13 +75,13 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[3] = 3 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(3)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(3)) + 'commit_timestamp=' + self.timestamp_str(3)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(3)) self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)), + 'commit_timestamp=' + self.timestamp_str(2)), '/less than the oldest timestamp/') self.session.rollback_transaction() @@ -92,48 +89,48 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): c[2] = 2 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(2)), + 'commit_timestamp=' + self.timestamp_str(2)), '/less than the oldest timestamp/') self.session.begin_transaction() c[4] = 4 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(4)) + 'commit_timestamp=' + self.timestamp_str(4)) # Oldest timestamp <= Stable timestamp and both oldest and stable # timestamp should proceed forward. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.conn.set_timestamp('oldest_timestamp=' + - timestamp_str(3) + ',stable_timestamp=' + timestamp_str(1)), + self.timestamp_str(3) + ',stable_timestamp=' + self.timestamp_str(1)), '/oldest timestamp \(0, 3\) must not be later than stable timestamp \(0, 1\)/') # Oldest timestamp is 3 at the moment, trying to set it to an earlier # timestamp is a no-op. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) - self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), timestamp_str(3)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), self.timestamp_str(3)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(3) + - ',stable_timestamp=' + timestamp_str(3)) - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(3) + + ',stable_timestamp=' + self.timestamp_str(3)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) # Stable timestamp is 5 at the moment, trying to set it to an earlier # timestamp is a no-op. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(4)) - self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), timestamp_str(5)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(4)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), self.timestamp_str(5)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(5)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.conn.set_timestamp('oldest_timestamp=' + - timestamp_str(6)), + self.timestamp_str(6)), '/oldest timestamp \(0, 6\) must not be later than stable timestamp \(0, 5\)/') # Commit timestamp >= Stable timestamp. # Check both timestamp_transaction and commit_transaction APIs. # Oldest and stable timestamp are set to 5 at the moment. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(6)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(6)) self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(5)), + 'commit_timestamp=' + self.timestamp_str(5)), '/less than the stable timestamp/') self.session.rollback_transaction() @@ -141,7 +138,7 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): c[5] = 5 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(5)), + 'commit_timestamp=' + self.timestamp_str(5)), '/less than the stable timestamp/') # When explicitly set, commit timestamp for a transaction can be earlier @@ -149,48 +146,48 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c[6] = 6 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(6)) + 'commit_timestamp=' + self.timestamp_str(6)) self.session.begin_transaction() c[8] = 8 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(8)) + 'commit_timestamp=' + self.timestamp_str(8)) self.session.begin_transaction() c[7] = 7 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(7)) + 'commit_timestamp=' + self.timestamp_str(7)) # Read timestamp >= Oldest timestamp - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(7) + - ',stable_timestamp=' + timestamp_str(7)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(7) + + ',stable_timestamp=' + self.timestamp_str(7)) with self.expectedStdoutPattern('less than the oldest timestamp'): self.assertRaisesException(wiredtiger.WiredTigerError, - lambda: self.session.begin_transaction('read_timestamp=' + timestamp_str(6))) + lambda: self.session.begin_transaction('read_timestamp=' + self.timestamp_str(6))) # c[8] is not visible at read_timestamp < 8 - self.session.begin_transaction('read_timestamp=' + timestamp_str(7)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(7)) self.assertEqual(c[6], 6) self.assertEqual(c[7], 7) c.set_key(8) self.assertEqual(c.search(), wiredtiger.WT_NOTFOUND) self.session.commit_transaction() - self.session.begin_transaction('read_timestamp=' + timestamp_str(8)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(8)) self.assertEqual(c[6], 6) self.assertEqual(c[7], 7) self.assertEqual(c[8], 8) self.assertTimestampsEqual( - self.conn.query_timestamp('get=oldest_reader'), timestamp_str(8)) + self.conn.query_timestamp('get=oldest_reader'), self.timestamp_str(8)) self.session.commit_transaction() # We can move the oldest timestamp backwards with "force" self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(5) + ',force') + 'oldest_timestamp=' + self.timestamp_str(5) + ',force') with self.expectedStdoutPattern('less than the oldest timestamp'): self.assertRaisesException(wiredtiger.WiredTigerError, - lambda: self.session.begin_transaction('read_timestamp=' + timestamp_str(4))) - self.session.begin_transaction('read_timestamp=' + timestamp_str(6)) + lambda: self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4))) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(6)) self.assertTimestampsEqual( - self.conn.query_timestamp('get=oldest_reader'), timestamp_str(6)) + self.conn.query_timestamp('get=oldest_reader'), self.timestamp_str(6)) self.session.commit_transaction() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp10.py b/src/third_party/wiredtiger/test/suite/test_timestamp10.py index 207370755ab..4af2004c162 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp10.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp10.py @@ -34,9 +34,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): conn_config = 'config_base=false,create,log=(enabled)' session_config = 'isolation=snapshot' @@ -90,16 +87,16 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): curs[i] = i self.pr("i: " + str(i)) self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(i)) + 'commit_timestamp=' + self.timestamp_str(i)) # Set the oldest and stable timestamp a bit earlier than the data # we inserted. Take a checkpoint to the stable timestamp. self.pr("stable ts: " + str(ts)) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(ts) + - ',stable_timestamp=' + timestamp_str(ts)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(ts) + + ',stable_timestamp=' + self.timestamp_str(ts)) # This forces a different checkpoint timestamp for each table. self.session.checkpoint() q = self.conn.query_timestamp('get=last_checkpoint') - self.assertTimestampsEqual(q, timestamp_str(ts)) + self.assertTimestampsEqual(q, self.timestamp_str(ts)) return ts def close_and_recover(self, expected_rec_ts): @@ -124,7 +121,7 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): self.open_conn() q = self.conn.query_timestamp('get=recovery') self.pr("query recovery ts: " + q) - self.assertTimestampsEqual(q, timestamp_str(expected_rec_ts)) + self.assertTimestampsEqual(q, self.timestamp_str(expected_rec_ts)) def test_timestamp_recovery(self): # Add some data and checkpoint at a stable timestamp. diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp11.py b/src/third_party/wiredtiger/test/suite/test_timestamp11.py index de5d51ec45e..c4b4210df43 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp11.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp11.py @@ -33,9 +33,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' @@ -51,7 +48,7 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(2)) + 'commit_timestamp=' + self.timestamp_str(2)) c['key'] = 'value2' c['key2'] = 'value2' self.session.commit_transaction() @@ -64,7 +61,7 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(5)) + 'commit_timestamp=' + self.timestamp_str(5)) c['key'] = 'value5' self.session.commit_transaction() c.close() @@ -81,7 +78,7 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # remain at the non-timestamped value. Also the non-timestamped value # stays regardless of rollbacks or reading at a timestamp. # - stable_ts = timestamp_str(2) + stable_ts = self.timestamp_str(2) self.conn.set_timestamp('stable_timestamp=' + stable_ts) self.session.checkpoint() self.conn.rollback_to_stable() @@ -106,7 +103,7 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(5)) + 'commit_timestamp=' + self.timestamp_str(5)) c['key2'] = 'value5' self.session.commit_transaction() c.close() @@ -140,7 +137,7 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # one at that timestamp and inserted without a timestamp. For the second # we inserted at timestamp 5 after the non-timestamped insert. c = self.session.open_cursor(uri) - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) self.assertEquals(c['key'], 'valueNOTS') self.assertEquals(c['key2'], 'value5') self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp12.py b/src/third_party/wiredtiger/test/suite/test_timestamp12.py index e21ba16158f..e54ed923474 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp12.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp12.py @@ -33,9 +33,6 @@ import shutil, os, wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp12(wttest.WiredTigerTestCase): conn_config = 'config_base=false,create,log=(enabled)' session_config = 'isolation=snapshot' @@ -85,10 +82,10 @@ class test_timestamp12(wttest.WiredTigerTestCase): c_op[i] = 1 c_coll[i] = 1 self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(i)) + 'commit_timestamp=' + self.timestamp_str(i)) # Set the oldest and stable timestamp to the end. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(nentries-1) + - ',stable_timestamp=' + timestamp_str(nentries-1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(nentries-1) + + ',stable_timestamp=' + self.timestamp_str(nentries-1)) # Add more data but don't advance the stable timestamp. for i in second_range: @@ -97,7 +94,7 @@ class test_timestamp12(wttest.WiredTigerTestCase): c_coll[i] = 1 self.pr("i: " + str(i)) self.session.commit_transaction( - 'commit_timestamp=' + timestamp_str(i)) + 'commit_timestamp=' + self.timestamp_str(i)) # Close and reopen the connection. We cannot use reopen_conn because # we want to test the specific close configuration string. diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp14.py b/src/third_party/wiredtiger/test/suite/test_timestamp14.py index c7a0738fa74..8d6e753bf4d 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp14.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp14.py @@ -35,9 +35,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp14' uri = 'table:' + tablename diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp16.py b/src/third_party/wiredtiger/test/suite/test_timestamp16.py index 5a378eae312..f74557ad2c6 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp16.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp16.py @@ -36,9 +36,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp16(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp16' uri = 'table:' + tablename diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp17.py b/src/third_party/wiredtiger/test/suite/test_timestamp17.py index b282ee49b0a..3c2453b0aa2 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp17.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp17.py @@ -38,9 +38,6 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp17(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp17' uri = 'table:' + tablename diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp18.py b/src/third_party/wiredtiger/test/suite/test_timestamp18.py index 746551b1dec..c7f20dbd9a4 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp18.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp18.py @@ -38,9 +38,6 @@ import wiredtiger, wttest from wtscenario import make_scenarios -def timestamp_str(t): - return '%x' % t - class test_timestamp18(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' @@ -53,7 +50,7 @@ class test_timestamp18(wttest.WiredTigerTestCase): def test_ts_writes_with_non_ts_write(self): uri = 'table:test_timestamp18' self.session.create(uri, 'key_format=S,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) value1 = 'a' * 500 @@ -65,17 +62,17 @@ class test_timestamp18(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Add a non-timestamped delete. # Let's do every second key to ensure that we get the truncation right and don't @@ -91,7 +88,7 @@ class test_timestamp18(wttest.WiredTigerTestCase): self.session.checkpoint() for ts in range(2, 4): - self.session.begin_transaction('read_timestamp=' + timestamp_str(ts)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts)) for i in range(1, 10000): # The non-timestamped delete should cover all the previous writes and make them effectively # invisible. diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp19.py b/src/third_party/wiredtiger/test/suite/test_timestamp19.py index fa83d53827a..9f5b71df843 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp19.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp19.py @@ -31,11 +31,8 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet -def timestamp_str(t): - return '%x' % t - class test_timestamp19(wttest.WiredTigerTestCase): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' def updates(self, uri, value, ds, nrows, commit_ts): @@ -44,7 +41,7 @@ class test_timestamp19(wttest.WiredTigerTestCase): for i in range(0, nrows): session.begin_transaction() cursor[ds.key(i)] = value - session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def test_timestamp(self): @@ -62,8 +59,8 @@ class test_timestamp19(wttest.WiredTigerTestCase): value_z = 'z' * 1000 # Set the oldest and stable timestamps to 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ', stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ', stable_timestamp=' + self.timestamp_str(10)) # Insert values with varying timestamps. self.updates(uri, value_x, ds, nrows, 20) @@ -74,8 +71,8 @@ class test_timestamp19(wttest.WiredTigerTestCase): self.session.checkpoint('use_timestamp=true') # Move the oldest and stable timestamps to 40. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(40) + - ', stable_timestamp=' + timestamp_str(40)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(40) + + ', stable_timestamp=' + self.timestamp_str(40)) # Update values. self.updates(uri, value_z, ds, nrows, 50) @@ -91,20 +88,20 @@ class test_timestamp19(wttest.WiredTigerTestCase): self.session = self.setUpSessionOpen(self.conn) # The oldest timestamp on recovery is 40. Trying to set it earlier is a no-op. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10)) - self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), timestamp_str(40)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), self.timestamp_str(40)) # Trying to set an earlier stable timestamp is an error. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)), + lambda: self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)), '/oldest timestamp \(0, 40\) must not be later than stable timestamp \(0, 10\)/') - self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), timestamp_str(40)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), self.timestamp_str(40)) # Move the oldest and stable timestamps to 70. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(70) + - ', stable_timestamp=' + timestamp_str(70)) - self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), timestamp_str(70)) - self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), timestamp_str(70)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(70) + + ', stable_timestamp=' + self.timestamp_str(70)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), self.timestamp_str(70)) + self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), self.timestamp_str(70)) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp20.py b/src/third_party/wiredtiger/test/suite/test_timestamp20.py index c2ff132cd0b..c4edcbfe592 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp20.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp20.py @@ -28,9 +28,6 @@ import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - # test_timestamp20.py # Exercise fixing up of out-of-order updates in the history store. class test_timestamp20(wttest.WiredTigerTestCase): @@ -40,7 +37,7 @@ class test_timestamp20(wttest.WiredTigerTestCase): def test_timestamp20_standard(self): uri = 'table:test_timestamp20' self.session.create(uri, 'key_format=S,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) value1 = 'a' * 500 @@ -52,33 +49,33 @@ class test_timestamp20(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) old_reader_session = self.conn.open_session() old_reader_cursor = old_reader_session.open_cursor(uri) - old_reader_session.begin_transaction('read_timestamp=' + timestamp_str(20)) + old_reader_session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) # Now put two updates out of order. 5 will go to the history store and will trigger a # correction to the existing contents. for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value4 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(25)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(25)) self.session.begin_transaction() cursor[str(i)] = value5 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(40)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) - self.session.begin_transaction('read_timestamp=' + timestamp_str(30)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(30)) for i in range(1, 10000): self.assertEqual(cursor[str(i)], value4) self.session.rollback_transaction() @@ -94,7 +91,7 @@ class test_timestamp20(wttest.WiredTigerTestCase): def test_timestamp20_modify(self): uri = 'table:test_timestamp20' self.session.create(uri, 'key_format=S,value_format=S') - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) value1 = 'a' * 500 @@ -105,20 +102,20 @@ class test_timestamp20(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Now apply a series of modifies. for i in range(1, 10000): self.session.begin_transaction() cursor.set_key(str(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(20)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) for i in range(1, 10000): self.session.begin_transaction() cursor.set_key(str(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) # Open an old reader at this point. # @@ -126,7 +123,7 @@ class test_timestamp20(wttest.WiredTigerTestCase): # has been squashed into a full update. old_reader_session = self.conn.open_session() old_reader_cursor = old_reader_session.open_cursor(uri) - old_reader_session.begin_transaction('read_timestamp=' + timestamp_str(20)) + old_reader_session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) # Now apply the last modify. # This will be the end of the chain of modifies. @@ -134,21 +131,21 @@ class test_timestamp20(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor.set_key(str(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(40)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) # Now put two updates out of order. 5 will go to the history store and will trigger a # correction to the existing contents. for i in range(1, 10000): self.session.begin_transaction() cursor[str(i)] = value2 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(25)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(25)) self.session.begin_transaction() cursor[str(i)] = value3 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(50)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) # Open up a new transaction and read at 30. # We shouldn't be able to see past 5 due to txnid visibility. - self.session.begin_transaction('read_timestamp=' + timestamp_str(30)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(30)) for i in range(1, 10000): self.assertEqual(cursor[str(i)], value2) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp21.py b/src/third_party/wiredtiger/test/suite/test_timestamp21.py index a3b9ca03ac1..6d553a36246 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp21.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp21.py @@ -28,9 +28,6 @@ import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - # test_timestamp21.py # Test read timestamp configuration that allows read timestamp to be older than oldest. class test_timestamp21(wttest.WiredTigerTestCase): @@ -40,58 +37,58 @@ class test_timestamp21(wttest.WiredTigerTestCase): uri = 'table:test_timestamp21' self.session.create(uri, 'key_format=i,value_format=i') session2 = self.setUpSessionOpen(self.conn) - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) cursor2 = session2.open_cursor(uri) # Insert first value at timestamp 10. self.session.begin_transaction() cursor[1] = 1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Begin a read transaction at timestamp 5. - self.session.begin_transaction('read_timestamp=' + timestamp_str(5)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) # Move the oldest timestamp beyond the currently open transactions read timestamp. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(8)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(8)) # Begin a transaction with a read timestamp of 6 and read_before_oldest specified. self.assertEqual(session2.begin_transaction( - 'read_timestamp=' + timestamp_str(6) + ',read_before_oldest=true'), 0) + 'read_timestamp=' + self.timestamp_str(6) + ',read_before_oldest=true'), 0) session2.rollback_transaction() # Begin a transaction with a read timestamp of 6 and no additional config. with self.expectedStdoutPattern('less than the oldest timestamp'): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: session2.begin_transaction( - 'read_timestamp=' + timestamp_str(6))) + 'read_timestamp=' + self.timestamp_str(6))) # Begin a transaction with the config specified but no read timestamp. session2.begin_transaction('read_before_oldest=true') # Set a read timestamp behind the oldest timestamp. - self.assertEqual(session2.timestamp_transaction('read_timestamp=' + timestamp_str(5)), 0) + self.assertEqual(session2.timestamp_transaction('read_timestamp=' + self.timestamp_str(5)), 0) session2.rollback_transaction() # Begin a transaction with a read timestamp of 5 and read_before_oldest specified. self.assertEqual(session2.begin_transaction( - 'read_timestamp=' + timestamp_str(5) + ',read_before_oldest=true'), 0) + 'read_timestamp=' + self.timestamp_str(5) + ',read_before_oldest=true'), 0) session2.rollback_transaction() # Begin a transaction with a read timestamp of 4 and read_before_oldest specified. We get a # different std out message in this scenario. with self.expectedStdoutPattern('less than the pinned timestamp'): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: session2.begin_transaction( - 'read_timestamp=' + timestamp_str(4) + ',read_before_oldest=true')) + 'read_timestamp=' + self.timestamp_str(4) + ',read_before_oldest=true')) # Begin a transaction with a read timestamp of 6 and read_before_oldest off, this will have # the same behaviour as not specifying it. with self.expectedStdoutPattern('less than the oldest timestamp'): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: session2.begin_transaction( - 'read_timestamp=' + timestamp_str(6) + ',read_before_oldest=false')) + 'read_timestamp=' + self.timestamp_str(6) + ',read_before_oldest=false')) # Expect an error when we use roundup timestamps alongside allow read timestamp before # oldest. self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: session2.begin_transaction( - 'read_timestamp=' + timestamp_str(6) + + 'read_timestamp=' + self.timestamp_str(6) + ',read_before_oldest=true,roundup_timestamps=(read)'), '/cannot specify roundup_timestamps.read and read_before_oldest/') diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp22.py b/src/third_party/wiredtiger/test/suite/test_timestamp22.py index d03e18d8d72..e02dbf99e09 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp22.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp22.py @@ -32,9 +32,6 @@ import wiredtiger, wttest, re, suite_random from wtdataset import SimpleDataSet from contextlib import contextmanager -def timestamp_str(t): - return '%x' % t - class test_timestamp22(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' @@ -124,7 +121,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): this_commit_ts = -1 if self.do_illegal(): # setting durable timestamp must be after prepare call - config += ',durable_timestamp=' + timestamp_str(self.gen_ts(commit_ts)) + config += ',durable_timestamp=' + self.timestamp_str(self.gen_ts(commit_ts)) ok = False # ODDITY: if we set the durable timestamp (which is illegal at this point), and set a @@ -143,7 +140,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): else: # It's possible this will succeed, we'll check below. this_commit_ts = self.gen_ts(commit_ts) - config += ',commit_timestamp=' + timestamp_str(this_commit_ts) + config += ',commit_timestamp=' + self.timestamp_str(this_commit_ts) if this_commit_ts >= 0: if this_commit_ts < running_commit_ts: @@ -161,7 +158,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): session = self.session needs_rollback = False prepare_config = None - commit_config = 'commit_timestamp=' + timestamp_str(commit_ts) + commit_config = 'commit_timestamp=' + self.timestamp_str(commit_ts) tstxn1_config = '' tstxn2_config = '' @@ -173,11 +170,11 @@ class test_timestamp22(wttest.WiredTigerTestCase): # Occasionally put a durable timestamp on a commit without a prepare, # that will be an error. if do_prepare or not ok_commit: - commit_config += ',durable_timestamp=' + timestamp_str(durable_ts) + commit_config += ',durable_timestamp=' + self.timestamp_str(durable_ts) cursor = session.open_cursor(self.uri) prepare_ts = self.gen_ts(commit_ts) - prepare_config = 'prepare_timestamp=' + timestamp_str(prepare_ts) - begin_config = '' if read_ts < 0 else 'read_timestamp=' + timestamp_str(read_ts) + prepare_config = 'prepare_timestamp=' + self.timestamp_str(prepare_ts) + begin_config = '' if read_ts < 0 else 'read_timestamp=' + self.timestamp_str(read_ts) # We might do timestamp_transaction calls either before/after inserting # values, or both. @@ -302,7 +299,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): for ts_name in ['oldest', 'stable', 'commit', 'durable']: val = eval(ts_name) if val >= 0: - configs.append(ts_name + '_timestamp=' + timestamp_str(val)) + configs.append(ts_name + '_timestamp=' + self.timestamp_str(val)) return ','.join(configs) # Determine whether we expect the set_timestamp to succeed. @@ -357,8 +354,8 @@ class test_timestamp22(wttest.WiredTigerTestCase): self.pr('updating stable: ' + str(stable)) # Make sure the state of global timestamps is what we think. - expect_query_oldest = timestamp_str(self.oldest_ts) - expect_query_stable = timestamp_str(self.stable_ts) + expect_query_oldest = self.timestamp_str(self.oldest_ts) + expect_query_stable = self.timestamp_str(self.stable_ts) query_oldest = self.conn.query_timestamp('get=oldest') query_stable = self.conn.query_timestamp('get=stable') diff --git a/src/third_party/wiredtiger/test/suite/test_truncate05.py b/src/third_party/wiredtiger/test/suite/test_truncate05.py index eb9e1ec922f..73f9f5444ed 100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate05.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate05.py @@ -28,9 +28,6 @@ import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - # test_truncate05.py # Test various fast truncate visibility scenarios class test_truncate05(wttest.WiredTigerTestCase): @@ -49,7 +46,7 @@ class test_truncate05(wttest.WiredTigerTestCase): for i in range(1, 1000): self.session.begin_transaction() cursor[i] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Reopen the connection to force all content to disk. self.reopen_conn() @@ -59,17 +56,17 @@ class test_truncate05(wttest.WiredTigerTestCase): # Insert a single update at a later timestamp. self.session.begin_transaction() cursor[500] = value2 - self.assertEqual(self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)), 0) + self.assertEqual(self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)), 0) # Insert a bunch of other content to fill the database and evict the committed update. for i in range(1000, 20000): self.session.begin_transaction() cursor[i] = value1 - self.session.commit_transaction('commit_timestamp=' + timestamp_str(4)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Start a transaction with an earlier read timestamp than the commit timestamp of the # previous update. - self.session.begin_transaction('read_timestamp=' + timestamp_str(2)) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) # Truncate from key 1 to 1000. start = self.session.open_cursor(uri, None) diff --git a/src/third_party/wiredtiger/test/suite/test_txn03.py b/src/third_party/wiredtiger/test/suite/test_txn03.py index fd365240301..151b98ca8e2 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn03.py +++ b/src/third_party/wiredtiger/test/suite/test_txn03.py @@ -37,13 +37,18 @@ class test_txn03(wttest.WiredTigerTestCase): tablename = 'test_txn03' uri1 = 'table:' + tablename + "_1" uri2 = 'table:' + tablename + "_2" - key_str = "TEST_KEY1" - data_str1 = "VAL" - data_str2 = "TEST_VAL1" + key = "TEST_KEY1" + data1 = "VAL" + data2 = "TEST_VAL1" nentries = 1000 scenarios = make_scenarios([ - ('var', dict(create_params = "key_format=S,value_format=S")), + ('row', dict(create_params = "key_format=S,value_format=S", + key = "TEST_KEY1", data1 = "VAL", data2 = "TEST_VAL1")), + ('var', dict(create_params = "key_format=r,value_format=S", + key = 123, data1 = "VAL", data2 = "TEST_VAL1")), + ('fix', dict(create_params = "key_format=r,value_format=8t", + key = 123, data1 = 0x17, data2 = 0xaa)), ]) def test_ops(self): @@ -52,17 +57,17 @@ class test_txn03(wttest.WiredTigerTestCase): # Set up the table with entries for 1 and 10 # We use the overwrite config so insert can update as needed. c = self.session.open_cursor(self.uri1, None, 'overwrite') - c[self.key_str] = self.data_str1 + c[self.key] = self.data1 c.close() c = self.session.open_cursor(self.uri2, None, 'overwrite') - c[self.key_str] = self.data_str1 + c[self.key] = self.data1 c.close() # Update the first table - this update should be visible in the # new session. self.session.begin_transaction() c = self.session.open_cursor(self.uri1, None, 'overwrite') - c[self.key_str] = self.data_str2 + c[self.key] = self.data2 self.session.commit_transaction() c.close() @@ -75,16 +80,16 @@ class test_txn03(wttest.WiredTigerTestCase): # Make an update in the first session. self.session.begin_transaction() c = self.session.open_cursor(self.uri2, None, 'overwrite') - c[self.key_str] = self.data_str2 + c[self.key] = self.data2 self.session.commit_transaction() c.close() - t1c.set_key(self.key_str) + t1c.set_key(self.key) t1c.search() - t2c.set_key(self.key_str) + t2c.set_key(self.key) t2c.search() - self.assertEqual(t1c.get_value(), self.data_str2) - self.assertEqual(t2c.get_value(), self.data_str1) + self.assertEqual(t1c.get_value(), self.data2) + self.assertEqual(t2c.get_value(), self.data1) # Clean up t1c.close() diff --git a/src/third_party/wiredtiger/test/suite/test_txn06.py b/src/third_party/wiredtiger/test/suite/test_txn06.py index 86b0913a17b..1bcdac1d4b0 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn06.py +++ b/src/third_party/wiredtiger/test/suite/test_txn06.py @@ -32,6 +32,7 @@ from suite_subprocess import suite_subprocess from wtdataset import SimpleDataSet import wiredtiger, wttest +from wtscenario import make_scenarios class test_txn06(wttest.WiredTigerTestCase, suite_subprocess): conn_config = 'verbose=[transaction]' @@ -40,13 +41,23 @@ class test_txn06(wttest.WiredTigerTestCase, suite_subprocess): source_uri = 'table:' + tablename + "_src" nrows = 100000 + format_values = [ + ('row', dict(key_format = 'S', value_format='S')), + ('var', dict(key_format = 'r', value_format='S')), + ('fix', dict(key_format = 'r', value_format='8t')), + ] + scenarios = make_scenarios(format_values) + def test_long_running(self): # Populate a table - SimpleDataSet(self, self.source_uri, self.nrows).populate() + ds = SimpleDataSet(self, self.source_uri, self.nrows, + key_format=self.key_format, value_format=self.value_format) + ds.populate() # Now scan the table and copy the rows into a new table. The cursor will keep the snapshot # in self.session pinned while the inserts cause new IDs to be allocated. - c_src = self.session.create(self.uri, "key_format=S,value_format=S") + format = "key_format={},value_format={}".format(self.key_format, self.value_format) + c_src = self.session.create(self.uri, format) c_src = self.session.open_cursor(self.source_uri) insert_session = self.conn.open_session() c = insert_session.open_cursor(self.uri) diff --git a/src/third_party/wiredtiger/test/suite/test_txn07.py b/src/third_party/wiredtiger/test/suite/test_txn07.py index 44624f82af9..b25b9edf6cc 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn07.py +++ b/src/third_party/wiredtiger/test/suite/test_txn07.py @@ -49,13 +49,9 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): ] types = [ - ('row', dict(tabletype='row', - create_params = 'key_format=i,value_format=S')), - # The commented columnar tests needs to be enabled once rollback to stable for columnar is fixed in (WT-5548). - # ('var', dict(tabletype='var', - # create_params = 'key_format=r,value_format=S')), - # ('fix', dict(tabletype='fix', - # create_params = 'key_format=r,value_format=8t')), + ('row', dict(tabletype='row', create_params = 'key_format=i,value_format=S')), + ('var', dict(tabletype='var', create_params = 'key_format=r,value_format=S')), + ('fix', dict(tabletype='fix', create_params = 'key_format=r,value_format=8t')), ] op1s = [ ('trunc-all', dict(op1=('all', 0))), diff --git a/src/third_party/wiredtiger/test/suite/test_txn08.py b/src/third_party/wiredtiger/test/suite/test_txn08.py index d8f4188a5d8..9f1032272cd 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn08.py +++ b/src/third_party/wiredtiger/test/suite/test_txn08.py @@ -34,20 +34,27 @@ import fnmatch, os, shutil, run, time from suite_subprocess import suite_subprocess from wiredtiger import stat import wttest +from wtscenario import make_scenarios class test_txn08(wttest.WiredTigerTestCase, suite_subprocess): logmax = "100K" tablename = 'test_txn08' uri = 'table:' + tablename + key_format_values = [ + ('col', dict(key_format='r')), + ('row', dict(key_format='i')) + ] + scenarios = make_scenarios(key_format_values) + # Turn on logging for this test. def conn_config(self): return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ 'transaction_sync="(method=dsync,enabled)"' def test_printlog_unicode(self): - # print "Creating %s with config '%s'" % (self.uri, self.create_params) - create_params = 'key_format=i,value_format=S' + create_params = 'key_format={},value_format=S'.format(self.key_format) + # print "Creating %s with config '%s'" % (self.uri, create_params) self.session.create(self.uri, create_params) c = self.session.open_cursor(self.uri, None) @@ -56,7 +63,7 @@ class test_txn08(wttest.WiredTigerTestCase, suite_subprocess): value = u'\u0001\u0002abcd\u0003\u0004' self.session.begin_transaction() - for k in range(5): + for k in range(1, 6): c[k] = value self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_txn13.py b/src/third_party/wiredtiger/test/suite/test_txn13.py index 72bfe35386d..b11a3cb41be 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn13.py +++ b/src/third_party/wiredtiger/test/suite/test_txn13.py @@ -43,14 +43,20 @@ class test_txn13(wttest.WiredTigerTestCase, suite_subprocess): # We use 8 ops here to get around the 10 operation check done by WiredTiger to determine if # a transaction is blocking or not. nops = 8 - create_params = 'key_format=i,value_format=S' + + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] # The 1gb, 2gb and 4gb scenario names refer to the valuesize * nops. - scenarios = make_scenarios([ + size_values = [ ('1gb', dict(expect_err=False, valuesize=134217728)), ('2gb', dict(expect_err=False, valuesize=268435456)), ('4gb', dict(expect_err=True, valuesize=536870912)) - ]) + ] + + scenarios = make_scenarios(key_format_values, size_values) # Turn on logging for this test. def conn_config(self): @@ -59,9 +65,11 @@ class test_txn13(wttest.WiredTigerTestCase, suite_subprocess): @wttest.longtest('txn tests with huge values') def test_large_values(self): - # print "Creating %s with config '%s'" % (self.uri, self.create_params) + create_params = 'key_format={},value_format=S'.format(self.key_format) + + # print "Creating %s with config '%s'" % (self.uri, create_params) # print "Running with %d" % (self.valuesize) - self.session.create(self.uri, self.create_params) + self.session.create(self.uri, create_params) c = self.session.open_cursor(self.uri, None) # We want to test very large values. Generate 'nops' records within @@ -70,7 +78,7 @@ class test_txn13(wttest.WiredTigerTestCase, suite_subprocess): gotException = False self.session.begin_transaction() - for k in range(self.nops): + for k in range(1, self.nops + 1): value = valuepfx + str(k) c[k] = value diff --git a/src/third_party/wiredtiger/test/suite/test_txn14.py b/src/third_party/wiredtiger/test/suite/test_txn14.py index 24c51023cb4..f3eaac6cd6b 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn14.py +++ b/src/third_party/wiredtiger/test/suite/test_txn14.py @@ -38,7 +38,6 @@ import wttest class test_txn14(wttest.WiredTigerTestCase, suite_subprocess): t1 = 'table:test_txn14_1' - create_params = 'key_format=i,value_format=i' entries = 10000 extra_entries = 5 conn_config = 'log=(archive=false,enabled,file_max=100K)' @@ -46,9 +45,12 @@ class test_txn14(wttest.WiredTigerTestCase, suite_subprocess): sync_list = [ ('write', dict(sync='off')), ('sync', dict(sync='on')), - ('bg', dict(sync='background')), ] - scenarios = make_scenarios(sync_list) + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(sync_list, key_format_values) def test_log_flush(self): # Here's the strategy: @@ -59,32 +61,27 @@ class test_txn14(wttest.WiredTigerTestCase, suite_subprocess): # - Make recovery run. # - Confirm flushed data is in the table. # - self.session.create(self.t1, self.create_params) + create_params = 'key_format={},value_format=i'.format(self.key_format) + self.session.create(self.t1, create_params) c = self.session.open_cursor(self.t1, None, None) - for i in range(self.entries): + for i in range(1, self.entries + 1): c[i] = i + 1 cfgarg='sync=%s' % self.sync self.pr('cfgarg ' + cfgarg) self.session.log_flush(cfgarg) - for i in range(self.extra_entries): + for i in range(1, self.extra_entries + 1): c[i+self.entries] = i + self.entries + 1 c.close() self.session.log_flush(cfgarg) - if self.sync == 'background': - # If doing a background flush, wait 30 seconds. I have seen an - # individual log file's fsync take more than a second on some - # systems, and we've seen timeouts at 10 seconds on systems - # with slow I/O. So give it time to flush perhaps a few files. - self.session.transaction_sync('timeout_ms=30000') simulate_crash_restart(self, ".", "RESTART") c = self.session.open_cursor(self.t1, None, None) - i = 0 + i = 1 for key, value in c: self.assertEqual(i, key) self.assertEqual(i+1, value) i += 1 all = self.entries + self.extra_entries - self.assertEqual(i, all) + self.assertEqual(i, all + 1) c.close() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_txn15.py b/src/third_party/wiredtiger/test/suite/test_txn15.py index 319f9a9ec3a..a266bf37582 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn15.py +++ b/src/third_party/wiredtiger/test/suite/test_txn15.py @@ -38,7 +38,6 @@ import wttest class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:test_txn15_1' - create_params = 'key_format=i,value_format=i' entries = 100 # Turn on logging for this test. def conn_config(self): @@ -48,6 +47,10 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): 'transaction_sync=(enabled=%s),' % self.conn_enable + \ 'transaction_sync=(method=%s),' % self.conn_method + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] conn_sync_enabled = [ ('en_off', dict(conn_enable='false')), ('en_on', dict(conn_enable='true')), @@ -71,7 +74,7 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): ('c_none', dict(commit_sync=None)), ('c_off', dict(commit_sync='sync=off')), ] - scenarios = make_scenarios(conn_sync_enabled, conn_sync_method, + scenarios = make_scenarios(key_format_values, conn_sync_enabled, conn_sync_method, begin_sync, commit_sync) # Given the different configuration settings determine if this group @@ -111,7 +114,8 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): if self.begin_sync != None and self.commit_sync != None: return - self.session.create(self.uri, self.create_params) + create_params = 'key_format={},value_format=i'.format(self.key_format) + self.session.create(self.uri, create_params) stat_cursor = self.session.open_cursor('statistics:', None, None) # @@ -126,7 +130,7 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(self.uri, None, None) self.session.begin_transaction(self.begin_sync) - for i in range(self.entries): + for i in range(1, self.entries + 1): c[i] = i + 1 self.session.commit_transaction(self.commit_sync) c.close() diff --git a/src/third_party/wiredtiger/test/suite/test_txn17.py b/src/third_party/wiredtiger/test/suite/test_txn17.py index 84b7a5feaa3..f1f2c8a2b49 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn17.py +++ b/src/third_party/wiredtiger/test/suite/test_txn17.py @@ -35,16 +35,13 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - class test_txn17(wttest.WiredTigerTestCase, suite_subprocess): def test_txn_api(self): # Test API functionality tagged as requires_transaction. # Cannot set a timestamp on a non-running transaction. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( - 'commit_timestamp=' + timestamp_str(1 << 5000)), + 'commit_timestamp=' + self.timestamp_str(1 << 5000)), '/only permitted in a running/') # Cannot call commit on a non-running transaction. @@ -71,12 +68,5 @@ class test_txn17(wttest.WiredTigerTestCase, suite_subprocess): '/not permitted in a running transaction/') self.session.rollback_transaction() - # Cannot call transaction_sync while a transaction is running. - self.session.begin_transaction() - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.transaction_sync(), - '/not permitted in a running transaction/') - self.session.rollback_transaction() - if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_txn18.py b/src/third_party/wiredtiger/test/suite/test_txn18.py index c764993d682..e8bb10bdbcf 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn18.py +++ b/src/third_party/wiredtiger/test/suite/test_txn18.py @@ -33,15 +33,21 @@ import fnmatch, os, shutil, time from suite_subprocess import suite_subprocess import wiredtiger, wttest +from wtscenario import make_scenarios class test_txn18(wttest.WiredTigerTestCase, suite_subprocess): t1 = 'table:test_txn18' - create_params = 'key_format=i,value_format=i' conn_config = 'log=(archive=false,enabled,file_max=100K),' + \ 'transaction_sync=(method=dsync,enabled)' conn_recerror = conn_config + ',log=(recover=error)' conn_recon = conn_config + ',log=(recover=on)' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def simulate_crash(self, olddir, newdir): ''' Simulate a crash from olddir and restart in newdir. ''' # with the connection still open, copy files to new directory @@ -71,13 +77,14 @@ class test_txn18(wttest.WiredTigerTestCase, suite_subprocess): # # If we aren't tracking file IDs properly, it's possible that # we'd end up apply the log records for t2 to table t1. - self.session.create(self.t1, self.create_params) + create_params = 'key_format={},value_format=i'.format(self.key_format) + self.session.create(self.t1, create_params) # # Since we're logging, we need to flush out the meta-data file # from the create. self.session.checkpoint() c = self.session.open_cursor(self.t1, None, None) - for i in range(10000): + for i in range(1, 10001): c[i] = i + 1 c.close() olddir = "." @@ -103,12 +110,12 @@ class test_txn18(wttest.WiredTigerTestCase, suite_subprocess): # Make sure the data we added originally is there self.session = self.setUpSessionOpen(self.conn) c = self.session.open_cursor(self.t1, None, None) - i = 0 + i = 1 for key, value in c: self.assertEqual(i, key) self.assertEqual(i+1, value) i += 1 - self.assertEqual(i, 10000) + self.assertEqual(i, 10001) c.close() self.close_conn() # Reopening with recover=error after a clean shutdown should succeed. diff --git a/src/third_party/wiredtiger/test/suite/test_txn19.py b/src/third_party/wiredtiger/test/suite/test_txn19.py index 18c0636d603..d975dc43a7c 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn19.py +++ b/src/third_party/wiredtiger/test/suite/test_txn19.py @@ -113,6 +113,11 @@ class test_txn19(wttest.WiredTigerTestCase, suite_subprocess): nrecords = [('nrecords=10', dict(nrecords=10)), ('nrecords=11', dict(nrecords=11))] + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + # This function prunes out unnecessary or problematic test cases # from the list of scenarios. def includeFunc(name, dictarg): @@ -131,11 +136,10 @@ class test_txn19(wttest.WiredTigerTestCase, suite_subprocess): return True scenarios = make_scenarios( - corruption_type, corruption_pos, nrecords, + key_format_values, corruption_type, corruption_pos, nrecords, include=includeFunc, prune=20, prunelong=1000) uri = 'table:test_txn19' - create_params = 'key_format=i,value_format=S' # Return the log file number that contains the given record # number. In this test, two records fit into each log file, and @@ -282,7 +286,8 @@ class test_txn19(wttest.WiredTigerTestCase, suite_subprocess): # Then does a restart with recovery, then starts again with salvage, # and finally starts again with recovery (adding new records). - self.session.create(self.uri, self.create_params) + create_params = 'key_format=i,value_format=S'.format(self.key_format) + self.session.create(self.uri, create_params) self.inserts([x for x in range(0, self.nrecords)]) newdir = "RESTART" copy_for_crash_restart(self.home, newdir) @@ -385,6 +390,11 @@ class test_txn19_meta(wttest.WiredTigerTestCase, suite_subprocess): ('WiredTiger.wt', dict(filename='WiredTiger.wt')), ('WiredTigerHS.wt', dict(filename='WiredTigerHS.wt')), ] + # Configure the database type. + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] # In many cases, wiredtiger_open without any salvage options will # just work. We list those cases here. @@ -433,10 +443,9 @@ class test_txn19_meta(wttest.WiredTigerTestCase, suite_subprocess): "garbage-end:WiredTiger.basecfg", ] - scenarios = make_scenarios(corruption_scenarios, filename_scenarios) + scenarios = make_scenarios(key_format_values, corruption_scenarios, filename_scenarios) uri = 'table:test_txn19_meta_' ntables = 5 - create_params = 'key_format=i,value_format=S' nrecords = 1000 # records per table. suffixes = [ str(x) for x in range(0, ntables)] # [ '0', '1', ... ] @@ -507,11 +516,12 @@ class test_txn19_meta(wttest.WiredTigerTestCase, suite_subprocess): def test_corrupt_meta(self): newdir = "RESTART" newdir2 = "RESTART2" - expect = list(range(0, self.nrecords)) + expect = list(range(1, self.nrecords + 1)) salvage_config = self.base_config + ',salvage=true' + create_params = 'key_format={},value_format=S'.format(self.key_format) for suffix in self.suffixes: - self.session.create(self.uri + suffix, self.create_params) + self.session.create(self.uri + suffix, create_params) self.inserts(expect) # Simulate a crash by copying the contents of the directory diff --git a/src/third_party/wiredtiger/test/suite/test_txn20.py b/src/third_party/wiredtiger/test/suite/test_txn20.py index d51e180fe20..b6d0c8306f2 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn20.py +++ b/src/third_party/wiredtiger/test/suite/test_txn20.py @@ -36,18 +36,21 @@ from wtscenario import make_scenarios class test_txn20(wttest.WiredTigerTestCase): uri = 'table:test_txn' + key_format_values = [ + ('string-row', dict(key_format='S', key='key')), + ('column', dict(key_format='r', key=12)), + ] iso_types = [ ('isolation_read_uncommitted', dict(isolation='read-uncommitted')), ('isolation_read_committed', dict(isolation='read-committed')), ('isolation_snapshot', dict(isolation='snapshot')) ] - scenarios = make_scenarios(iso_types) - key = 'key' + scenarios = make_scenarios(key_format_values, iso_types) old_value = 'value: old' new_value = 'value: new' def test_isolation_level(self): - self.session.create(self.uri, 'key_format=S,value_format=S') + self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(self.uri, None) cursor[self.key] = self.old_value diff --git a/src/third_party/wiredtiger/test/suite/test_txn22.py b/src/third_party/wiredtiger/test/suite/test_txn22.py index 310c6f8a829..42da29540e4 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn22.py +++ b/src/third_party/wiredtiger/test/suite/test_txn22.py @@ -52,6 +52,11 @@ class test_txn22(wttest.WiredTigerTestCase, suite_subprocess): base_config = 'cache_size=1GB' conn_config = base_config + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + # File to be corrupted filename_scenarios = [ ('WiredTiger', dict(filename='WiredTiger')), @@ -75,9 +80,8 @@ class test_txn22(wttest.WiredTigerTestCase, suite_subprocess): "removal:WiredTiger.wt", ] - scenarios = make_scenarios(filename_scenarios) + scenarios = make_scenarios(key_format_values, filename_scenarios) uri = 'table:test_txn22' - create_params = 'key_format=i,value_format=S' nrecords = 1000 # records per table. def valuegen(self, i): @@ -115,10 +119,11 @@ class test_txn22(wttest.WiredTigerTestCase, suite_subprocess): def test_corrupt_meta(self): newdir = "RESTART" newdir2 = "RESTART2" - expect = list(range(0, self.nrecords)) + expect = list(range(1, self.nrecords + 1)) salvage_config = self.base_config + ',salvage=true' - self.session.create(self.uri, self.create_params) + create_params = 'key_format={},value_format=S'.format(self.key_format) + self.session.create(self.uri, create_params) self.inserts(expect) # Simulate a crash by copying the contents of the directory diff --git a/src/third_party/wiredtiger/test/suite/test_txn23.py b/src/third_party/wiredtiger/test/suite/test_txn23.py index 87c2c53a103..c0d420ca92b 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn23.py +++ b/src/third_party/wiredtiger/test/suite/test_txn23.py @@ -32,26 +32,30 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet - -def timestamp_str(t): - return '%x' % t +from wtscenario import make_scenarios class test_txn23(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' conn_config = 'cache_size=5MB' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def large_updates(self, uri, value, ds, nrows, commit_ts): # Update a large number of records. cursor = self.session.open_cursor(uri) - for i in range(0, nrows): + for i in range(1, nrows + 1): self.session.begin_transaction() cursor[ds.key(i)] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() def check(self, check_value, uri, ds, nrows, read_ts): - for i in range(0, nrows): - self.session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + for i in range(1, nrows + 1): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = self.session.open_cursor(uri) self.assertEqual(cursor[ds.key(i)], check_value) cursor.close() @@ -63,18 +67,18 @@ class test_txn23(wttest.WiredTigerTestCase): # Create a table. uri_1 = "table:txn23_1" ds_1 = SimpleDataSet( - self, uri_1, 0, key_format="i", value_format="S") + self, uri_1, 0, key_format=self.key_format, value_format="S") ds_1.populate() # Create another table. uri_2 = "table:txn23_2" ds_2 = SimpleDataSet( - self, uri_2, 0, key_format="i", value_format="S") + self, uri_2, 0, key_format=self.key_format, value_format="S") ds_2.populate() # Pin oldest and stable to timestamp 10. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + - ',stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 diff --git a/src/third_party/wiredtiger/test/suite/test_txn24.py b/src/third_party/wiredtiger/test/suite/test_txn24.py index d6196d6fe85..11bb8a02f4e 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn24.py +++ b/src/third_party/wiredtiger/test/suite/test_txn24.py @@ -33,11 +33,18 @@ import wiredtiger, wttest import time +from wtscenario import make_scenarios class test_txn24(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def conn_config(self): # We want to either eliminate or keep the application thread role in eviction to minimum. # This will ensure that the dedicated eviction threads are doing the heavy lifting. @@ -49,14 +56,14 @@ class test_txn24(wttest.WiredTigerTestCase): # Create and populate a table. uri = "table:test_txn24" - table_params = 'key_format=i,value_format=S' + table_params = 'key_format={},value_format=S'.format(self.key_format) default_val = 'ABCD' * 60 new_val = 'YYYY' * 60 n_rows = 480000 self.session.create(uri, table_params) cursor = self.session.open_cursor(uri, None) - for i in range(0, n_rows): + for i in range(1, n_rows + 1): cursor[i] = default_val cursor.close() @@ -66,7 +73,7 @@ class test_txn24(wttest.WiredTigerTestCase): # Start a transaction, make an update and keep it running. cursor = self.session.open_cursor(uri, None) self.session.begin_transaction('isolation=snapshot') - cursor[0] = new_val + cursor[1] = new_val # Start few sessions and transactions, make updates and try committing them. session2 = self.setUpSessionOpen(self.conn) @@ -94,7 +101,7 @@ class test_txn24(wttest.WiredTigerTestCase): session4 = self.setUpSessionOpen(self.conn) cursor4 = session4.open_cursor(uri) - start_row = 1 + start_row = 2 for i in range(0, 120000): cursor4[start_row] = new_val start_row += 1 diff --git a/src/third_party/wiredtiger/test/suite/test_txn25.py b/src/third_party/wiredtiger/test/suite/test_txn25.py index c8cebdeb05e..6f1b25d4eec 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn25.py +++ b/src/third_party/wiredtiger/test/suite/test_txn25.py @@ -31,17 +31,31 @@ # import wiredtiger, wttest +from wtscenario import make_scenarios class test_txn25(wttest.WiredTigerTestCase): - conn_config = 'cache_size=50MB,log=(enabled),statistics=(all)' + conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' + key_format_values = [ + ('string-row', dict(key_format='S', usestrings=True)), + ('column', dict(key_format='r', usestrings=False)), + ] + scenarios = make_scenarios(key_format_values) + + def getkey(self, i): + if self.usestrings: + return str(i) + else: + return i + def test_txn25(self): uri = 'file:test_txn25' - create_config = 'allocation_size=512,key_format=S,value_format=S' + create_config = 'allocation_size=512,key_format={},value_format=S'.format(self.key_format) self.session.create(uri, create_config) # Populate the file and ensure that we start seeing some high transaction IDs in the system. + nrows = 1000 value1 = 'aaaaa' * 100 value2 = 'bbbbb' * 100 value3 = 'ccccc' * 100 @@ -51,19 +65,19 @@ class test_txn25(wttest.WiredTigerTestCase): session2.begin_transaction() cursor = self.session.open_cursor(uri) - for i in range(1, 1000): + for i in range(1, nrows): self.session.begin_transaction() - cursor[str(i)] = value1 + cursor[self.getkey(i)] = value1 self.session.commit_transaction() - for i in range(1, 1000): + for i in range(1, nrows): self.session.begin_transaction() - cursor[str(i)] = value2 + cursor[self.getkey(i)] = value2 self.session.commit_transaction() - for i in range(1, 1000): + for i in range(1, nrows): self.session.begin_transaction() - cursor[str(i)] = value3 + cursor[self.getkey(i)] = value3 self.session.commit_transaction() session2.rollback_transaction() @@ -81,6 +95,6 @@ class test_txn25(wttest.WiredTigerTestCase): # so we have to wipe the cell's transaction IDs in order to see them. cursor = self.session.open_cursor(uri) self.session.begin_transaction() - for i in range(1, 1000): - self.assertEqual(cursor[str(i)], value3) + for i in range(1, nrows): + self.assertEqual(cursor[self.getkey(i)], value3) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_txn26.py b/src/third_party/wiredtiger/test/suite/test_txn26.py index 4618585b7f1..e5b0a719867 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn26.py +++ b/src/third_party/wiredtiger/test/suite/test_txn26.py @@ -31,37 +31,44 @@ # [END_TAGS] import wiredtiger, wttest +from wtscenario import make_scenarios # test_txn26.py -# Test that commit should fail if commit timestamp is smaller or equal to the active timestamp. Our handling of out of order timestamp relies on this to ensure repeated reads are working as expected. -def timestamp_str(t): - return '%x' % t +# Test that commit should fail if commit timestamp is smaller or equal to the active timestamp. +# Our handling of out of order timestamp relies on this to ensure repeated reads are working as +# expected. class test_txn26(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('string-row', dict(key_format='S', key=str(0))), + ('column', dict(key_format='r', key=16)), + ] + scenarios = make_scenarios(key_format_values) + def test_commit_larger_than_active_timestamp(self): if not wiredtiger.diagnostic_build(): self.skipTest('requires a diagnostic build') uri = 'table:test_txn26' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( - 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) value = 'a' # Start a session with timestamp 10 session2 = self.conn.open_session(self.session_config) - session2.begin_transaction('read_timestamp=' + timestamp_str(10)) + session2.begin_transaction('read_timestamp=' + self.timestamp_str(10)) # Try to commit at timestamp 10 self.session.begin_transaction() - cursor[str(0)] = value + cursor[self.key] = value with self.expectedStderrPattern("must be greater than the latest active read timestamp"): try: - self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) except wiredtiger.WiredTigerError as e: gotException = True self.pr('got expected exception: ' + str(e)) diff --git a/src/third_party/wiredtiger/test/suite/test_util01.py b/src/third_party/wiredtiger/test/suite/test_util01.py index f01eb310c23..20c6cea62d6 100755 --- a/src/third_party/wiredtiger/test/suite/test_util01.py +++ b/src/third_party/wiredtiger/test/suite/test_util01.py @@ -30,9 +30,6 @@ import string, os, sys, random from suite_subprocess import suite_subprocess import wiredtiger, wttest -def timestamp_str(t): - return '%x' % t - # test_util01.py # Utilities: wt dump, as well as the dump cursor class test_util01(wttest.WiredTigerTestCase, suite_subprocess): @@ -164,7 +161,7 @@ class test_util01(wttest.WiredTigerTestCase, suite_subprocess): expectout.write(self.dumpstr(key, hexoutput)) expectout.write(self.dumpstr(value, hexoutput)) if commit_timestamp is not None: - self.session.commit_transaction('commit_timestamp=' + timestamp_str(commit_timestamp)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_timestamp)) def dump(self, usingapi, hexoutput, commit_timestamp, read_timestamp): params = self.table_config() diff --git a/src/third_party/wiredtiger/test/suite/test_util21.py b/src/third_party/wiredtiger/test/suite/test_util21.py index 2271ad8b312..6dc3e05e042 100644 --- a/src/third_party/wiredtiger/test/suite/test_util21.py +++ b/src/third_party/wiredtiger/test/suite/test_util21.py @@ -30,9 +30,6 @@ import wttest from suite_subprocess import suite_subprocess from helper import compare_files -def timestamp_str(t): - return '%x' % t - # test_util21.py # Ensure that wt dump can dump obsolete data in the history store. class test_util21(wttest.WiredTigerTestCase, suite_subprocess): @@ -45,7 +42,7 @@ class test_util21(wttest.WiredTigerTestCase, suite_subprocess): for i in range(1, 5): self.session.begin_transaction() cursor[str(i)] = value - self.session.commit_transaction('commit_timestamp=' + timestamp_str(ts)) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def test_dump_obsolete_data(self): @@ -58,7 +55,7 @@ class test_util21(wttest.WiredTigerTestCase, suite_subprocess): value3 = 'c' * 100 value4 = 'd' * 100 - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) self.add_data_with_timestamp(uri, value1, 2) self.add_data_with_timestamp(uri, value2, 3) @@ -68,14 +65,14 @@ class test_util21(wttest.WiredTigerTestCase, suite_subprocess): self.session.checkpoint() # Set stable timestamp, so we don't lose data when closing/opening connection when using wt dump. - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Call dump on the values before the oldest timestamp is set self.runWt(['dump', 'file:WiredTigerHS.wt'], outfilename="before_oldest") # Set oldest timestamp, and checkpoint, the obsolete data should not removed as # the pages are clean. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(6)) + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(6)) self.session.checkpoint() self.runWt(['dump', 'file:WiredTigerHS.wt'], outfilename="after_oldest") diff --git a/src/third_party/wiredtiger/test/suite/wtbackup.py b/src/third_party/wiredtiger/test/suite/wtbackup.py index fe34d563da8..25de77ba7fa 100644 --- a/src/third_party/wiredtiger/test/suite/wtbackup.py +++ b/src/third_party/wiredtiger/test/suite/wtbackup.py @@ -202,16 +202,21 @@ class backup_base(wttest.WiredTigerTestCase, suite_subprocess): # # Perform a block range copy for a given offset and file. # - def range_copy(self, filename, offset, size, backup_incr_dir): + def range_copy(self, filename, offset, size, backup_incr_dir, consolidate): read_from = filename write_to = backup_incr_dir + '/' + filename rfp = open(read_from, "rb") rfp.seek(offset, 0) buf = rfp.read(size) # Perform between previous incremental directory, to check that - # the old file and the new file is different. + # the old file and the new file is different. We can only ensure + # the data are different if running in consolidate mode. It's + # possible that we change multiple blocks in a single write and + # some of the blocks are the same as before. If we are not running + # in consolidate mode, these blocks which are copied separately one + # by one will trigger this assert. old_to = self.home_tmp + '/' + filename - if os.path.exists(old_to): + if os.path.exists(old_to) and consolidate: self.pr('RANGE CHECK file ' + old_to + ' offset ' + str(offset) + ' len ' + str(size)) old_rfp = open(old_to, "rb") old_rfp.seek(offset, 0) @@ -238,7 +243,7 @@ class backup_base(wttest.WiredTigerTestCase, suite_subprocess): # # Note: we return the sizes of WT_BACKUP_RANGE type files for tests that check for consolidate config. # - def take_incr_backup_block(self, bkup_c, newfile, backup_incr_dir): + def take_incr_backup_block(self, bkup_c, newfile, backup_incr_dir, consolidate): config = 'incremental=(file=' + newfile + ')' self.pr('Open incremental cursor with ' + config) # For each file listed, open a duplicate backup cursor and copy the blocks. @@ -262,7 +267,7 @@ class backup_base(wttest.WiredTigerTestCase, suite_subprocess): else: # Copy the block range. self.pr('Range copy file ' + newfile + ' offset ' + str(offset) + ' len ' + str(size)) - self.range_copy(newfile, offset, size, backup_incr_dir) + self.range_copy(newfile, offset, size, backup_incr_dir, consolidate) lens.append(size) incr_c.close() return lens @@ -319,7 +324,7 @@ class backup_base(wttest.WiredTigerTestCase, suite_subprocess): # If that changes then this, and the use of the duplicate below can change. while bkup_c.next() == 0: newfile = bkup_c.get_key() - file_sizes += self.take_incr_backup_block(bkup_c, newfile, backup_incr_dir) + file_sizes += self.take_incr_backup_block(bkup_c, newfile, backup_incr_dir, consolidate) file_names.append(newfile) # Copy into temp directory for tests that require further iterations of incremental backups. self.copy_file(newfile, self.home_tmp) diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 1d22162f535..01c4f315f9c 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -653,6 +653,9 @@ class WiredTigerTestCase(unittest.TestCase): """ self.assertEqual(int(ts1, 16), int(ts2, 16)) + def timestamp_str(self, t): + return '%x' % t + def exceptionToStderr(self, expr): """ Used by assertRaisesHavingMessage to convert an expression |