From 9ff8d7452ddf7370fc9047ebe9517fd7558914e6 Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Thu, 27 Dec 2018 13:48:37 +1100 Subject: Import wiredtiger: e6c1b9724ed6ed2879a36d7e140f4fa9daceb261 from branch mongodb-4.2 ref: d5793d4dd5..e6c1b9724e for: 4.1.7 WT-4366 Fix how test/format handles prepare conflict errors WT-4426 Change WT data format to include timestamps in leaf page key/value cells WT-4475 clang detected memory leak while executing csuite tests WT-4499 Fix prepared transactions for cursor key order check failure WT-4506 Bypass some csuite tests for valgrind --- .../wiredtiger/test/checkpoint/smoke.sh | 3 + .../test/csuite/wt2246_col_append/main.c | 4 + .../test/csuite/wt2323_join_visibility/main.c | 4 + .../test/csuite/wt2834_join_bloom_fix/main.c | 2 - .../wiredtiger/test/csuite/wt2853_perf/main.c | 3 - src/third_party/wiredtiger/test/format/format.h | 7 ++ src/third_party/wiredtiger/test/format/ops.c | 97 ++++++++++++++++------ .../wiredtiger/test/suite/test_dictionary.py | 70 ++++++++++++++++ .../wiredtiger/test/suite/test_empty_value.py | 60 +++++++++++++ src/third_party/wiredtiger/test/utility/misc.c | 6 +- 10 files changed, 224 insertions(+), 32 deletions(-) create mode 100644 src/third_party/wiredtiger/test/suite/test_dictionary.py create mode 100644 src/third_party/wiredtiger/test/suite/test_empty_value.py (limited to 'src/third_party/wiredtiger/test') diff --git a/src/third_party/wiredtiger/test/checkpoint/smoke.sh b/src/third_party/wiredtiger/test/checkpoint/smoke.sh index 2f1d4345ad7..8db6fc1ebc4 100755 --- a/src/third_party/wiredtiger/test/checkpoint/smoke.sh +++ b/src/third_party/wiredtiger/test/checkpoint/smoke.sh @@ -2,6 +2,9 @@ set -e +# Bypass this test for valgrind +test "$TESTUTIL_BYPASS_VALGRIND" = "1" && exit 0 + # Smoke-test checkpoints as part of running "make check". echo "checkpoint: 3 mixed tables" $TEST_WRAPPER ./t -T 3 -t m diff --git a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c index 2757f991c2a..6df68da932d 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c @@ -98,6 +98,10 @@ main(int argc, char *argv[]) uint64_t i, id; char buf[100]; + /* Bypass this test for valgrind */ + if (testutil_is_flag_set("TESTUTIL_BYPASS_VALGRIND")) + return (EXIT_SUCCESS); + opts = &_opts; memset(opts, 0, sizeof(*opts)); opts->table_type = TABLE_ROW; diff --git a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c index 0b99df76cf3..6cd94ba7572 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c @@ -92,6 +92,10 @@ main(int argc, char *argv[]) TEST_OPTS *opts, _opts; const char *tablename; + /* Bypass this test for valgrind */ + if (testutil_is_flag_set("TESTUTIL_BYPASS_VALGRIND")) + return (EXIT_SUCCESS); + opts = &_opts; sharedopts = &_sharedopts; memset(opts, 0, sizeof(*opts)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c b/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c index a8d44bf3dab..521e67b2439 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c @@ -163,9 +163,7 @@ main(int argc, char *argv[]) testutil_assert(count == 0); testutil_progress(opts, "cleanup starting"); -#if 0 testutil_cleanup(opts); -#endif return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c index 80911ddfd2d..2021ff1849e 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c @@ -186,10 +186,7 @@ main(int argc, char *argv[]) testutil_assert(nfail == 0); testutil_progress(opts, "cleanup starting"); -#if 0 testutil_cleanup(opts); -#endif - return (0); } diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index e9063674476..4b8eadeea1d 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -377,6 +377,13 @@ mmrand(WT_RAND_STATE *rnd, u_int min, u_int max) uint32_t v; u_int range; + /* + * Test runs with small row counts can easily pass a max of 0 (for + * example, "g.rows / 20"). Avoid the problem. + */ + if (min <= max) + return (min); + v = rng(rnd); range = (max - min) + 1; v %= range; diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index f92f438a4f1..d5ed0320761 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -276,6 +276,44 @@ wts_ops(int lastrun) free(tinfo_list); } +typedef enum { NEXT, PREV, SEARCH, SEARCH_NEAR } read_operation; + +/* + * read_op -- + * Perform a read operation, waiting out prepare conflicts. + */ +static inline int +read_op(WT_CURSOR *cursor, read_operation op, int *exactp) +{ + WT_DECL_RET; + + /* + * Read operations wait out prepare-conflicts. (As part of the snapshot + * isolation checks, we repeat reads that succeeded before, they should + * be repeatable.) + */ + switch (op) { + case NEXT: + while ((ret = cursor->next(cursor)) == WT_PREPARE_CONFLICT) + __wt_yield(); + break; + case PREV: + while ((ret = cursor->prev(cursor)) == WT_PREPARE_CONFLICT) + __wt_yield(); + break; + case SEARCH: + while ((ret = cursor->search(cursor)) == WT_PREPARE_CONFLICT) + __wt_yield(); + break; + case SEARCH_NEAR: + while ((ret = + cursor->search_near(cursor, exactp)) == WT_PREPARE_CONFLICT) + __wt_yield(); + break; + } + return (ret); +} + typedef enum { INSERT, MODIFY, READ, REMOVE, TRUNCATE, UPDATE } thread_op; typedef struct { thread_op op; /* Operation */ @@ -401,7 +439,7 @@ snap_check(WT_CURSOR *cursor, } } - switch (ret = cursor->search(cursor)) { + switch (ret = read_op(cursor, SEARCH, NULL)) { case 0: if (g.type == FIX) { testutil_check( @@ -634,12 +672,22 @@ prepare_transaction(TINFO *tinfo, WT_SESSION *session) */ #define OP_FAILED(notfound_ok) do { \ positioned = false; \ - if (intxn && (ret == WT_CACHE_FULL || \ - ret == WT_PREPARE_CONFLICT || ret == WT_ROLLBACK)) \ + if (intxn && (ret == WT_CACHE_FULL || ret == WT_ROLLBACK)) \ goto rollback; \ testutil_assert((notfound_ok && ret == WT_NOTFOUND) || \ - ret == WT_CACHE_FULL || \ - ret == WT_PREPARE_CONFLICT || ret == WT_ROLLBACK); \ + ret == WT_CACHE_FULL || ret == WT_ROLLBACK); \ +} while (0) + +/* + * Rollback updates returning prepare-conflict, they're unlikely to succeed + * unless the prepare aborts. Reads wait out the error, so it's unexpected. + */ +#define READ_OP_FAILED(notfound_ok) \ + OP_FAILED(notfound_ok) +#define WRITE_OP_FAILED(notfound_ok) do { \ + if (ret == WT_PREPARE_CONFLICT) \ + ret = WT_ROLLBACK; \ + OP_FAILED(notfound_ok); \ } while (0) /* @@ -826,7 +874,7 @@ ops(void *arg) positioned = true; SNAP_TRACK(READ, tinfo); } else - OP_FAILED(true); + READ_OP_FAILED(true); } /* Optionally reserve a row. */ @@ -845,7 +893,7 @@ ops(void *arg) __wt_yield(); /* Let other threads proceed. */ } else - OP_FAILED(true); + WRITE_OP_FAILED(true); } /* Perform the operation. */ @@ -875,7 +923,7 @@ ops(void *arg) ++tinfo->insert; SNAP_TRACK(INSERT, tinfo); } else - OP_FAILED(false); + WRITE_OP_FAILED(false); break; case MODIFY: /* @@ -899,7 +947,7 @@ ops(void *arg) positioned = true; SNAP_TRACK(MODIFY, tinfo); } else - OP_FAILED(true); + WRITE_OP_FAILED(true); break; case READ: ++tinfo->search; @@ -908,7 +956,7 @@ ops(void *arg) positioned = true; SNAP_TRACK(READ, tinfo); } else - OP_FAILED(true); + READ_OP_FAILED(true); break; case REMOVE: remove_instead_of_truncate: @@ -929,7 +977,7 @@ remove_instead_of_truncate: */ SNAP_TRACK(REMOVE, tinfo); } else - OP_FAILED(true); + WRITE_OP_FAILED(true); break; case TRUNCATE: /* @@ -958,7 +1006,8 @@ remove_instead_of_truncate: * vice-versa). */ greater_than = mmrand(&tinfo->rnd, 0, 1) == 1; - range = mmrand(&tinfo->rnd, 1, (u_int)g.rows / 20); + range = g.rows < 20 ? + 1 : mmrand(&tinfo->rnd, 1, (u_int)g.rows / 20); tinfo->last = tinfo->keyno; if (greater_than) { if (g.c_reverse) { @@ -992,14 +1041,15 @@ remove_instead_of_truncate: ret = col_truncate(tinfo, cursor); break; } - positioned = false; (void)__wt_atomic_subv64(&g.truncate_cnt, 1); + /* Truncate never leaves the cursor positioned. */ + positioned = false; if (ret == 0) { ++tinfo->truncate; SNAP_TRACK(TRUNCATE, tinfo); } else - OP_FAILED(false); + WRITE_OP_FAILED(false); break; case UPDATE: update_instead_of_chosen_op: @@ -1017,7 +1067,7 @@ update_instead_of_chosen_op: positioned = true; SNAP_TRACK(UPDATE, tinfo); } else - OP_FAILED(false); + WRITE_OP_FAILED(false); break; } @@ -1033,7 +1083,7 @@ update_instead_of_chosen_op: if ((ret = nextprev(tinfo, cursor, next)) == 0) continue; - OP_FAILED(true); + READ_OP_FAILED(true); break; } } @@ -1066,9 +1116,8 @@ update_instead_of_chosen_op: */ if (g.c_prepare && mmrand(&tinfo->rnd, 1, 10) == 1) { ret = prepare_transaction(tinfo, session); - testutil_assert(ret == 0 || ret == WT_PREPARE_CONFLICT); - if (ret == WT_PREPARE_CONFLICT) - goto rollback; + if (ret != 0) + WRITE_OP_FAILED(false); __wt_yield(); /* Let other threads proceed. */ } @@ -1193,11 +1242,11 @@ read_row_worker( } if (sn) { - ret = cursor->search_near(cursor, &exact); + ret = read_op(cursor, SEARCH_NEAR, &exact); if (ret == 0 && exact != 0) ret = WT_NOTFOUND; } else - ret = cursor->search(cursor); + ret = read_op(cursor, SEARCH, NULL); switch (ret) { case 0: if (g.type == FIX) { @@ -1288,7 +1337,7 @@ nextprev(TINFO *tinfo, WT_CURSOR *cursor, bool next) keyno = 0; which = next ? "WT_CURSOR.next" : "WT_CURSOR.prev"; - switch (ret = (next ? cursor->next(cursor) : cursor->prev(cursor))) { + switch (ret = read_op(cursor, next ? NEXT : PREV, NULL)) { case 0: switch (g.type) { case FIX: @@ -2019,7 +2068,7 @@ row_remove(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) } /* We use the cursor in overwrite mode, check for existence. */ - if ((ret = cursor->search(cursor)) == 0) + if ((ret = read_op(cursor, SEARCH, NULL)) == 0) ret = cursor->remove(cursor); if (ret != 0 && ret != WT_NOTFOUND) @@ -2053,7 +2102,7 @@ col_remove(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) cursor->set_key(cursor, tinfo->keyno); /* We use the cursor in overwrite mode, check for existence. */ - if ((ret = cursor->search(cursor)) == 0) + if ((ret = read_op(cursor, SEARCH, NULL)) == 0) ret = cursor->remove(cursor); if (ret != 0 && ret != WT_NOTFOUND) diff --git a/src/third_party/wiredtiger/test/suite/test_dictionary.py b/src/third_party/wiredtiger/test/suite/test_dictionary.py new file mode 100644 index 00000000000..f624e1ade35 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_dictionary.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2018 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_dictionary.py +# Smoke test dictionary compression. + +from wtscenario import make_scenarios +from wtdataset import simple_key +from wiredtiger import stat +import wiredtiger, wttest + +# Smoke test dictionary compression. +class test_dictionary(wttest.WiredTigerTestCase): + conn_config = 'statistics=(all)' + scenarios = make_scenarios([ + ('row', dict(key_format='S', value_format='S')), + ('var', dict(key_format='r', value_format='S')), + ]) + + # Smoke test dictionary compression. + def test_dictionary(self): + nentries = 25000 + uri = 'file:test_dictionary' # This is a btree layer test. + + # Create the object, open the cursor, insert some records with identical values. Use + # alternating values, otherwise column-store will RLE compress them into a single item. + self.session.create(uri, 'dictionary=100,value_format=S,key_format=' + self.key_format) + cursor = self.session.open_cursor(uri, None) + i = 0 + while i < nentries: + i = i + 1 + cursor[simple_key(cursor, i)] = "the same value as the odd items" + i = i + 1 + cursor[simple_key(cursor, i)] = "the same value as the even items" + cursor.close() + + # Checkpoint to force the pages through reconciliation. + self.session.checkpoint() + + # Confirm the dictionary was effective. + cursor = self.session.open_cursor('statistics:' + uri, None, None) + self.assertGreater(cursor[stat.dsrc.rec_dictionary][2], nentries - 100) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_empty_value.py b/src/third_party/wiredtiger/test/suite/test_empty_value.py new file mode 100644 index 00000000000..b40eaaef3d1 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_empty_value.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2018 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_dictionary.py +# Smoke test empty row-store values. + +from wtdataset import simple_key +from wiredtiger import stat +import wiredtiger, wttest + +# Smoke test empty row-store values. +class test_row_store_empty_values(wttest.WiredTigerTestCase): + conn_config = 'statistics=(all)' + + # Smoke test empty row-store values. + def test_row_store_empty_values(self): + nentries = 25000 + uri = 'file:test_empty_values' # This is a btree layer test. + + # Create the object, open the cursor, insert some records with zero-length values. + self.session.create(uri, 'value_format=u,key_format=S') + cursor = self.session.open_cursor(uri, None) + for i in xrange(1, nentries + 1): + cursor[simple_key(cursor, i)] = "" + cursor.close() + + # Reopen to force the object to disk. + self.reopen_conn() + + # Confirm the values weren't stored.. + cursor = self.session.open_cursor('statistics:' + uri, None, 'statistics=(tree_walk)') + self.assertEqual(cursor[stat.dsrc.btree_row_empty_values][2], nentries) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/utility/misc.c b/src/third_party/wiredtiger/test/utility/misc.c index 8d5605208cf..b66ae497707 100644 --- a/src/third_party/wiredtiger/test/utility/misc.c +++ b/src/third_party/wiredtiger/test/utility/misc.c @@ -199,7 +199,7 @@ bool testutil_is_flag_set(const char *flag) { const char *res; - bool enable_long_tests; + bool flag_being_set; if (__wt_getenv(NULL, flag, &res) != 0 || res == NULL) return (false); @@ -208,11 +208,11 @@ testutil_is_flag_set(const char *flag) * This is a boolean test. So if the environment variable is set to any * value other than 0, we return success. */ - enable_long_tests = res[0] != '0'; + flag_being_set = res[0] != '0'; free((void *)res); - return (enable_long_tests); + return (flag_being_set); } /* -- cgit v1.2.1