diff options
author | Michael Cahill <michael.cahill@mongodb.com> | 2016-11-16 21:10:29 +1100 |
---|---|---|
committer | Michael Cahill <michael.cahill@mongodb.com> | 2016-11-16 21:10:29 +1100 |
commit | fb4ae3792065e98696e391ac1c4602216b8502cb (patch) | |
tree | 7d6ab84b45c4eb26bbe59e73a3950e9aa2233f41 /src/third_party | |
parent | 6904d0ac5ea4bba1822103eb4e7a623cc81de641 (diff) | |
download | mongo-fb4ae3792065e98696e391ac1c4602216b8502cb.tar.gz |
Import wiredtiger: ca6eee06ffdacc8e191987e64b3791740dad21e1 from branch mongodb-3.4
ref: 74430da40c..ca6eee06ff
for: 3.4.0
WT-2962 Provide a way to configure builtin extensions
WT-2984 Search of metadata for recently created collection gets WT_NOTFOUND
WT-3000 Missing log records in recovery when crashing after a log file switch
WT-3002 Allow applications to exempt threads from eviction.
WT-3004 lint: declare functions that don't return a value as void
WT-3011 __wt_curjoin_open() saves the wrong URI in the cursor.
WT-3012 Test format hanging on LSM configurations
WT-3015 Test format stuck with 2mb cache
WT-3016 Tests needed for systems without ftruncate
WT-3017 Hazard pointer race with page replace causes error
WT-3018 lint
WT-3020 LSM primary changes impact parallel-pop-lsm load time
WT-3022 LSM operations get stuck in __wt_clsm_await_switch waiting for switch on tree to complete
WT-3023 Test format hang on zSeries
WT-3024 wtperf medium-lsm-compact test can hang
Diffstat (limited to 'src/third_party')
61 files changed, 779 insertions, 456 deletions
diff --git a/src/third_party/wiredtiger/.gitignore b/src/third_party/wiredtiger/.gitignore index bef47daabf9..cd1873533ca 100644 --- a/src/third_party/wiredtiger/.gitignore +++ b/src/third_party/wiredtiger/.gitignore @@ -16,11 +16,12 @@ *.obj *.pdb *.pyc +.dirstamp .sconf_temp .sconsign.dblite -.dirstamp /Makefile.am /aclocal.m4 +/autom4te.cache/ /build_posix/aclocal/libtool.m4 /build_posix/aclocal/ltoptions.m4 /build_posix/aclocal/ltsugar.m4 @@ -32,7 +33,7 @@ /config.status /configure /configure.ac -/libtool/ +/libtool /stamp-h1 /wiredtiger.h /wiredtiger.pc @@ -63,9 +64,9 @@ WT_TEST/ # Python /lang/python/_wiredtiger.so +/lang/python/wiredtiger.py /lang/python/wiredtiger/__init__.py /lang/python/wiredtiger/_wiredtiger.pyd -/lang/python/wiredtiger.py /lang/python/wiredtiger_wrap.c _wiredtiger.pyd @@ -106,6 +107,7 @@ _wiredtiger.pyd **/test/csuite/test_wt2719_reconfig **/test/csuite/test_wt2834_join_bloom_fix **/test/csuite/test_wt2853_perf +**/test/csuite/test_wt2999_join_extractor **/test/cursor_order/cursor_order **/test/fops/t **/test/format/s_dumpcmp diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 22d06c380ae..5a81e8dd080 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -190,9 +190,8 @@ file_config = format_meta + [ WiredTiger to consume memory over the configured cache limit''', type='boolean'), Config('internal_key_truncate', 'true', r''' - configure internal key truncation, discarding unnecessary - trailing bytes on internal keys (ignored for custom - collators)''', + configure internal key truncation, discarding unnecessary trailing + bytes on internal keys (ignored for custom collators)''', type='boolean'), Config('internal_page_max', '4KB', r''' the maximum page size for internal nodes, in bytes; the size @@ -637,6 +636,12 @@ wiredtiger_open_statistics_log_configuration = [ ] session_config = [ + Config('ignore_cache_size', 'false', r''' + when set, operations performed by this session ignore the cache size + and are not blocked when the cache is full. Note that use of this + option for operations that create cache pressure can starve ordinary + sessions that obey the cache size.''', + type='boolean'), Config('isolation', 'read-committed', r''' the default isolation level for operations in this session''', choices=['read-uncommitted', 'read-committed', 'snapshot']), @@ -652,6 +657,11 @@ wiredtiger_open_common =\ should be used (4KB on Linux systems when direct I/O is configured, zero elsewhere)''', min='-1', max='1MB'), + Config('builtin_extension_config', '', r''' + A structure where the keys are the names of builtin extensions and the + values are passed to WT_CONNECTION::load_extension as the \c config + parameter (for example, + <code>builtin_extension_config={zlib={compression_level=3}}</code>)'''), Config('checkpoint_sync', 'true', r''' flush files to stable storage when closing or writing checkpoints''', diff --git a/src/third_party/wiredtiger/dist/flags.py b/src/third_party/wiredtiger/dist/flags.py index e200f95fba6..676f224cbb6 100644 --- a/src/third_party/wiredtiger/dist/flags.py +++ b/src/third_party/wiredtiger/dist/flags.py @@ -114,6 +114,7 @@ flags = { 'session' : [ 'SESSION_CAN_WAIT', 'SESSION_INTERNAL', + 'SESSION_IN_SPLIT', 'SESSION_LOCK_NO_WAIT', 'SESSION_LOCKED_CHECKPOINT', 'SESSION_LOCKED_HANDLE_LIST', diff --git a/src/third_party/wiredtiger/dist/s_all b/src/third_party/wiredtiger/dist/s_all index 31abab28910..4c9d4eccebb 100755 --- a/src/third_party/wiredtiger/dist/s_all +++ b/src/third_party/wiredtiger/dist/s_all @@ -84,22 +84,23 @@ run "sh ./s_copyright" run "sh ./s_style" COMMANDS=" -2>&1 ./s_tags > ${t_pfx}tags 2>&1 ./s_define > ${t_pfx}s_define -2>&1 ./s_typedef -c > ${t_pfx}s_typedef_c -2>&1 ./s_funcs > ${t_pfx}s_funcs +2>&1 ./s_docs > ${t_pfx}s_docs 2>&1 ./s_export > ${t_pfx}s_export +2>&1 ./s_funcs > ${t_pfx}s_funcs 2>&1 ./s_getopt > ${t_pfx}s_getopt 2>&1 ./s_label > ${t_pfx}s_label 2>&1 ./s_lang > ${t_pfx}s_lang 2>&1 ./s_longlines > ${t_pfx}s_longlines +2>&1 ./s_python > ${t_pfx}s_python 2>&1 ./s_stat > ${t_pfx}_stat 2>&1 ./s_string > ${t_pfx}s_string -2>&1 python style.py > ${t_pfx}py_style -2>&1 ./s_python > ${t_pfx}s_python +2>&1 ./s_tags > ${t_pfx}tags +2>&1 ./s_typedef -c > ${t_pfx}s_typedef_c +2>&1 ./s_void > ${t_pfx}s_void" 2>&1 ./s_whitespace > ${t_pfx}s_whitespace 2>&1 ./s_win > ${t_pfx}s_win -2>&1 ./s_docs > ${t_pfx}s_docs" +2>&1 python style.py > ${t_pfx}py_style # Parallelize if possible. xp="" diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 65c74b61995..d19c05e802b 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -863,6 +863,7 @@ lu lwsync lz lzo +mT madvise majorp malloc diff --git a/src/third_party/wiredtiger/dist/s_void b/src/third_party/wiredtiger/dist/s_void index b9c3f5e0ef7..16684e962e6 100644..100755 --- a/src/third_party/wiredtiger/dist/s_void +++ b/src/third_party/wiredtiger/dist/s_void @@ -134,8 +134,12 @@ for f in `find bench ext src test -name '*.[ci]'`; do -e 's/^ *//' | func_ok > $t test -s $t && { - echo "========== $f" - cat $t + echo "==============================================" + echo "$f:" + cat $t | sed 's/^/ /' + echo "Function could return void instead of int." + echo "Add false positives to the list in dist/s_void." + echo "==============================================" } done diff --git a/src/third_party/wiredtiger/examples/c/ex_file_system.c b/src/third_party/wiredtiger/examples/c/ex_file_system.c index 524a5d03f89..f51cad328c3 100644 --- a/src/third_party/wiredtiger/examples/c/ex_file_system.c +++ b/src/third_party/wiredtiger/examples/c/ex_file_system.c @@ -194,19 +194,9 @@ demo_file_system_create(WT_CONNECTION *conn, WT_CONFIG_ARG *config) * the underlying filesystem implementation. See the main function for * the setup of those configuration strings; here we parse configuration * information as passed in by main, through WiredTiger. - * - * Retrieve our configuration information, the "config" value. */ - if ((ret = wtext->config_get(wtext, NULL, config, "config", &v)) != 0) { - (void)wtext->err_printf(wtext, NULL, - "WT_EXTENSION_API.config_get: config: %s", - wtext->strerror(wtext, NULL, ret)); - goto err; - } - - /* Open a WiredTiger parser on the "config" value. */ - if ((ret = wtext->config_parser_open( - wtext, NULL, v.str, v.len, &config_parser)) != 0) { + if ((ret = wtext->config_parser_open_arg( + wtext, NULL, config, &config_parser)) != 0) { (void)wtext->err_printf(wtext, NULL, "WT_EXTENSION_API.config_parser_open: config: %s", wtext->strerror(wtext, NULL, ret)); diff --git a/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c b/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c index 3665ec48b9a..09a793646e7 100644 --- a/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c @@ -483,8 +483,7 @@ static int zlib_init_config( WT_CONNECTION *connection, WT_CONFIG_ARG *config, int *zlib_levelp) { - WT_CONFIG_ITEM k, v; - WT_CONFIG_PARSER *config_parser; + WT_CONFIG_ITEM v; WT_EXTENSION_API *wt_api; int ret, zlib_level; @@ -497,49 +496,27 @@ zlib_init_config( * level; review the configuration. */ wt_api = connection->get_extension_api(connection); - if ((ret = - wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_EXTENSION_API.config_get: zlib configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - if ((ret = wt_api->config_parser_open( - wt_api, NULL, v.str, v.len, &config_parser)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_EXTENSION_API.config_parser_open: zlib configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - while ((ret = config_parser->next(config_parser, &k, &v)) == 0) - if (strlen("compression_level") == k.len && - strncmp("compression_level", k.str, k.len) == 0) { - /* - * Between 0-9: level: see zlib manual. - */ - zlib_level = (int)v.val; - if (zlib_level < 0 || zlib_level > 9) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.next: zlib configure: " - "unsupported compression level %d", - zlib_level); - return (EINVAL); - } - *zlib_levelp = zlib_level; - continue; + if ((ret = wt_api->config_get( + wt_api, NULL, config, "compression_level", &v)) == 0) { + /* + * Between 0-9: level: see zlib manual. + */ + zlib_level = (int)v.val; + if (zlib_level < 0 || zlib_level > 9) { + (void)wt_api->err_printf(wt_api, NULL, + "zlib_init_config: " + "unsupported compression level %d", + zlib_level); + return (EINVAL); } - if (ret != WT_NOTFOUND) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.next: zlib configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - if ((ret = config_parser->close(config_parser)) != 0) { + *zlib_levelp = zlib_level; + } else if (ret != WT_NOTFOUND) { (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.close: zlib configure: %s", + "zlib_init_config: %s", wt_api->strerror(wt_api, NULL, ret)); return (ret); } + return (0); } diff --git a/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c b/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c index a459b01d60a..ea8ec97602f 100644 --- a/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c @@ -232,8 +232,7 @@ static int zstd_init_config( WT_CONNECTION *connection, WT_CONFIG_ARG *config, int *compression_levelp) { - WT_CONFIG_ITEM k, v; - WT_CONFIG_PARSER *config_parser; + WT_CONFIG_ITEM v; WT_EXTENSION_API *wt_api; int ret; @@ -246,38 +245,16 @@ zstd_init_config( * level; review the configuration. */ wt_api = connection->get_extension_api(connection); - if ((ret = - wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_EXTENSION_API.config_get: zstd configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - if ((ret = wt_api->config_parser_open( - wt_api, NULL, v.str, v.len, &config_parser)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_EXTENSION_API.config_parser_open: zstd configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - while ((ret = config_parser->next(config_parser, &k, &v)) == 0) - if (strlen("compression_level") == k.len && - strncmp("compression_level", k.str, k.len) == 0) { - *compression_levelp = (int)v.val; - continue; - } - if (ret != WT_NOTFOUND) { + if ((ret = wt_api->config_get( + wt_api, NULL, config, "compression_level", &v)) == 0) + *compression_levelp = (int)v.val; + else if (ret != WT_NOTFOUND) { (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.next: zstd configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - if ((ret = config_parser->close(config_parser)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.close: zstd configure: %s", + "zstd_init_config: %s", wt_api->strerror(wt_api, NULL, ret)); return (ret); } + return (0); } diff --git a/src/third_party/wiredtiger/ext/datasources/helium/helium.c b/src/third_party/wiredtiger/ext/datasources/helium/helium.c index 473c569f0cc..dff86bd73ac 100644 --- a/src/third_party/wiredtiger/ext/datasources/helium/helium.c +++ b/src/third_party/wiredtiger/ext/datasources/helium/helium.c @@ -3380,15 +3380,9 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) goto err; ds->lockinit = 1; - /* Get the configuration string. */ - if ((ret = wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) - EMSG_ERR(wt_api, NULL, ret, - "WT_EXTENSION_API.config_get: config: %s", - wt_api->strerror(wt_api, NULL, ret)); - /* Step through the list of Helium sources, opening each one. */ - if ((ret = wt_api->config_parser_open( - wt_api, NULL, v.str, v.len, &config_parser)) != 0) + if ((ret = wt_api->config_parser_open_arg( + wt_api, NULL, config, &config_parser)) != 0) EMSG_ERR(wt_api, NULL, ret, "WT_EXTENSION_API.config_parser_open: config: %s", wt_api->strerror(wt_api, NULL, ret)); diff --git a/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c b/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c index 559c8e6e33a..0b905a0540d 100644 --- a/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c +++ b/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c @@ -76,7 +76,7 @@ typedef struct { u_char *shift_forw; /* Encrypt shift data from secretkey */ u_char *shift_back; /* Decrypt shift data from secretkey */ size_t shift_len; /* Length of shift* byte arrays */ - int force_error; /* Force a decrypt error for testing */ + bool force_error; /* Force a decrypt error for testing */ } ROTN_ENCRYPTOR; /*! [WT_ENCRYPTOR initialization structure] */ @@ -429,43 +429,19 @@ rotn_terminate(WT_ENCRYPTOR *encryptor, WT_SESSION *session) static int rotn_configure(ROTN_ENCRYPTOR *rotn_encryptor, WT_CONFIG_ARG *config) { - WT_CONFIG_ITEM k, v; - WT_CONFIG_PARSER *config_parser; + WT_CONFIG_ITEM v; WT_EXTENSION_API *wt_api; /* Extension API */ - int ret, t_ret; + int ret; wt_api = rotn_encryptor->wt_api; /* Get the configuration string. */ - if ((ret = wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) - return (rotn_error(rotn_encryptor, NULL, ret, - "WT_EXTENSION_API.config_get")); - - /* Step through the list of configuration options. */ - if ((ret = wt_api->config_parser_open( - wt_api, NULL, v.str, v.len, &config_parser)) != 0) - return (rotn_error(rotn_encryptor, NULL, ret, - "WT_EXTENSION_API.config_parser_open")); - - while ((ret = config_parser->next(config_parser, &k, &v)) == 0) { - if (strncmp("rotn_force_error", k.str, k.len) == 0 && - strlen("rotn_force_error") == k.len) { - rotn_encryptor->force_error = v.val == 0 ? 0 : 1; - continue; - } else { - if ((ret = config_parser->close(config_parser)) != 0) - return (rotn_error(rotn_encryptor, - NULL, ret, "WT_CONFIG_PARSER.close")); - return (rotn_error(rotn_encryptor, NULL, EINVAL, - "unknown config key")); - } - } - if ((t_ret = config_parser->close(config_parser)) != 0) - return (rotn_error(rotn_encryptor, NULL, t_ret, - "WT_CONFIG_PARSER.close")); - if (ret != WT_NOTFOUND) - return (rotn_error(rotn_encryptor, NULL, ret, - "WT_CONFIG_PARSER.next")); + if ((ret = wt_api->config_get( + wt_api, NULL, config, "rotn_force_error", &v)) == 0) + rotn_encryptor->force_error = v.val != 0; + else if (ret != WT_NOTFOUND) + return (rotn_error(rotn_encryptor, NULL, EINVAL, + "error parsing config")); return (0); } diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index 48522768dc9..05e4dcc098e 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -53,7 +53,6 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_DECL_RET; uint8_t *endp; - WT_UNUSED(addr_size); ci = NULL; /* diff --git a/src/third_party/wiredtiger/src/block/block_read.c b/src/third_party/wiredtiger/src/block/block_read.c index 1eeabd63d92..869a92b6ae1 100644 --- a/src/third_party/wiredtiger/src/block/block_read.c +++ b/src/third_party/wiredtiger/src/block/block_read.c @@ -24,8 +24,6 @@ __wt_bm_preload( uint32_t checksum, size; bool mapped; - WT_UNUSED(addr_size); - block = bm->block; WT_STAT_CONN_INCR(session, block_preload); diff --git a/src/third_party/wiredtiger/src/block/block_slvg.c b/src/third_party/wiredtiger/src/block/block_slvg.c index 5ba95bb598e..b06a5062f50 100644 --- a/src/third_party/wiredtiger/src/block/block_slvg.c +++ b/src/third_party/wiredtiger/src/block/block_slvg.c @@ -168,7 +168,6 @@ __wt_block_salvage_valid(WT_SESSION_IMPL *session, wt_off_t offset; uint32_t size, checksum; - WT_UNUSED(session); WT_UNUSED(addr_size); /* diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 0454ea7c20e..a996b21f7ce 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -325,7 +325,7 @@ __wt_btcur_search(WT_CURSOR_BTREE *cbt) valid = false; if (F_ISSET(cbt, WT_CBT_ACTIVE) && cbt->ref->page->read_gen != WT_READGEN_OLDEST) { - WT_ERR(__wt_txn_cursor_op(session)); + __wt_txn_cursor_op(session); WT_ERR(btree->type == BTREE_ROW ? __cursor_row_search(session, cbt, cbt->ref, false) : @@ -405,7 +405,7 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) if (btree->type == BTREE_ROW && F_ISSET(cbt, WT_CBT_ACTIVE) && cbt->ref->page->read_gen != WT_READGEN_OLDEST) { - WT_ERR(__wt_txn_cursor_op(session)); + __wt_txn_cursor_op(session); WT_ERR(__cursor_row_search(session, cbt, cbt->ref, true)); diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index 337a3ea036f..47c7972dd57 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -341,7 +341,7 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) * always inherit from the connection. */ WT_RET(__wt_config_gets(session, cfg, "encryption.name", &cval)); - if (WT_IS_METADATA(session, btree->dhandle) || cval.len == 0) + if (WT_IS_METADATA(btree->dhandle) || cval.len == 0) btree->kencryptor = conn->kencryptor; else if (WT_STRING_MATCH("none", cval.str, cval.len)) btree->kencryptor = NULL; @@ -432,7 +432,7 @@ __wt_btree_tree_open( * Failure to open metadata means that the database is unavailable. * Try to provide a helpful failure message. */ - if (ret != 0 && WT_IS_METADATA(session, session->dhandle)) { + if (ret != 0 && WT_IS_METADATA(session->dhandle)) { __wt_errx(session, "WiredTiger has failed to open its metadata"); __wt_errx(session, "This may be due to the database" diff --git a/src/third_party/wiredtiger/src/btree/bt_sync.c b/src/third_party/wiredtiger/src/btree/bt_sync.c index 6d4ad9d0d0f..129d7fec05f 100644 --- a/src/third_party/wiredtiger/src/btree/bt_sync.c +++ b/src/third_party/wiredtiger/src/btree/bt_sync.c @@ -133,7 +133,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) if (__wt_page_is_modified(page) && WT_TXNID_LT(page->modify->update_txn, oldest_id)) { if (txn->isolation == WT_ISO_READ_COMMITTED) - WT_ERR(__wt_txn_get_snapshot(session)); + __wt_txn_get_snapshot(session); leaf_bytes += page->memory_footprint; ++leaf_pages; WT_ERR(__wt_reconcile( @@ -155,7 +155,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) * the checkpoint are included. */ if (txn->isolation == WT_ISO_READ_COMMITTED) - WT_ERR(__wt_txn_get_snapshot(session)); + __wt_txn_get_snapshot(session); /* * We cannot check the tree modified flag in the case of a diff --git a/src/third_party/wiredtiger/src/config/config.c b/src/third_party/wiredtiger/src/config/config.c index 3416153d160..a47dfe76aec 100644 --- a/src/third_party/wiredtiger/src/config/config.c +++ b/src/third_party/wiredtiger/src/config/config.c @@ -711,7 +711,7 @@ __wt_config_getones(WT_SESSION_IMPL *session, /* * __wt_config_getones_none -- * Get the value for a given string key from a single config string. - * Treat "none" as empty. + * Treat "none" as empty. */ int __wt_config_getones_none(WT_SESSION_IMPL *session, diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index d57bc418c93..35fea16b1a5 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -24,6 +24,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_load_extension[] = { }; static const WT_CONFIG_CHECK confchk_WT_CONNECTION_open_session[] = { + { "ignore_cache_size", "boolean", NULL, NULL, NULL, 0 }, { "isolation", "string", NULL, "choices=[\"read-uncommitted\",\"read-committed\"," "\"snapshot\"]", @@ -344,6 +345,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_open_cursor[] = { }; static const WT_CONFIG_CHECK confchk_WT_SESSION_reconfigure[] = { + { "ignore_cache_size", "boolean", NULL, NULL, NULL, 0 }, { "isolation", "string", NULL, "choices=[\"read-uncommitted\",\"read-committed\"," "\"snapshot\"]", @@ -657,6 +659,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3 }, { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0 }, + { "builtin_extension_config", "string", NULL, NULL, NULL, 0 }, { "cache_overhead", "int", NULL, "min=0,max=30", NULL, 0 }, { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0 }, { "checkpoint", "category", @@ -742,6 +745,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3 }, { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0 }, + { "builtin_extension_config", "string", NULL, NULL, NULL, 0 }, { "cache_overhead", "int", NULL, "min=0,max=30", NULL, 0 }, { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0 }, { "checkpoint", "category", @@ -828,6 +832,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3 }, { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0 }, + { "builtin_extension_config", "string", NULL, NULL, NULL, 0 }, { "cache_overhead", "int", NULL, "min=0,max=30", NULL, 0 }, { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0 }, { "checkpoint", "category", @@ -908,6 +913,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3 }, { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0 }, + { "builtin_extension_config", "string", NULL, NULL, NULL, 0 }, { "cache_overhead", "int", NULL, "min=0,max=30", NULL, 0 }, { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0 }, { "checkpoint", "category", @@ -1017,8 +1023,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { confchk_WT_CONNECTION_load_extension, 4 }, { "WT_CONNECTION.open_session", - "isolation=read-committed", - confchk_WT_CONNECTION_open_session, 1 + "ignore_cache_size=false,isolation=read-committed", + confchk_WT_CONNECTION_open_session, 2 }, { "WT_CONNECTION.reconfigure", "async=(enabled=false,ops_max=1024,threads=2),cache_overhead=8," @@ -1116,8 +1122,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { NULL, 0 }, { "WT_SESSION.reconfigure", - "isolation=read-committed", - confchk_WT_SESSION_reconfigure, 1 + "ignore_cache_size=false,isolation=read-committed", + confchk_WT_SESSION_reconfigure, 2 }, { "WT_SESSION.rename", "", @@ -1226,62 +1232,64 @@ static const WT_CONFIG_ENTRY config_entries[] = { }, { "wiredtiger_open", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," - "wait=0),checkpoint_sync=true,config_base=true,create=false," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=1,threads_min=1)," - "eviction_checkpoint_target=5,eviction_dirty_target=5," - "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" - ",exclusive=false,extensions=,file_extend=," - "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000,in_memory=false," - "log=(archive=true,compressor=,enabled=false,file_max=100MB," - "path=\".\",prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,readonly=false,session_max=100," - "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," - "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" - ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),transaction_sync=(enabled=false,method=fsync)," + ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," + "config_base=true,create=false,direct_io=,encryption=(keyid=," + "name=,secretkey=),error_prefix=,eviction=(threads_max=1," + "threads_min=1),eviction_checkpoint_target=5," + "eviction_dirty_target=5,eviction_dirty_trigger=20," + "eviction_target=80,eviction_trigger=95,exclusive=false," + "extensions=,file_extend=,file_manager=(close_handle_minimum=250," + "close_idle_time=30,close_scan_interval=10),hazard_max=1000," + "in_memory=false,log=(archive=true,compressor=,enabled=false," + "file_max=100MB,path=\".\",prealloc=true,recover=on," + "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," + "lsm_merge=true,mmap=true,multiprocess=false,readonly=false," + "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" + ",name=,quota=0,reserve=0,size=500MB),statistics=none," + "statistics_log=(json=false,on_close=false,path=\".\",sources=," + "timestamp=\"%b %d %H:%M:%S\",wait=0)," + "transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=," "write_through=", - confchk_wiredtiger_open, 39 + confchk_wiredtiger_open, 40 }, { "wiredtiger_open_all", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," - "wait=0),checkpoint_sync=true,config_base=true,create=false," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=1,threads_min=1)," - "eviction_checkpoint_target=5,eviction_dirty_target=5," - "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" - ",exclusive=false,extensions=,file_extend=," - "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000,in_memory=false," - "log=(archive=true,compressor=,enabled=false,file_max=100MB," - "path=\".\",prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,readonly=false,session_max=100," - "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," - "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" - ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),transaction_sync=(enabled=false,method=fsync)," + ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," + "config_base=true,create=false,direct_io=,encryption=(keyid=," + "name=,secretkey=),error_prefix=,eviction=(threads_max=1," + "threads_min=1),eviction_checkpoint_target=5," + "eviction_dirty_target=5,eviction_dirty_trigger=20," + "eviction_target=80,eviction_trigger=95,exclusive=false," + "extensions=,file_extend=,file_manager=(close_handle_minimum=250," + "close_idle_time=30,close_scan_interval=10),hazard_max=1000," + "in_memory=false,log=(archive=true,compressor=,enabled=false," + "file_max=100MB,path=\".\",prealloc=true,recover=on," + "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," + "lsm_merge=true,mmap=true,multiprocess=false,readonly=false," + "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" + ",name=,quota=0,reserve=0,size=500MB),statistics=none," + "statistics_log=(json=false,on_close=false,path=\".\",sources=," + "timestamp=\"%b %d %H:%M:%S\",wait=0)," + "transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=," "version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_all, 40 + confchk_wiredtiger_open_all, 41 }, { "wiredtiger_open_basecfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," - "wait=0),checkpoint_sync=true,direct_io=,encryption=(keyid=,name=" - ",secretkey=),error_prefix=,eviction=(threads_max=1," - "threads_min=1),eviction_checkpoint_target=5," - "eviction_dirty_target=5,eviction_dirty_trigger=20," - "eviction_target=80,eviction_trigger=95,extensions=,file_extend=," - "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000,log=(archive=true," - "compressor=,enabled=false,file_max=100MB,path=\".\"," - "prealloc=true,recover=on,zero_fill=false)," + ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true,direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=1,threads_min=1)," + "eviction_checkpoint_target=5,eviction_dirty_target=5," + "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" + ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" + ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," + "log=(archive=true,compressor=,enabled=false,file_max=100MB," + "path=\".\",prealloc=true,recover=on,zero_fill=false)," "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," "mmap=true,multiprocess=false,readonly=false,session_max=100," "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," @@ -1289,20 +1297,20 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),transaction_sync=(enabled=false,method=fsync),verbose=," "version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 34 + confchk_wiredtiger_open_basecfg, 35 }, { "wiredtiger_open_usercfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," - "wait=0),checkpoint_sync=true,direct_io=,encryption=(keyid=,name=" - ",secretkey=),error_prefix=,eviction=(threads_max=1," - "threads_min=1),eviction_checkpoint_target=5," - "eviction_dirty_target=5,eviction_dirty_trigger=20," - "eviction_target=80,eviction_trigger=95,extensions=,file_extend=," - "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000,log=(archive=true," - "compressor=,enabled=false,file_max=100MB,path=\".\"," - "prealloc=true,recover=on,zero_fill=false)," + ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true,direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=1,threads_min=1)," + "eviction_checkpoint_target=5,eviction_dirty_target=5," + "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" + ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" + ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," + "log=(archive=true,compressor=,enabled=false,file_max=100MB," + "path=\".\",prealloc=true,recover=on,zero_fill=false)," "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," "mmap=true,multiprocess=false,readonly=false,session_max=100," "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," @@ -1310,7 +1318,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),transaction_sync=(enabled=false,method=fsync),verbose=," "write_through=", - confchk_wiredtiger_open_usercfg, 33 + confchk_wiredtiger_open_usercfg, 34 }, { NULL, NULL, NULL, 0 } }; diff --git a/src/third_party/wiredtiger/src/config/config_ext.c b/src/third_party/wiredtiger/src/config/config_ext.c index 56c0018f8c3..88f1390843a 100644 --- a/src/third_party/wiredtiger/src/config/config_ext.c +++ b/src/third_party/wiredtiger/src/config/config_ext.c @@ -9,22 +9,9 @@ #include "wt_internal.h" /* - * __wt_ext_config_parser_open -- - * WT_EXTENSION_API->config_parser_open implementation - */ -int -__wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, - const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) -{ - WT_UNUSED(wt_ext); - return (wiredtiger_config_parser_open( - wt_session, config, len, config_parserp)); -} - -/* * __wt_ext_config_get -- * Given a NULL-terminated list of configuration strings, find the final - * value for a given string key (external API version). + * value for a given string key (external API version). */ int __wt_ext_config_get(WT_EXTENSION_API *wt_api, @@ -43,3 +30,65 @@ __wt_ext_config_get(WT_EXTENSION_API *wt_api, return (WT_NOTFOUND); return (__wt_config_gets(session, cfg, key, cval)); } + +/* + * __wt_ext_config_get_string -- + * Given a configuration string, find the value for a given string key + * (external API version). + */ +int +__wt_ext_config_get_string(WT_EXTENSION_API *wt_api, + WT_SESSION *wt_session, const char *config, const char *key, + WT_CONFIG_ITEM *cval) +{ + WT_CONNECTION_IMPL *conn; + WT_SESSION_IMPL *session; + + conn = (WT_CONNECTION_IMPL *)wt_api->conn; + if ((session = (WT_SESSION_IMPL *)wt_session) == NULL) + session = conn->default_session; + + return (__wt_config_getones(session, config, key, cval)); +} + +/* + * __wt_ext_config_parser_open -- + * WT_EXTENSION_API->config_parser_open implementation + */ +int +__wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, + const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) +{ + WT_UNUSED(wt_ext); + return (wiredtiger_config_parser_open( + wt_session, config, len, config_parserp)); +} + +/* + * __wt_ext_config_parser_open_arg -- + * WT_EXTENSION_API->config_parser_open_arg implementation + */ +int +__wt_ext_config_parser_open_arg(WT_EXTENSION_API *wt_ext, + WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, + WT_CONFIG_PARSER **config_parserp) +{ + const char **cfg, *p; + size_t len; + + WT_UNUSED(wt_ext); + + /* Find the last non-NULL entry in the configuration stack. */ + if ((cfg = (const char **)cfg_arg) == NULL || *cfg == NULL) { + p = NULL; + len = 0; + } else { + while (cfg[1] != NULL) + ++cfg; + p = *cfg; + len = strlen(p); + } + + return (wiredtiger_config_parser_open( + wt_session, p, len, config_parserp)); +} diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index a6e0f57f02e..1bc4a501ce2 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -239,8 +239,6 @@ __conn_add_compressor(WT_CONNECTION *wt_conn, WT_NAMED_COMPRESSOR *ncomp; WT_SESSION_IMPL *session; - WT_UNUSED(name); - WT_UNUSED(compressor); ncomp = NULL; conn = (WT_CONNECTION_IMPL *)wt_conn; @@ -756,8 +754,11 @@ __conn_get_extension_api(WT_CONNECTION *wt_conn) conn->extension_api.scr_free = __wt_ext_scr_free; conn->extension_api.collator_config = ext_collator_config; conn->extension_api.collate = ext_collate; - conn->extension_api.config_parser_open = __wt_ext_config_parser_open; conn->extension_api.config_get = __wt_ext_config_get; + conn->extension_api.config_get_string = __wt_ext_config_get_string; + conn->extension_api.config_parser_open = __wt_ext_config_parser_open; + conn->extension_api.config_parser_open_arg = + __wt_ext_config_parser_open_arg; conn->extension_api.metadata_insert = __wt_ext_metadata_insert; conn->extension_api.metadata_remove = __wt_ext_metadata_remove; conn->extension_api.metadata_search = __wt_ext_metadata_search; @@ -789,40 +790,75 @@ __conn_get_extension_api(WT_CONNECTION *wt_conn) return (&conn->extension_api); } +/* + * __conn_builtin_init -- + * Initialize and configure a builtin extension. + */ +static int +__conn_builtin_init(WT_CONNECTION_IMPL *conn, const char *name, + int (*extension_init)(WT_CONNECTION *, WT_CONFIG_ARG *), + const char *cfg[]) +{ + WT_CONFIG_ITEM all_configs, cval; + WT_DECL_RET; + WT_SESSION_IMPL *session; + char *config; + const char *ext_cfg[] = { NULL, NULL }; + + session = conn->default_session; + + WT_RET(__wt_config_gets( + session, cfg, "builtin_extension_config", &all_configs)); + WT_CLEAR(cval); + WT_RET_NOTFOUND_OK(__wt_config_subgets( + session, &all_configs, name, &cval)); + WT_RET(__wt_strndup(session, cval.str, cval.len, &config)); + ext_cfg[0] = config; + + ret = extension_init(&conn->iface, (WT_CONFIG_ARG *)ext_cfg); + __wt_free(session, config); + + return (ret); +} + #ifdef HAVE_BUILTIN_EXTENSION_LZ4 - extern int lz4_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); +extern int lz4_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); #endif #ifdef HAVE_BUILTIN_EXTENSION_SNAPPY - extern int snappy_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); +extern int snappy_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); #endif #ifdef HAVE_BUILTIN_EXTENSION_ZLIB - extern int zlib_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); +extern int zlib_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); #endif #ifdef HAVE_BUILTIN_EXTENSION_ZSTD - extern int zstd_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); +extern int zstd_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); #endif /* - * __conn_load_default_extensions -- + * __conn_builtin_extensions -- * Load extensions that are enabled via --with-builtins */ static int -__conn_load_default_extensions(WT_CONNECTION_IMPL *conn) +__conn_builtin_extensions(WT_CONNECTION_IMPL *conn, const char *cfg[]) { - WT_UNUSED(conn); - #ifdef HAVE_BUILTIN_EXTENSION_LZ4 - WT_RET(lz4_extension_init(&conn->iface, NULL)); + WT_RET(__conn_builtin_init(conn, "lz4", lz4_extension_init, cfg)); #endif #ifdef HAVE_BUILTIN_EXTENSION_SNAPPY - WT_RET(snappy_extension_init(&conn->iface, NULL)); + WT_RET(__conn_builtin_init(conn, "snappy", snappy_extension_init, cfg)); #endif #ifdef HAVE_BUILTIN_EXTENSION_ZLIB - WT_RET(zlib_extension_init(&conn->iface, NULL)); + WT_RET(__conn_builtin_init(conn, "zlib", zlib_extension_init, cfg)); #endif #ifdef HAVE_BUILTIN_EXTENSION_ZSTD - WT_RET(zstd_extension_init(&conn->iface, NULL)); + WT_RET(__conn_builtin_init(conn, "zstd", zstd_extension_init, cfg)); #endif + + /* Avoid warnings if no builtin extensions are configured. */ + WT_UNUSED(conn); + WT_UNUSED(cfg); + WT_UNUSED(__conn_builtin_init); + return (0); } @@ -839,10 +875,11 @@ __conn_load_extension_int(WT_SESSION_IMPL *session, WT_DLH *dlh; int (*load)(WT_CONNECTION *, WT_CONFIG_ARG *); bool is_local; - const char *init_name, *terminate_name; + const char *ext_config, *init_name, *terminate_name; + const char *ext_cfg[2]; dlh = NULL; - init_name = terminate_name = NULL; + ext_config = init_name = terminate_name = NULL; is_local = strcmp(path, "local") == 0; /* Ensure that the load matches the phase of startup we are in. */ @@ -872,8 +909,14 @@ __conn_load_extension_int(WT_SESSION_IMPL *session, WT_ERR( __wt_dlsym(session, dlh, terminate_name, false, &dlh->terminate)); + WT_CLEAR(cval); + WT_ERR_NOTFOUND_OK(__wt_config_gets(session, cfg, "config", &cval)); + WT_ERR(__wt_strndup(session, cval.str, cval.len, &ext_config)); + ext_cfg[0] = ext_config; + ext_cfg[1] = NULL; + /* Call the load function last, it simplifies error handling. */ - WT_ERR(load(&S2C(session)->iface, (WT_CONFIG_ARG *)cfg)); + WT_ERR(load(&S2C(session)->iface, (WT_CONFIG_ARG *)ext_cfg)); /* Link onto the environment's list of open libraries. */ __wt_spin_lock(session, &S2C(session)->api_lock); @@ -883,6 +926,7 @@ __conn_load_extension_int(WT_SESSION_IMPL *session, err: if (dlh != NULL) WT_TRET(__wt_dlclose(session, dlh)); + __wt_free(session, ext_config); __wt_free(session, init_name); __wt_free(session, terminate_name); return (ret); @@ -2355,7 +2399,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, * everything else to be in place, and the extensions call back into the * library. */ - WT_ERR(__conn_load_default_extensions(conn)); + WT_ERR(__conn_builtin_extensions(conn, cfg)); WT_ERR(__conn_load_extensions(session, cfg, false)); /* diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index ac72e330b67..e9e3925c57e 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -447,7 +447,7 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, F_ISSET(dhandle, WT_DHANDLE_DEAD) || dhandle->checkpoint != NULL || !WT_PREFIX_MATCH(dhandle->name, "file:") || - WT_IS_METADATA(session, dhandle)) + WT_IS_METADATA(dhandle)) continue; WT_RET(__conn_btree_apply_internal( session, dhandle, file_func, name_func, cfg)); @@ -627,7 +627,7 @@ __wt_conn_dhandle_discard(WT_SESSION_IMPL *session) */ restart: TAILQ_FOREACH(dhandle, &conn->dhqh, q) { - if (WT_IS_METADATA(session, dhandle)) + if (WT_IS_METADATA(dhandle)) continue; WT_WITH_DHANDLE(session, dhandle, diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index 69b50147bf5..6454503d6cb 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -95,7 +95,8 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) for (;;) { WT_TRET(__wt_txn_update_oldest(session, WT_TXN_OLDEST_STRICT | WT_TXN_OLDEST_WAIT)); - if (txn_global->oldest_id == txn_global->current) + if (txn_global->oldest_id == txn_global->current && + txn_global->metadata_pinned == txn_global->current) break; __wt_yield(); } diff --git a/src/third_party/wiredtiger/src/conn/conn_sweep.c b/src/third_party/wiredtiger/src/conn/conn_sweep.c index dba37fa2eb0..d1254d8afcc 100644 --- a/src/third_party/wiredtiger/src/conn/conn_sweep.c +++ b/src/third_party/wiredtiger/src/conn/conn_sweep.c @@ -26,7 +26,7 @@ __sweep_mark(WT_SESSION_IMPL *session, time_t now) conn = S2C(session); TAILQ_FOREACH(dhandle, &conn->dhqh, q) { - if (WT_IS_METADATA(session, dhandle)) + if (WT_IS_METADATA(dhandle)) continue; /* @@ -122,7 +122,7 @@ __sweep_expire(WT_SESSION_IMPL *session, time_t now) if (conn->open_btree_count < conn->sweep_handles_min) break; - if (WT_IS_METADATA(session, dhandle) || + if (WT_IS_METADATA(dhandle) || !F_ISSET(dhandle, WT_DHANDLE_OPEN) || dhandle->session_inuse != 0 || dhandle->timeofdeath == 0 || @@ -228,7 +228,7 @@ __sweep_remove_handles(WT_SESSION_IMPL *session) dhandle != NULL; dhandle = dhandle_next) { dhandle_next = TAILQ_NEXT(dhandle, q); - if (WT_IS_METADATA(session, dhandle)) + if (WT_IS_METADATA(dhandle)) continue; if (!WT_DHANDLE_CAN_DISCARD(dhandle)) continue; diff --git a/src/third_party/wiredtiger/src/cursor/cur_config.c b/src/third_party/wiredtiger/src/cursor/cur_config.c index 2d3f3ffd176..4001188e21c 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_config.c +++ b/src/third_party/wiredtiger/src/cursor/cur_config.c @@ -49,8 +49,6 @@ __wt_curconfig_open(WT_SESSION_IMPL *session, WT_STATIC_ASSERT(offsetof(WT_CURSOR_CONFIG, iface) == 0); - WT_UNUSED(uri); - WT_RET(__wt_calloc_one(session, &cconfig)); cursor = &cconfig->iface; diff --git a/src/third_party/wiredtiger/src/cursor/cur_ds.c b/src/third_party/wiredtiger/src/cursor/cur_ds.c index 458e0f1b1f0..131d1ffa930 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_ds.c +++ b/src/third_party/wiredtiger/src/cursor/cur_ds.c @@ -12,13 +12,11 @@ * __curds_txn_enter -- * Do transactional initialization when starting an operation. */ -static int +static void __curds_txn_enter(WT_SESSION_IMPL *session) { session->ncursors++; /* XXX */ - WT_RET(__wt_txn_cursor_op(session)); - - return (0); + __wt_txn_cursor_op(session); } /* @@ -187,7 +185,7 @@ __curds_next(WT_CURSOR *cursor) WT_STAT_CONN_INCR(session, cursor_next); WT_STAT_DATA_INCR(session, cursor_next); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); ret = __curds_cursor_resolve(cursor, source->next(source)); @@ -215,7 +213,7 @@ __curds_prev(WT_CURSOR *cursor) WT_STAT_CONN_INCR(session, cursor_prev); WT_STAT_DATA_INCR(session, cursor_prev); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); ret = __curds_cursor_resolve(cursor, source->prev(source)); @@ -267,7 +265,7 @@ __curds_search(WT_CURSOR *cursor) WT_STAT_CONN_INCR(session, cursor_search); WT_STAT_DATA_INCR(session, cursor_search); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); WT_ERR(__curds_key_set(cursor)); ret = __curds_cursor_resolve(cursor, source->search(source)); @@ -295,7 +293,7 @@ __curds_search_near(WT_CURSOR *cursor, int *exact) WT_STAT_CONN_INCR(session, cursor_search_near); WT_STAT_DATA_INCR(session, cursor_search_near); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); WT_ERR(__curds_key_set(cursor)); ret = @@ -321,7 +319,7 @@ __curds_insert(WT_CURSOR *cursor) CURSOR_UPDATE_API_CALL(cursor, session, insert, NULL); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); WT_STAT_CONN_INCR(session, cursor_insert); WT_STAT_DATA_INCR(session, cursor_insert); @@ -358,7 +356,7 @@ __curds_update(WT_CURSOR *cursor) WT_STAT_DATA_INCR(session, cursor_update); WT_STAT_DATA_INCRV(session, cursor_update_bytes, cursor->value.size); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); WT_ERR(__curds_key_set(cursor)); WT_ERR(__curds_value_set(cursor)); @@ -389,7 +387,7 @@ __curds_remove(WT_CURSOR *cursor) WT_STAT_DATA_INCR(session, cursor_remove); WT_STAT_DATA_INCRV(session, cursor_remove_bytes, cursor->key.size); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); WT_ERR(__curds_key_set(cursor)); ret = __curds_cursor_resolve(cursor, source->remove(source)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_join.c b/src/third_party/wiredtiger/src/cursor/cur_join.c index 806436ebf38..2fa2a207c8a 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_join.c +++ b/src/third_party/wiredtiger/src/cursor/cur_join.c @@ -326,8 +326,7 @@ __curjoin_close(WT_CURSOR *cursor) JOINABLE_CURSOR_API_CALL(cursor, session, close, NULL); __wt_schema_release_table(session, cjoin->table); - /* These are owned by the table */ - cursor->internal_uri = NULL; + /* This is owned by the table */ cursor->key_format = NULL; if (cjoin->projection != NULL) { __wt_free(session, cjoin->projection); @@ -921,7 +920,7 @@ __curjoin_init_next(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, "cursors"); /* Get a consistent view of our subordinate cursors if appropriate. */ - WT_RET(__wt_txn_cursor_op(session)); + __wt_txn_cursor_op(session); if (F_ISSET((WT_CURSOR *)cjoin, WT_CURSTD_RAW)) config = &raw_cfg[0]; @@ -1310,11 +1309,10 @@ __wt_curjoin_open(WT_SESSION_IMPL *session, WT_RET_MSG(session, EINVAL, "unable to initialize a join cursor with existing owner"); - if (!WT_PREFIX_SKIP(uri, "join:")) - return (__wt_unexpected_object_type(session, uri, "join:")); tablename = uri; - if (!WT_PREFIX_SKIP(tablename, "table:")) - return (__wt_unexpected_object_type(session, uri, "table:")); + if (!WT_PREFIX_SKIP(tablename, "join:table:")) + return ( + __wt_unexpected_object_type(session, uri, "join:table:")); columns = strchr(tablename, '('); if (columns == NULL) @@ -1327,7 +1325,6 @@ __wt_curjoin_open(WT_SESSION_IMPL *session, cursor = &cjoin->iface; *cursor = iface; cursor->session = &session->iface; - cursor->internal_uri = table->name; cursor->key_format = table->key_format; cursor->value_format = table->value_format; cjoin->table = table; diff --git a/src/third_party/wiredtiger/src/cursor/cur_json.c b/src/third_party/wiredtiger/src/cursor/cur_json.c index 093ec3c59ac..4ba10ddabb0 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_json.c +++ b/src/third_party/wiredtiger/src/cursor/cur_json.c @@ -420,7 +420,8 @@ __wt_json_column_init(WT_CURSOR *cursor, const char *keyformat, const char *_bad = in; \ while (__wt_isalnum((u_char)*in)) \ in++; \ - __wt_errx(session, "unknown keyword \"%.*s\" in JSON", \ + WT_RET_MSG(session, EINVAL, \ + "unknown keyword \"%.*s\" in JSON", \ (int)(in - _bad), _bad); \ } \ } while (0) @@ -501,9 +502,9 @@ __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, } src++; } - if (result != 's') - __wt_errx(session, "unterminated string in JSON"); - break; + if (result == 's') + break; + WT_RET_MSG(session, EINVAL, "unterminated string in JSON"); case '-': case '0': case '1': @@ -561,15 +562,14 @@ __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, if (isalph) while (*src != '\0' && __wt_isalnum((u_char)*src)) src++; - __wt_errx(session, "unknown token \"%.*s\" in JSON", - (int)(src - bad), bad); - break; + WT_RET_MSG(session, EINVAL, + "unknown token \"%.*s\" in JSON", (int)(src - bad), bad); + /* NOTREACHED */ } + WT_ASSERT(session, result != -1); + *toklen = (size_t)(src - *tokstart); *toktype = result; - - if (result < 0) - WT_RET_MSG(session, EINVAL, "illegal token in JSON"); return (0); } diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 909fe813f98..ee30bd4b5b3 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -1449,7 +1449,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, /* Limit internal pages to 50% of the total. */ if (WT_PAGE_IS_INTERNAL(page) && - internal_pages >= (int)(evict - start) / 2) + internal_pages > (int)(evict - start) / 2) continue; /* If eviction gets aggressive, anything else is fair game. */ @@ -1498,11 +1498,13 @@ fast: /* If the page can't be evicted, give up. */ */ if (give_up) btree->evict_walk_reverse = !btree->evict_walk_reverse; - if (give_up && !urgent_queued) + if (pages_queued == 0 && !urgent_queued) btree->evict_walk_period = WT_MIN( WT_MAX(1, 2 * btree->evict_walk_period), 100); else if (pages_queued == target_pages) btree->evict_walk_period = 0; + else if (btree->evict_walk_period > 0) + btree->evict_walk_period /= 2; /* * If we happen to end up on the root page or a page requiring urgent diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index 3d1557e027e..b15e1c1f26c 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -163,8 +163,19 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool closing) */ WT_ERR(__evict_page_clean_update( session, ref, tree_dead || closing)); - else - WT_ERR(__evict_page_dirty_update(session, ref, closing)); + else { + /* + * The page is not being completely evicted: instead it is + * being split or replaced. In that case, don't increment the + * count of pages evicted, which we use to decide whether + * eviction is making progress. Repeatedly rewriting the same + * page isn't progress. + */ + F_SET(session, WT_SESSION_IN_SPLIT); + ret = __evict_page_dirty_update(session, ref, closing); + F_CLR(session, WT_SESSION_IN_SPLIT); + WT_ERR(ret); + } if (clean_page) { WT_STAT_CONN_INCR(session, cache_eviction_clean); diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h index 713d46ae85f..0e0f7b4c40a 100644 --- a/src/third_party/wiredtiger/src/include/btree.h +++ b/src/third_party/wiredtiger/src/include/btree.h @@ -158,15 +158,16 @@ struct __wt_btree { #define WT_BTREE_IGNORE_CACHE 0x000200 /* Cache-resident object */ #define WT_BTREE_IN_MEMORY 0x000400 /* Cache-resident object */ #define WT_BTREE_LOOKASIDE 0x000800 /* Look-aside table */ -#define WT_BTREE_NO_CHECKPOINT 0x001000 /* Disable checkpoints */ -#define WT_BTREE_NO_EVICTION 0x002000 /* Disable eviction */ -#define WT_BTREE_NO_LOGGING 0x004000 /* Disable logging */ -#define WT_BTREE_NO_RECONCILE 0x008000 /* Allow splits, even with no evict */ -#define WT_BTREE_REBALANCE 0x010000 /* Handle is for rebalance */ -#define WT_BTREE_SALVAGE 0x020000 /* Handle is for salvage */ -#define WT_BTREE_SKIP_CKPT 0x040000 /* Handle skipped checkpoint */ -#define WT_BTREE_UPGRADE 0x080000 /* Handle is for upgrade */ -#define WT_BTREE_VERIFY 0x100000 /* Handle is for verify */ +#define WT_BTREE_LSM_PRIMARY 0x001000 /* Handle is current LSM primary */ +#define WT_BTREE_NO_CHECKPOINT 0x002000 /* Disable checkpoints */ +#define WT_BTREE_NO_EVICTION 0x004000 /* Disable eviction */ +#define WT_BTREE_NO_LOGGING 0x008000 /* Disable logging */ +#define WT_BTREE_NO_RECONCILE 0x010000 /* Allow splits, even with no evict */ +#define WT_BTREE_REBALANCE 0x020000 /* Handle is for rebalance */ +#define WT_BTREE_SALVAGE 0x040000 /* Handle is for salvage */ +#define WT_BTREE_SKIP_CKPT 0x080000 /* Handle skipped checkpoint */ +#define WT_BTREE_UPGRADE 0x100000 /* Handle is for upgrade */ +#define WT_BTREE_VERIFY 0x200000 /* Handle is for verify */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index e48189d50ea..ad603f3ea53 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -107,7 +107,7 @@ __wt_cache_page_inmem_incr(WT_SESSION_IMPL *session, WT_PAGE *page, size_t size) (void)__wt_atomic_addsize(&page->modify->bytes_dirty, size); if (WT_PAGE_IS_INTERNAL(page)) (void)__wt_atomic_add64(&cache->bytes_dirty_intl, size); - else { + else if (!F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) { (void)__wt_atomic_add64(&cache->bytes_dirty_leaf, size); (void)__wt_atomic_add64(&btree->bytes_dirty_leaf, size); } @@ -241,7 +241,7 @@ __wt_cache_page_byte_dirty_decr( if (WT_PAGE_IS_INTERNAL(page)) __wt_cache_decr_check_uint64(session, &cache->bytes_dirty_intl, decr, "WT_CACHE.bytes_dirty_intl"); - else { + else if (!F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) { __wt_cache_decr_check_uint64(session, &btree->bytes_dirty_leaf, decr, "WT_BTREE.bytes_dirty_leaf"); __wt_cache_decr_check_uint64(session, &cache->bytes_dirty_leaf, @@ -300,8 +300,10 @@ __wt_cache_dirty_incr(WT_SESSION_IMPL *session, WT_PAGE *page) (void)__wt_atomic_add64(&cache->bytes_dirty_intl, size); (void)__wt_atomic_add64(&cache->pages_dirty_intl, 1); } else { - (void)__wt_atomic_add64(&btree->bytes_dirty_leaf, size); - (void)__wt_atomic_add64(&cache->bytes_dirty_leaf, size); + if (!F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) { + (void)__wt_atomic_add64(&btree->bytes_dirty_leaf, size); + (void)__wt_atomic_add64(&cache->bytes_dirty_leaf, size); + } (void)__wt_atomic_add64(&cache->pages_dirty_leaf, 1); } (void)__wt_atomic_addsize(&page->modify->bytes_dirty, size); @@ -394,7 +396,7 @@ __wt_cache_page_evict(WT_SESSION_IMPL *session, WT_PAGE *page) __wt_cache_decr_zero_uint64(session, &cache->bytes_dirty_intl, modify->bytes_dirty, "WT_CACHE.bytes_dirty_intl"); - else { + else if (!F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) { __wt_cache_decr_zero_uint64(session, &cache->bytes_dirty_leaf, modify->bytes_dirty, "WT_CACHE.bytes_dirty_leaf"); @@ -406,7 +408,11 @@ __wt_cache_page_evict(WT_SESSION_IMPL *session, WT_PAGE *page) /* Update pages and bytes evicted. */ (void)__wt_atomic_add64(&cache->bytes_evict, page->memory_footprint); - (void)__wt_atomic_addv64(&cache->pages_evict, 1); + + if (F_ISSET(session, WT_SESSION_IN_SPLIT)) + (void)__wt_atomic_subv64(&cache->pages_inmem, 1); + else + (void)__wt_atomic_addv64(&cache->pages_evict, 1); } /* @@ -1543,6 +1549,55 @@ __wt_btree_lsm_over_size(WT_SESSION_IMPL *session, uint64_t maxsize) } /* + * __wt_btree_lsm_switch_primary -- + * Switch a btree handle to/from the current primary chunk of an LSM tree. + */ +static inline void +__wt_btree_lsm_switch_primary(WT_SESSION_IMPL *session, bool on) +{ + WT_BTREE *btree; + WT_CACHE *cache; + WT_PAGE *child, *root; + WT_PAGE_INDEX *pindex; + WT_REF *first; + size_t size; + + btree = S2BT(session); + cache = S2C(session)->cache; + root = btree->root.page; + + if (!F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) + F_SET(btree, WT_BTREE_LSM_PRIMARY | WT_BTREE_NO_EVICTION); + if (!on && F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) { + pindex = WT_INTL_INDEX_GET_SAFE(root); + if (!F_ISSET(btree, WT_BTREE_NO_EVICTION) || + pindex->entries != 1) + return; + first = pindex->index[0]; + + /* + * We're reaching down into the page without a hazard pointer, + * but that's OK because we know that no-eviction is set so the + * page can't disappear. + * + * While this tree was the primary, its dirty bytes were not + * included in the cache accounting. Fix that now before we + * open it up for eviction. + */ + child = first->page; + if (first->state == WT_REF_MEM && + child->type == WT_PAGE_ROW_LEAF && + __wt_page_is_modified(child)) { + size = child->modify->bytes_dirty; + (void)__wt_atomic_add64(&btree->bytes_dirty_leaf, size); + (void)__wt_atomic_add64(&cache->bytes_dirty_leaf, size); + } + + F_CLR(btree, WT_BTREE_LSM_PRIMARY | WT_BTREE_NO_EVICTION); + } +} + +/* * __wt_split_descent_race -- * Return if we raced with an internal page split when descending the tree. */ diff --git a/src/third_party/wiredtiger/src/include/cursor.i b/src/third_party/wiredtiger/src/include/cursor.i index e142441e0a6..c3fcef9a13d 100644 --- a/src/third_party/wiredtiger/src/include/cursor.i +++ b/src/third_party/wiredtiger/src/include/cursor.i @@ -267,7 +267,7 @@ __cursor_func_init(WT_CURSOR_BTREE *cbt, bool reenter) * to read. */ if (!F_ISSET(cbt, WT_CBT_NO_TXN)) - WT_RET(__wt_txn_cursor_op(session)); + __wt_txn_cursor_op(session); return (0); } diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 6234f2f6bc5..ef2e9efa9fd 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -226,8 +226,10 @@ extern int __wt_config_merge(WT_SESSION_IMPL *session, const char **cfg, const c extern int __wt_conn_config_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_conn_config_discard(WT_SESSION_IMPL *session); extern const WT_CONFIG_ENTRY *__wt_conn_config_match(const char *method); -extern int __wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_config_get(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, const char *key, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_config_get_string(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *config, const char *key, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_config_parser_open_arg(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, WT_CONFIG_PARSER **config_parserp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_upgrade(WT_SESSION_IMPL *session, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_wiredtiger_error(int error); extern int __wt_collator_config(WT_SESSION_IMPL *session, const char *uri, WT_CONFIG_ITEM *cname, WT_CONFIG_ITEM *metadata, WT_COLLATOR **collatorp, int *ownp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -723,7 +725,7 @@ extern int __wt_thread_group_create( WT_SESSION_IMPL *session, WT_THREAD_GROUP * extern int __wt_thread_group_destroy(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_group_start_one( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, bool wait) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_txn_release_snapshot(WT_SESSION_IMPL *session); -extern int __wt_txn_get_snapshot(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_txn_get_snapshot(WT_SESSION_IMPL *session); extern int __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_txn_release(WT_SESSION_IMPL *session); diff --git a/src/third_party/wiredtiger/src/include/flags.h b/src/third_party/wiredtiger/src/include/flags.h index b0d167525b2..d4c0b519cb1 100644 --- a/src/third_party/wiredtiger/src/include/flags.h +++ b/src/third_party/wiredtiger/src/include/flags.h @@ -52,24 +52,25 @@ #define WT_READ_WONT_NEED 0x00001000 #define WT_SESSION_CAN_WAIT 0x00000001 #define WT_SESSION_INTERNAL 0x00000002 -#define WT_SESSION_LOCKED_CHECKPOINT 0x00000004 -#define WT_SESSION_LOCKED_HANDLE_LIST 0x00000008 -#define WT_SESSION_LOCKED_METADATA 0x00000010 -#define WT_SESSION_LOCKED_PASS 0x00000020 -#define WT_SESSION_LOCKED_SCHEMA 0x00000040 -#define WT_SESSION_LOCKED_SLOT 0x00000080 -#define WT_SESSION_LOCKED_TABLE 0x00000100 -#define WT_SESSION_LOCKED_TURTLE 0x00000200 -#define WT_SESSION_LOCK_NO_WAIT 0x00000400 -#define WT_SESSION_LOGGING_INMEM 0x00000800 -#define WT_SESSION_LOOKASIDE_CURSOR 0x00001000 -#define WT_SESSION_NO_CACHE 0x00002000 -#define WT_SESSION_NO_DATA_HANDLES 0x00004000 -#define WT_SESSION_NO_EVICTION 0x00008000 -#define WT_SESSION_NO_LOGGING 0x00010000 -#define WT_SESSION_NO_SCHEMA_LOCK 0x00020000 -#define WT_SESSION_QUIET_CORRUPT_FILE 0x00040000 -#define WT_SESSION_SERVER_ASYNC 0x00080000 +#define WT_SESSION_IN_SPLIT 0x00000004 +#define WT_SESSION_LOCKED_CHECKPOINT 0x00000008 +#define WT_SESSION_LOCKED_HANDLE_LIST 0x00000010 +#define WT_SESSION_LOCKED_METADATA 0x00000020 +#define WT_SESSION_LOCKED_PASS 0x00000040 +#define WT_SESSION_LOCKED_SCHEMA 0x00000080 +#define WT_SESSION_LOCKED_SLOT 0x00000100 +#define WT_SESSION_LOCKED_TABLE 0x00000200 +#define WT_SESSION_LOCKED_TURTLE 0x00000400 +#define WT_SESSION_LOCK_NO_WAIT 0x00000800 +#define WT_SESSION_LOGGING_INMEM 0x00001000 +#define WT_SESSION_LOOKASIDE_CURSOR 0x00002000 +#define WT_SESSION_NO_CACHE 0x00004000 +#define WT_SESSION_NO_DATA_HANDLES 0x00008000 +#define WT_SESSION_NO_EVICTION 0x00010000 +#define WT_SESSION_NO_LOGGING 0x00020000 +#define WT_SESSION_NO_SCHEMA_LOCK 0x00040000 +#define WT_SESSION_QUIET_CORRUPT_FILE 0x00080000 +#define WT_SESSION_SERVER_ASYNC 0x00100000 #define WT_STAT_CLEAR 0x00000001 #define WT_STAT_JSON 0x00000002 #define WT_STAT_ON_CLOSE 0x00000004 diff --git a/src/third_party/wiredtiger/src/include/lsm.h b/src/third_party/wiredtiger/src/include/lsm.h index b433e4c3c44..fefed9daa81 100644 --- a/src/third_party/wiredtiger/src/include/lsm.h +++ b/src/third_party/wiredtiger/src/include/lsm.h @@ -249,17 +249,21 @@ struct __wt_lsm_tree { int64_t lsm_merge_throttle; /* - * The tree is open for business. This used to be a flag, but it is - * susceptible to races. + * Following fields used to be flags but are susceptible to races. + * Don't merge them with flags. */ - bool active; - -#define WT_LSM_TREE_AGGRESSIVE_TIMER 0x01 /* Timer for merge aggression */ -#define WT_LSM_TREE_COMPACTING 0x02 /* Tree being compacted */ -#define WT_LSM_TREE_MERGES 0x04 /* Tree should run merges */ -#define WT_LSM_TREE_NEED_SWITCH 0x08 /* New chunk needs creating */ -#define WT_LSM_TREE_OPEN 0x10 /* The tree is open */ -#define WT_LSM_TREE_THROTTLE 0x20 /* Throttle updates */ + bool active; /* The tree is open for business */ + bool aggressive_timer_enabled; /* Timer for merge aggression enabled */ + bool need_switch; /* New chunk needs creating */ + + /* + * flags here are not protected for concurrent access, don't put + * anything here that is susceptible to races. + */ +#define WT_LSM_TREE_COMPACTING 0x01 /* Tree being compacted */ +#define WT_LSM_TREE_MERGES 0x02 /* Tree should run merges */ +#define WT_LSM_TREE_OPEN 0x04 /* The tree is open */ +#define WT_LSM_TREE_THROTTLE 0x08 /* Throttle updates */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h index 6d4a167a8e5..74df3c57ce4 100644 --- a/src/third_party/wiredtiger/src/include/meta.h +++ b/src/third_party/wiredtiger/src/include/meta.h @@ -32,8 +32,7 @@ * Optimize comparisons against the metafile URI, flag handles that reference * the metadata file. */ -#define WT_IS_METADATA(session, dh) \ - F_ISSET((dh), WT_DHANDLE_IS_METADATA) +#define WT_IS_METADATA(dh) F_ISSET((dh), WT_DHANDLE_IS_METADATA) #define WT_METAFILE_ID 0 /* Metadata file ID */ #define WT_METADATA_VERSION "WiredTiger version" /* Version keys */ diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index 8128e8e4cc2..344275e23d0 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -49,8 +49,11 @@ WT_ASSERT((s), (s)->txn.forced_iso > 0); \ (s)->txn.forced_iso--; \ WT_ASSERT((s), txn_state->id == saved_state.id && \ + (txn_state->metadata_pinned == saved_state.metadata_pinned ||\ + saved_state.metadata_pinned == WT_TXN_NONE) && \ (txn_state->pinned_id == saved_state.pinned_id || \ saved_state.pinned_id == WT_TXN_NONE)); \ + txn_state->metadata_pinned = saved_state.metadata_pinned; \ txn_state->pinned_id = saved_state.pinned_id; \ } while (0) @@ -67,6 +70,7 @@ struct __wt_named_snapshot { struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_txn_state { volatile uint64_t id; volatile uint64_t pinned_id; + volatile uint64_t metadata_pinned; }; struct __wt_txn_global { @@ -94,12 +98,18 @@ struct __wt_txn_global { * for a long time so we keep them out of regular visibility checks. * Eviction and checkpoint operations know when they need to be aware * of checkpoint transactions. + * + * We rely on the fact that (a) the only table a checkpoint updates is + * the metadata; and (b) once checkpoint has finished reading a table, + * it won't revisit it. */ volatile uint32_t checkpoint_id; /* Checkpoint's session ID */ - volatile uint64_t checkpoint_gen; - volatile uint64_t checkpoint_pinned; + volatile uint64_t checkpoint_gen; /* Checkpoint generation */ + volatile uint64_t checkpoint_pinned; /* Oldest ID for checkpoint */ volatile uint64_t checkpoint_txnid; /* Checkpoint's txn ID */ + volatile uint64_t metadata_pinned; /* Oldest ID for metadata */ + /* Named snapshot state. */ WT_RWLOCK *nsnap_rwlock; volatile uint64_t nsnap_oldest_id; diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index cf7e2eafc65..3e8dd45003e 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -112,6 +112,13 @@ __wt_txn_oldest_id(WT_SESSION_IMPL *session) btree = S2BT_SAFE(session); /* + * The metadata is tracked specially because of optimizations for + * checkpoints. + */ + if (session->dhandle != NULL && WT_IS_METADATA(session->dhandle)) + return (txn_global->metadata_pinned); + + /* * Take a local copy of these IDs in case they are updated while we are * checking visibility. The read of the transaction ID pinned by a * checkpoint needs to be carefully ordered: if a checkpoint is @@ -262,7 +269,7 @@ __wt_txn_begin(WT_SESSION_IMPL *session, const char *cfg[]) * eviction, it's better to do it beforehand. */ WT_RET(__wt_cache_eviction_check(session, false, NULL)); - WT_RET(__wt_txn_get_snapshot(session)); + __wt_txn_get_snapshot(session); } F_SET(txn, WT_TXN_RUNNING); @@ -451,7 +458,7 @@ __wt_txn_read_last(WT_SESSION_IMPL *session) * __wt_txn_cursor_op -- * Called for each cursor operation. */ -static inline int +static inline void __wt_txn_cursor_op(WT_SESSION_IMPL *session) { WT_TXN *txn; @@ -482,10 +489,10 @@ __wt_txn_cursor_op(WT_SESSION_IMPL *session) if (txn->isolation == WT_ISO_READ_UNCOMMITTED) { if (txn_state->pinned_id == WT_TXN_NONE) txn_state->pinned_id = txn_global->last_running; + if (txn_state->metadata_pinned == WT_TXN_NONE) + txn_state->metadata_pinned = txn_state->pinned_id; } else if (!F_ISSET(txn, WT_TXN_HAS_SNAPSHOT)) - WT_RET(__wt_txn_get_snapshot(session)); - - return (0); + __wt_txn_get_snapshot(session); } /* diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 665e8eaf4b0..8da46582924 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -827,6 +827,11 @@ struct __wt_session { * * @param session the session handle * @configstart{WT_SESSION.reconfigure, see dist/api_data.py} + * @config{ignore_cache_size, when set\, operations performed by this + * session ignore the cache size and are not blocked when the cache is + * full. Note that use of this option for operations that create cache + * pressure can starve ordinary sessions that obey the cache size., a + * boolean flag; default \c false.} * @config{isolation, the default isolation level for operations in this * session., a string\, chosen from the following options: \c * "read-uncommitted"\, \c "read-committed"\, \c "snapshot"; default \c @@ -2005,6 +2010,11 @@ struct __wt_connection { * connection's error handler is used. See @ref error_handling_event * for more information. * @configstart{WT_CONNECTION.open_session, see dist/api_data.py} + * @config{ignore_cache_size, when set\, operations performed by this + * session ignore the cache size and are not blocked when the cache is + * full. Note that use of this option for operations that create cache + * pressure can starve ordinary sessions that obey the cache size., a + * boolean flag; default \c false.} * @config{isolation, the default isolation level for operations in this * session., a string\, chosen from the following options: \c * "read-uncommitted"\, \c "read-committed"\, \c "snapshot"; default \c @@ -2204,6 +2214,11 @@ struct __wt_connection { * I/O. The default value of -1 indicates a platform-specific alignment value * should be used (4KB on Linux systems when direct I/O is configured\, zero * elsewhere)., an integer between -1 and 1MB; default \c -1.} + * @config{builtin_extension_config, A structure where the keys are the names of + * builtin extensions and the values are passed to WT_CONNECTION::load_extension + * as the \c config parameter (for example\, + * <code>builtin_extension_config={zlib={compression_level=3}}</code>)., a + * string; default empty.} * @config{cache_overhead, assume the heap allocator overhead is the specified * percentage\, and adjust the cache usage by that amount (for example\, if * there is 10GB of data in cache\, a percentage of 10 means WiredTiger treats diff --git a/src/third_party/wiredtiger/src/include/wiredtiger_ext.h b/src/third_party/wiredtiger/src/include/wiredtiger_ext.h index 3d65cd1fc24..236d4e07e67 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger_ext.h +++ b/src/third_party/wiredtiger/src/include/wiredtiger_ext.h @@ -204,25 +204,47 @@ struct __wt_extension_api { WT_COLLATOR *collator, WT_ITEM *first, WT_ITEM *second, int *cmp); /*! - * @copydoc wiredtiger_config_parser_open + * Return the value of a configuration key. + * + * @param wt_api the extension handle + * @param session the session handle (or NULL if none available) + * @param config the configuration information passed to an application + * @param key configuration key string + * @param value the returned value + * @errors + * + * @snippet ex_data_source.c WT_EXTENSION config_get */ - int (*config_parser_open)(WT_EXTENSION_API *wt_api, WT_SESSION *session, - const char *config, size_t len, WT_CONFIG_PARSER **config_parserp); + int (*config_get)(WT_EXTENSION_API *wt_api, WT_SESSION *session, + WT_CONFIG_ARG *config, const char *key, WT_CONFIG_ITEM *value); /*! - * Return the value of a configuration string. + * Return the value of a configuration key from a string. * * @param wt_api the extension handle * @param session the session handle (or NULL if none available) + * @param config the configuration string * @param key configuration key string - * @param config the configuration information passed to an application * @param value the returned value * @errors * * @snippet ex_data_source.c WT_EXTENSION config_get */ - int (*config_get)(WT_EXTENSION_API *wt_api, WT_SESSION *session, - WT_CONFIG_ARG *config, const char *key, WT_CONFIG_ITEM *value); + int (*config_get_string)(WT_EXTENSION_API *wt_api, WT_SESSION *session, + const char *config, const char *key, WT_CONFIG_ITEM *value); + + /*! + * @copydoc wiredtiger_config_parser_open + */ + int (*config_parser_open)(WT_EXTENSION_API *wt_api, WT_SESSION *session, + const char *config, size_t len, WT_CONFIG_PARSER **config_parserp); + + /*! + * @copydoc wiredtiger_config_parser_open + */ + int (*config_parser_open_arg)(WT_EXTENSION_API *wt_api, + WT_SESSION *session, WT_CONFIG_ARG *config, + WT_CONFIG_PARSER **config_parserp); /*! * Insert a row into the metadata if it does not already exist. diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index 00e4ea5f441..8b6c8b2c490 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -21,6 +21,60 @@ static int __log_write_internal( #define WT_LOG_OPEN_VERIFY 0x02 /* + * __log_wait_for_earlier_slot -- + * Wait for write_lsn to catch up to this slot. + */ +static void +__log_wait_for_earlier_slot(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) +{ + WT_CONNECTION_IMPL *conn; + WT_LOG *log; + int yield_count; + + conn = S2C(session); + log = conn->log; + yield_count = 0; + + while (__wt_log_cmp(&log->write_lsn, &slot->slot_release_lsn) != 0) { + /* + * If we're on a locked path and the write LSN is not advancing, + * unlock in case an earlier thread is trying to switch its + * slot and complete its operation. + */ + if (F_ISSET(session, WT_SESSION_LOCKED_SLOT)) + __wt_spin_unlock(session, &log->log_slot_lock); + __wt_cond_auto_signal(session, conn->log_wrlsn_cond); + if (++yield_count < WT_THOUSAND) + __wt_yield(); + else + __wt_cond_wait(session, log->log_write_cond, 200); + if (F_ISSET(session, WT_SESSION_LOCKED_SLOT)) + __wt_spin_lock(session, &log->log_slot_lock); + } +} + +/* + * __log_fs_write -- + * Wrapper when writing to a log file. If we're writing to a new log + * file for the first time wait for writes to the previous log file. + */ +static int +__log_fs_write(WT_SESSION_IMPL *session, + WT_LOGSLOT *slot, wt_off_t offset, size_t len, const void *buf) +{ + /* + * If we're writing into a new log file, we have to wait for all + * writes to the previous log file to complete otherwise there could + * be a hole at the end of the previous log file that we cannot detect. + */ + if (slot->slot_release_lsn.l.file < slot->slot_start_lsn.l.file) { + __log_wait_for_earlier_slot(session, slot); + WT_RET(__wt_log_force_sync(session, &slot->slot_release_lsn)); + } + return (__wt_write(session, slot->slot_fh, offset, len, buf)); +} + +/* * __wt_log_ckpt -- * Record the given LSN as the checkpoint LSN and signal the archive * thread as needed. @@ -371,8 +425,6 @@ __wt_log_extract_lognum( { const char *p; - WT_UNUSED(session); - if (id == NULL || name == NULL) return (WT_ERROR); if ((p = strrchr(name, '.')) == NULL || @@ -576,7 +628,7 @@ __log_fill(WT_SESSION_IMPL *session, /* * If this is a force or unbuffered write, write it now. */ - WT_ERR(__wt_write(session, myslot->slot->slot_fh, + WT_ERR(__log_fs_write(session, myslot->slot, myslot->offset + myslot->slot->slot_start_offset, record->size, record->mem)); @@ -1352,13 +1404,11 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) WT_LSN sync_lsn; int64_t release_buffered, release_bytes; uint64_t fsync_duration_usecs; - int yield_count; bool locked; conn = S2C(session); log = conn->log; locked = false; - yield_count = 0; if (freep != NULL) *freep = 1; release_buffered = WT_LOG_SLOT_RELEASED_BUFFERED(slot->slot_state); @@ -1379,8 +1429,7 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) /* Write the buffered records */ if (release_buffered != 0) - WT_ERR(__wt_write(session, - slot->slot_fh, slot->slot_start_offset, + WT_ERR(__log_fs_write(session, slot, slot->slot_start_offset, (size_t)release_buffered, slot->slot_buf.mem)); /* @@ -1411,22 +1460,7 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) * be holes in the log file. */ WT_STAT_CONN_INCR(session, log_release_write_lsn); - while (__wt_log_cmp(&log->write_lsn, &slot->slot_release_lsn) != 0) { - /* - * If we're on a locked path and the write LSN is not advancing, - * unlock in case an earlier thread is trying to switch its - * slot and complete its operation. - */ - if (F_ISSET(session, WT_SESSION_LOCKED_SLOT)) - __wt_spin_unlock(session, &log->log_slot_lock); - __wt_cond_auto_signal(session, conn->log_wrlsn_cond); - if (++yield_count < WT_THOUSAND) - __wt_yield(); - else - __wt_cond_wait(session, log->log_write_cond, 200); - if (F_ISSET(session, WT_SESSION_LOCKED_SLOT)) - __wt_spin_lock(session, &log->log_slot_lock); - } + __log_wait_for_earlier_slot(session, slot); log->write_start_lsn = slot->slot_start_lsn; log->write_lsn = slot->slot_end_lsn; diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index c20673563d9..9c124885086 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -34,7 +34,7 @@ __wt_clsm_request_switch(WT_CURSOR_LSM *clsm) lsm_tree = clsm->lsm_tree; session = (WT_SESSION_IMPL *)clsm->iface.session; - if (!F_ISSET(lsm_tree, WT_LSM_TREE_NEED_SWITCH)) { + if (!lsm_tree->need_switch) { /* * Check that we are up-to-date: don't set the switch if the * tree has changed since we last opened cursors: that can lead @@ -44,8 +44,8 @@ __wt_clsm_request_switch(WT_CURSOR_LSM *clsm) __wt_lsm_tree_readlock(session, lsm_tree); if (lsm_tree->nchunks == 0 || (clsm->dsk_gen == lsm_tree->dsk_gen && - !F_ISSET(lsm_tree, WT_LSM_TREE_NEED_SWITCH))) { - F_SET(lsm_tree, WT_LSM_TREE_NEED_SWITCH); + !lsm_tree->need_switch)) { + lsm_tree->need_switch = true; ret = __wt_lsm_manager_push_entry( session, WT_LSM_WORK_SWITCH, 0, lsm_tree); } @@ -129,7 +129,7 @@ __clsm_enter_update(WT_CURSOR_LSM *clsm) * chunk grows twice as large as the configured size, block until it * can be switched. */ - hard_limit = F_ISSET(lsm_tree, WT_LSM_TREE_NEED_SWITCH); + hard_limit = lsm_tree->need_switch; if (have_primary) { WT_ENTER_PAGE_INDEX(session); @@ -215,7 +215,7 @@ __clsm_enter(WT_CURSOR_LSM *clsm, bool reset, bool update) goto open; if (txn->isolation == WT_ISO_SNAPSHOT) - WT_RET(__wt_txn_cursor_op(session)); + __wt_txn_cursor_op(session); /* * Figure out how many updates are required for @@ -700,7 +700,7 @@ retry: if (F_ISSET(clsm, WT_CLSM_MERGE)) { if (btree->bulk_load_ok) { btree->bulk_load_ok = false; WT_WITH_BTREE(session, btree, - __wt_btree_evictable(session, false)); + __wt_btree_lsm_switch_primary(session, true)); } } diff --git a/src/third_party/wiredtiger/src/lsm/lsm_merge.c b/src/third_party/wiredtiger/src/lsm/lsm_merge.c index 493855d489a..dc47d0118a2 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_merge.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_merge.c @@ -46,7 +46,7 @@ __wt_lsm_merge_update_tree(WT_SESSION_IMPL *session, static void __lsm_merge_aggressive_clear(WT_LSM_TREE *lsm_tree) { - F_CLR(lsm_tree, WT_LSM_TREE_AGGRESSIVE_TIMER); + lsm_tree->aggressive_timer_enabled = false; lsm_tree->merge_aggressiveness = 0; } @@ -85,12 +85,12 @@ __lsm_merge_aggressive_update(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) } /* - * Start the timer if it isn't running. Use a flag to define whether + * Start the timer if it isn't running. Use a bool to define whether * the timer is running - since clearing and checking a special * timer value isn't simple. */ - if (!F_ISSET(lsm_tree, WT_LSM_TREE_AGGRESSIVE_TIMER)) { - F_SET(lsm_tree, WT_LSM_TREE_AGGRESSIVE_TIMER); + if (!lsm_tree->aggressive_timer_enabled) { + lsm_tree->aggressive_timer_enabled = true; __wt_epoch(session, &lsm_tree->merge_aggressive_ts); } diff --git a/src/third_party/wiredtiger/src/lsm/lsm_tree.c b/src/third_party/wiredtiger/src/lsm/lsm_tree.c index 0054dcd1583..714007cda98 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_tree.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_tree.c @@ -736,7 +736,7 @@ __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) if (!first_switch && (last_chunk = lsm_tree->chunk[nchunks - 1]) != NULL && !F_ISSET(last_chunk, WT_LSM_CHUNK_ONDISK) && - !F_ISSET(lsm_tree, WT_LSM_TREE_NEED_SWITCH)) + !lsm_tree->need_switch) goto err; /* Update the throttle time. */ @@ -759,7 +759,7 @@ __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_ERR(__wt_lsm_tree_setup_chunk(session, lsm_tree, chunk)); WT_ERR(__wt_lsm_meta_write(session, lsm_tree)); - F_CLR(lsm_tree, WT_LSM_TREE_NEED_SWITCH); + lsm_tree->need_switch = false; ++lsm_tree->dsk_gen; lsm_tree->modified = true; diff --git a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c index 917104031fc..66a57f15875 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c @@ -171,12 +171,12 @@ __wt_lsm_work_switch( *ran = false; *entryp = NULL; - if (F_ISSET(entry->lsm_tree, WT_LSM_TREE_NEED_SWITCH)) { + if (entry->lsm_tree->need_switch) { WT_WITH_SCHEMA_LOCK(session, ret, ret = __wt_lsm_tree_switch(session, entry->lsm_tree)); /* Failing to complete the switch is fine */ if (ret == EBUSY) { - if (F_ISSET(entry->lsm_tree, WT_LSM_TREE_NEED_SWITCH)) + if (entry->lsm_tree->need_switch) WT_ERR(__wt_lsm_manager_push_entry(session, WT_LSM_WORK_SWITCH, 0, entry->lsm_tree)); ret = 0; @@ -383,7 +383,7 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, * forced eviction. */ WT_ERR(__wt_session_get_btree(session, chunk->uri, NULL, NULL, 0)); - __wt_btree_evictable(session, true); + __wt_btree_lsm_switch_primary(session, false); WT_ERR(__wt_session_release_btree(session)); /* Make sure we aren't pinning a transaction ID. */ diff --git a/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c b/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c index 70a82007300..1670e97be45 100644 --- a/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c +++ b/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c @@ -515,8 +515,6 @@ __im_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session) WT_FILE_SYSTEM_INMEM *im_fs; WT_SESSION_IMPL *session; - WT_UNUSED(file_system); - session = (WT_SESSION_IMPL *)wt_session; im_fs = (WT_FILE_SYSTEM_INMEM *)file_system; diff --git a/src/third_party/wiredtiger/src/os_posix/os_dir.c b/src/third_party/wiredtiger/src/os_posix/os_dir.c index 768a1324cd8..627278540d1 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_dir.c +++ b/src/third_party/wiredtiger/src/os_posix/os_dir.c @@ -28,8 +28,6 @@ __wt_posix_directory_list(WT_FILE_SYSTEM *file_system, int tret; char **entries; - WT_UNUSED(file_system); - session = (WT_SESSION_IMPL *)wt_session; *dirlistp = NULL; diff --git a/src/third_party/wiredtiger/src/os_posix/os_fs.c b/src/third_party/wiredtiger/src/os_posix/os_fs.c index c6272b5da52..5f06892ce6e 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_fs.c +++ b/src/third_party/wiredtiger/src/os_posix/os_fs.c @@ -746,8 +746,6 @@ __posix_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session) { WT_SESSION_IMPL *session; - WT_UNUSED(file_system); - session = (WT_SESSION_IMPL *)wt_session; __wt_free(session, file_system); diff --git a/src/third_party/wiredtiger/src/os_win/os_dir.c b/src/third_party/wiredtiger/src/os_win/os_dir.c index f024d131387..47d4f95b793 100644 --- a/src/third_party/wiredtiger/src/os_win/os_dir.c +++ b/src/third_party/wiredtiger/src/os_win/os_dir.c @@ -30,8 +30,6 @@ __wt_win_directory_list(WT_FILE_SYSTEM *file_system, uint32_t count; char *dir_copy, **entries; - WT_UNUSED(file_system); - session = (WT_SESSION_IMPL *)wt_session; *dirlistp = NULL; diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 3afea383b08..00c2852649f 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -249,18 +249,36 @@ __session_reconfigure(WT_SESSION *wt_session, const char *config) session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, reconfigure, config, cfg); + /* + * Note that this method only checks keys that are passed in by the + * application: we don't want to reset other session settings to their + * default values. + */ + WT_UNUSED(cfg); + if (F_ISSET(&session->txn, WT_TXN_RUNNING)) WT_ERR_MSG(session, EINVAL, "transaction in progress"); - WT_TRET(__wt_session_reset_cursors(session, false)); + WT_ERR(__wt_session_reset_cursors(session, false)); - WT_ERR(__wt_config_gets_def(session, cfg, "isolation", 0, &cval)); - if (cval.len != 0) + ret = __wt_config_getones(session, config, "isolation", &cval); + if (ret == 0 && cval.len != 0) { session->isolation = session->txn.isolation = WT_STRING_MATCH("snapshot", cval.str, cval.len) ? WT_ISO_SNAPSHOT : WT_STRING_MATCH("read-uncommitted", cval.str, cval.len) ? WT_ISO_READ_UNCOMMITTED : WT_ISO_READ_COMMITTED; + } + WT_ERR_NOTFOUND_OK(ret); + + ret = __wt_config_getones(session, config, "ignore_cache_size", &cval); + if (ret == 0) { + if (cval.val) + F_SET(session, WT_SESSION_NO_EVICTION); + else + F_CLR(session, WT_SESSION_NO_EVICTION); + } + WT_ERR_NOTFOUND_OK(ret); err: API_END_RET_NOTFOUND_MAP(session, ret); } diff --git a/src/third_party/wiredtiger/src/session/session_dhandle.c b/src/third_party/wiredtiger/src/session/session_dhandle.c index 725854c6001..94326aebe46 100644 --- a/src/third_party/wiredtiger/src/session/session_dhandle.c +++ b/src/third_party/wiredtiger/src/session/session_dhandle.c @@ -68,7 +68,7 @@ __session_find_dhandle(WT_SESSION_IMPL *session, retry: TAILQ_FOREACH(dhandle_cache, &session->dhhash[bucket], hashq) { dhandle = dhandle_cache->dhandle; if (WT_DHANDLE_INACTIVE(dhandle) && - !WT_IS_METADATA(session, dhandle)) { + !WT_IS_METADATA(dhandle)) { __session_discard_dhandle(session, dhandle_cache); /* We deleted our entry, retry from the start. */ goto retry; @@ -401,7 +401,7 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session) difftime(now, dhandle->timeofdeath) > conn->sweep_idle_time))) { WT_STAT_CONN_INCR(session, dh_session_handles); - WT_ASSERT(session, !WT_IS_METADATA(session, dhandle)); + WT_ASSERT(session, !WT_IS_METADATA(dhandle)); __session_discard_dhandle(session, dhandle_cache); } dhandle_cache = dhandle_cache_next; diff --git a/src/third_party/wiredtiger/src/support/hazard.c b/src/third_party/wiredtiger/src/support/hazard.c index b1978dbe97a..27508bbdece 100644 --- a/src/third_party/wiredtiger/src/support/hazard.c +++ b/src/third_party/wiredtiger/src/support/hazard.c @@ -26,6 +26,7 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp WT_BTREE *btree; WT_CONNECTION_IMPL *conn; WT_HAZARD *hp; + WT_PAGE *page; int restarts = 0; btree = S2BT(session); @@ -37,6 +38,16 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp return (0); /* + * If there isn't a valid page pointer, we're done. This read can race + * with eviction and splits, we re-check it after a barrier to make + * sure we have a valid pointer. + */ + if (ref->state != WT_REF_MEM || (page = ref->page) == NULL) { + *busyp = true; + return (0); + } + + /* * Do the dance: * * The memory location which makes a page "real" is the WT_REF's state @@ -82,7 +93,7 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp if (hp->page != NULL) continue; - hp->page = ref->page; + hp->page = page; #ifdef HAVE_DIAGNOSTIC hp->file = file; hp->line = line; @@ -101,9 +112,12 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp * found this page using the tree's key space, whatever page we * find here is the page for us to use.) */ - if (ref->page == hp->page && ref->state == WT_REF_MEM) { - ++session->nhazard; - return (0); + if (ref->state == WT_REF_MEM) { + WT_READ_BARRIER(); + if (ref->page == page) { + ++session->nhazard; + return (0); + } } /* diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 1c47f94bb60..a70551cdeb2 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -100,7 +100,7 @@ __wt_txn_release_snapshot(WT_SESSION_IMPL *session) session->txn.isolation == WT_ISO_READ_UNCOMMITTED || !__wt_txn_visible_all(session, txn_state->pinned_id)); - txn_state->pinned_id = WT_TXN_NONE; + txn_state->metadata_pinned = txn_state->pinned_id = WT_TXN_NONE; F_CLR(txn, WT_TXN_HAS_SNAPSHOT); } @@ -108,7 +108,7 @@ __wt_txn_release_snapshot(WT_SESSION_IMPL *session) * __wt_txn_get_snapshot -- * Allocate a snapshot. */ -int +void __wt_txn_get_snapshot(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; @@ -137,8 +137,10 @@ __wt_txn_get_snapshot(WT_SESSION_IMPL *session) * metadata. We don't have to keep the checkpoint's changes pinned so * don't including it in the published pinned ID. */ - if ((id = txn_global->checkpoint_txnid) != WT_TXN_NONE) + if ((id = txn_global->checkpoint_txnid) != WT_TXN_NONE) { txn->snapshot[n++] = id; + txn_state->metadata_pinned = id; + } /* For pure read-only workloads, avoid scanning. */ if (prev_oldest_id == current_id) { @@ -180,7 +182,6 @@ __wt_txn_get_snapshot(WT_SESSION_IMPL *session) done: __wt_readunlock(session, txn_global->scan_rwlock); __txn_sort_snapshot(session, n, current_id); - return (0); } /* @@ -189,14 +190,14 @@ done: __wt_readunlock(session, txn_global->scan_rwlock); */ static void __txn_oldest_scan(WT_SESSION_IMPL *session, - uint64_t *oldest_idp, uint64_t *last_runningp, + uint64_t *oldest_idp, uint64_t *last_runningp, uint64_t *metadata_pinnedp, WT_SESSION_IMPL **oldest_sessionp) { WT_CONNECTION_IMPL *conn; WT_SESSION_IMPL *oldest_session; WT_TXN_GLOBAL *txn_global; WT_TXN_STATE *s; - uint64_t id, last_running, oldest_id, prev_oldest_id; + uint64_t id, last_running, metadata_pinned, oldest_id, prev_oldest_id; uint32_t i, session_cnt; conn = S2C(session); @@ -205,24 +206,24 @@ __txn_oldest_scan(WT_SESSION_IMPL *session, /* The oldest ID cannot change while we are holding the scan lock. */ prev_oldest_id = txn_global->oldest_id; - oldest_id = last_running = txn_global->current; + last_running = oldest_id = txn_global->current; + if ((metadata_pinned = txn_global->checkpoint_txnid) == WT_TXN_NONE) + metadata_pinned = oldest_id; /* Walk the array of concurrent transactions. */ WT_ORDERED_READ(session_cnt, conn->session_cnt); for (i = 0, s = txn_global->states; i < session_cnt; i++, s++) { - /* - * Update the oldest ID. - * - * Ignore: IDs older than the oldest ID we saw. This can happen - * if we race with a thread that is allocating an ID -- the ID - * will not be used because the thread will keep spinning until - * it gets a valid one. - */ + /* Update the last running transaction ID. */ if ((id = s->id) != WT_TXN_NONE && WT_TXNID_LE(prev_oldest_id, id) && WT_TXNID_LT(id, last_running)) last_running = id; + /* Update the metadata pinned ID. */ + if ((id = s->metadata_pinned) != WT_TXN_NONE && + WT_TXNID_LT(id, metadata_pinned)) + metadata_pinned = id; + /* * !!! * Note: Don't ignore pinned ID values older than the previous @@ -246,9 +247,14 @@ __txn_oldest_scan(WT_SESSION_IMPL *session, WT_TXNID_LT(id, oldest_id)) oldest_id = id; + /* The metadata pinned ID can't move past the oldest ID. */ + if (WT_TXNID_LT(oldest_id, metadata_pinned)) + metadata_pinned = oldest_id; + + *last_runningp = last_running; + *metadata_pinnedp = metadata_pinned; *oldest_idp = oldest_id; *oldest_sessionp = oldest_session; - *last_runningp = last_running; } /* @@ -262,8 +268,8 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) WT_DECL_RET; WT_SESSION_IMPL *oldest_session; WT_TXN_GLOBAL *txn_global; - uint64_t current_id, last_running, oldest_id; - uint64_t prev_last_running, prev_oldest_id; + uint64_t current_id, last_running, metadata_pinned, oldest_id; + uint64_t prev_last_running, prev_metadata_pinned, prev_oldest_id; bool strict, wait; conn = S2C(session); @@ -271,15 +277,17 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) strict = LF_ISSET(WT_TXN_OLDEST_STRICT); wait = LF_ISSET(WT_TXN_OLDEST_WAIT); - current_id = last_running = txn_global->current; + current_id = last_running = metadata_pinned = txn_global->current; prev_last_running = txn_global->last_running; + prev_metadata_pinned = txn_global->metadata_pinned; prev_oldest_id = txn_global->oldest_id; /* * For pure read-only workloads, or if the update isn't forced and the * oldest ID isn't too far behind, avoid scanning. */ - if (prev_oldest_id == current_id || + if ((prev_oldest_id == current_id && + prev_metadata_pinned == current_id) || (!strict && WT_TXNID_LT(current_id, prev_oldest_id + 100))) return (0); @@ -289,7 +297,8 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) else if ((ret = __wt_try_readlock(session, txn_global->scan_rwlock)) != 0) return (ret == EBUSY ? 0 : ret); - __txn_oldest_scan(session, &oldest_id, &last_running, &oldest_session); + __txn_oldest_scan(session, + &oldest_id, &last_running, &metadata_pinned, &oldest_session); __wt_readunlock(session, txn_global->scan_rwlock); /* @@ -299,7 +308,8 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) if ((oldest_id == prev_oldest_id || (!strict && WT_TXNID_LT(oldest_id, prev_oldest_id + 100))) && ((last_running == prev_last_running) || - (!strict && WT_TXNID_LT(last_running, prev_last_running + 100)))) + (!strict && WT_TXNID_LT(last_running, prev_last_running + 100))) && + metadata_pinned == prev_metadata_pinned) return (0); /* It looks like an update is necessary, wait for exclusive access. */ @@ -314,7 +324,8 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) * scanning. */ if (WT_TXNID_LE(oldest_id, txn_global->oldest_id) && - WT_TXNID_LE(last_running, txn_global->last_running)) + WT_TXNID_LE(last_running, txn_global->last_running) && + WT_TXNID_LE(metadata_pinned, txn_global->metadata_pinned)) goto done; /* @@ -323,7 +334,8 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) * sure that there isn't a thread that has got a snapshot locally but * not yet published its snap_min. */ - __txn_oldest_scan(session, &oldest_id, &last_running, &oldest_session); + __txn_oldest_scan(session, + &oldest_id, &last_running, &metadata_pinned, &oldest_session); #ifdef HAVE_DIAGNOSTIC { @@ -339,7 +351,9 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) id == WT_TXN_NONE || !WT_TXNID_LT(id, oldest_id)); } #endif - /* Update the oldest ID. */ + /* Update the public IDs. */ + if (WT_TXNID_LT(txn_global->metadata_pinned, metadata_pinned)) + txn_global->metadata_pinned = metadata_pinned; if (WT_TXNID_LT(txn_global->oldest_id, oldest_id)) txn_global->oldest_id = oldest_id; if (WT_TXNID_LT(txn_global->last_running, last_running)) { @@ -750,7 +764,7 @@ __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]) txn_global = &conn->txn_global; txn_global->current = txn_global->last_running = - txn_global->oldest_id = WT_TXN_FIRST; + txn_global->metadata_pinned = txn_global->oldest_id = WT_TXN_FIRST; WT_RET(__wt_spin_init(session, &txn_global->id_lock, "transaction id lock")); @@ -766,7 +780,7 @@ __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]) WT_CACHE_LINE_ALIGNMENT_VERIFY(session, txn_global->states); for (i = 0, s = txn_global->states; i < conn->session_size; i++, s++) - s->id = s->pinned_id = WT_TXN_NONE; + s->id = s->metadata_pinned = s->pinned_id = WT_TXN_NONE; return (0); } diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 1efd0200a3d..802ccd84915 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -242,8 +242,6 @@ __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) WT_DECL_RET; const char *name; - WT_UNUSED(cfg); - /* Should not be called with anything other than a file object. */ WT_ASSERT(session, session->dhandle->checkpoint == NULL); WT_ASSERT(session, WT_PREFIX_MATCH(session->dhandle->name, "file:")); @@ -301,7 +299,7 @@ __checkpoint_update_generation(WT_SESSION_IMPL *session) WT_BTREE *btree; btree = S2BT(session); - if (!WT_IS_METADATA(session, session->dhandle)) + if (!WT_IS_METADATA(session->dhandle)) WT_PUBLISH(btree->include_checkpoint_txn, false); WT_PUBLISH(btree->checkpoint_gen, @@ -703,7 +701,8 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) * can safely ignore the checkpoint ID (see the visible all check for * details). */ - txn_state->id = txn_state->pinned_id = WT_TXN_NONE; + txn_state->id = txn_state->pinned_id = + txn_state->metadata_pinned = WT_TXN_NONE; __wt_writeunlock(session, txn_global->scan_rwlock); /* @@ -1057,7 +1056,7 @@ __checkpoint_lock_tree(WT_SESSION_IMPL *session, * - On connection close when we know there can't be any races. */ WT_ASSERT(session, !need_tracking || - WT_IS_METADATA(session, dhandle) || WT_META_TRACKING(session)); + WT_IS_METADATA(dhandle) || WT_META_TRACKING(session)); /* Get the list of checkpoints for this file. */ WT_RET(__wt_meta_ckptlist_get(session, dhandle->name, &ckptbase)); @@ -1421,7 +1420,7 @@ fake: /* * sync the file here or we could roll forward the metadata in * recovery and open a checkpoint that isn't yet durable. */ - if (WT_IS_METADATA(session, dhandle) || + if (WT_IS_METADATA(dhandle) || !F_ISSET(&session->txn, WT_TXN_RUNNING)) WT_ERR(__wt_checkpoint_sync(session, NULL)); @@ -1532,7 +1531,7 @@ __wt_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_ASSERT(session, session->dhandle->checkpoint == NULL); /* We must hold the metadata lock if checkpointing the metadata. */ - WT_ASSERT(session, !WT_IS_METADATA(session, session->dhandle) || + WT_ASSERT(session, !WT_IS_METADATA(session->dhandle) || F_ISSET(session, WT_SESSION_LOCKED_METADATA)); WT_SAVE_DHANDLE(session, diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index 65811aa3bf4..a6390dcbd06 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -248,8 +248,6 @@ static int __txn_commit_apply( WT_RECOVERY *r, WT_LSN *lsnp, const uint8_t **pp, const uint8_t *end) { - WT_UNUSED(lsnp); - /* The logging subsystem zero-pads records. */ while (*pp < end && **pp) WT_RET(__txn_op_apply(r, lsnp, pp, end)); diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 839ff5058de..cf922b5db04 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -301,7 +301,7 @@ config_compression(const char *conf_name) break; #endif #ifdef HAVE_BUILTIN_EXTENSION_ZSTD - case 15: case 16 case 17: /* 15% zstd */ + case 15: case 16: case 17: /* 15% zstd */ cstr = "zstd"; break; #endif diff --git a/src/third_party/wiredtiger/test/recovery/random-abort.c b/src/third_party/wiredtiger/test/recovery/random-abort.c index 03e67e2f723..c407361c7eb 100644 --- a/src/third_party/wiredtiger/test/recovery/random-abort.c +++ b/src/third_party/wiredtiger/test/recovery/random-abort.c @@ -34,6 +34,7 @@ static char home[512]; /* Program working dir */ static const char *progname; /* Program name */ static const char * const uri = "table:main"; +static bool inmem; #define MAX_TH 12 #define MIN_TH 5 @@ -41,7 +42,9 @@ static const char * const uri = "table:main"; #define MIN_TIME 10 #define RECORDS_FILE "records-%" PRIu32 -#define ENV_CONFIG \ +#define ENV_CONFIG_DEF \ + "create,log=(file_max=10M,archive=false,enabled)" +#define ENV_CONFIG_TXNSYNC \ "create,log=(file_max=10M,archive=false,enabled)," \ "transaction_sync=(enabled,method=none)" #define ENV_CONFIG_REC "log=(recover=on)" @@ -73,11 +76,15 @@ thread_run(void *arg) WT_THREAD_DATA *td; uint64_t i; int ret; - char buf[MAX_VAL], kname[64]; + size_t lsize; + char buf[MAX_VAL], kname[64], lgbuf[8]; + char large[128*1024]; __wt_random_init(&rnd); memset(buf, 0, sizeof(buf)); memset(kname, 0, sizeof(kname)); + lsize = sizeof(large); + memset(large, 0, lsize); td = (WT_THREAD_DATA *)arg; /* @@ -85,6 +92,13 @@ thread_run(void *arg) */ snprintf(buf, sizeof(buf), RECORDS_FILE, td->id); /* + * Set up a large value putting our id in it. Write it in there a + * bunch of times, but the rest of the buffer can just be zero. + */ + snprintf(lgbuf, sizeof(lgbuf), "th-%" PRIu32, td->id); + for (i = 0; i < 128; i += strlen(lgbuf)) + snprintf(&large[i], lsize - i, "%s", lgbuf); + /* * Keep a separate file with the records we wrote for checking. */ (void)unlink(buf); @@ -107,8 +121,18 @@ thread_run(void *arg) */ for (i = td->start; ; ++i) { snprintf(kname, sizeof(kname), "%" PRIu64, i); - data.size = __wt_random(&rnd) % MAX_VAL; cursor->set_key(cursor, kname); + /* + * Every 30th record write a very large record that exceeds the + * log buffer size. This forces us to use the unbuffered path. + */ + if (i % 30 == 0) { + data.size = 128 * 1024; + data.data = large; + } else { + data.size = __wt_random(&rnd) % MAX_VAL; + data.data = buf; + } cursor->set_value(cursor, &data); if ((ret = cursor->insert(cursor)) != 0) testutil_die(ret, "WT_CURSOR.insert"); @@ -136,12 +160,17 @@ fill_db(uint32_t nth) WT_THREAD_DATA *td; uint32_t i; int ret; + const char *envconf; thr = dcalloc(nth, sizeof(pthread_t)); td = dcalloc(nth, sizeof(WT_THREAD_DATA)); if (chdir(home) != 0) testutil_die(errno, "Child chdir: %s", home); - if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG, &conn)) != 0) + if (inmem) + envconf = ENV_CONFIG_DEF; + else + envconf = ENV_CONFIG_TXNSYNC; + if ((ret = wiredtiger_open(NULL, NULL, envconf, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) testutil_die(ret, "WT_CONNECTION:open_session"); @@ -187,11 +216,11 @@ main(int argc, char *argv[]) WT_CURSOR *cursor; WT_SESSION *session; WT_RAND_STATE rnd; - uint64_t key, last_key; - uint32_t absent, count, i, nth, timeout; + uint64_t absent, count, key, last_key, middle; + uint32_t i, nth, timeout; int ch, status, ret; pid_t pid; - bool rand_th, rand_time, verify_only; + bool fatal, rand_th, rand_time, verify_only; const char *working_dir; char fname[64], kname[64]; @@ -200,17 +229,21 @@ main(int argc, char *argv[]) else ++progname; + inmem = false; nth = MIN_TH; rand_th = rand_time = true; timeout = MIN_TIME; verify_only = false; working_dir = "WT_TEST.random-abort"; - while ((ch = __wt_getopt(progname, argc, argv, "h:T:t:v")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "h:mT:t:v")) != EOF) switch (ch) { case 'h': working_dir = __wt_optarg; break; + case 'm': + inmem = true; + break; case 'T': rand_th = false; nth = (uint32_t)atoi(__wt_optarg); @@ -303,7 +336,9 @@ main(int argc, char *argv[]) testutil_die(ret, "WT_SESSION.open_cursor: %s", uri); absent = count = 0; + fatal = false; for (i = 0; i < nth; ++i) { + middle = 0; snprintf(fname, sizeof(fname), RECORDS_FILE, i); if ((fp = fopen(fname, "r")) == NULL) { fprintf(stderr, @@ -313,8 +348,10 @@ main(int argc, char *argv[]) /* * For every key in the saved file, verify that the key exists - * in the table after recovery. Since we did write-no-sync, we - * expect every key to have been recovered. + * in the table after recovery. If we're doing in-memory + * log buffering we never expect a record missing in the middle, + * but records may be missing at the end. If we did + * write-no-sync, we expect every key to have been recovered. */ for (last_key = UINT64_MAX;; ++count, last_key = key) { ret = fscanf(fp, "%" SCNu64 "\n", &key); @@ -338,9 +375,20 @@ main(int argc, char *argv[]) if ((ret = cursor->search(cursor)) != 0) { if (ret != WT_NOTFOUND) testutil_die(ret, "search"); - printf("%s: no record with key %" PRIu64 "\n", - fname, key); - ++absent; + if (!inmem) + printf("%s: no record with key %" + PRIu64 "\n", fname, key); + absent++; + middle = key; + } else if (middle != 0) { + /* + * We should never find an existing key after + * we have detected one missing. + */ + printf("%s: after absent record at %" PRIu64 + " key %" PRIu64 " exists\n", + fname, middle, key); + fatal = true; } } if (fclose(fp) != 0) @@ -348,11 +396,13 @@ main(int argc, char *argv[]) } if ((ret = conn->close(conn, NULL)) != 0) testutil_die(ret, "WT_CONNECTION:close"); - if (absent) { - printf("%" PRIu32 " record(s) absent from %" PRIu32 "\n", + if (fatal) + return (EXIT_FAILURE); + if (!inmem && absent) { + printf("%" PRIu64 " record(s) absent from %" PRIu64 "\n", absent, count); return (EXIT_FAILURE); } - printf("%" PRIu32 " records verified\n", count); + printf("%" PRIu64 " records verified\n", count); return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/recovery/smoke.sh b/src/third_party/wiredtiger/test/recovery/smoke.sh index c7677b64503..ce0662d3b2b 100755 --- a/src/third_party/wiredtiger/test/recovery/smoke.sh +++ b/src/third_party/wiredtiger/test/recovery/smoke.sh @@ -5,4 +5,5 @@ set -e # Smoke-test recovery as part of running "make check". $TEST_WRAPPER ./random-abort -t 10 -T 5 +$TEST_WRAPPER ./random-abort -m -t 10 -T 5 $TEST_WRAPPER ./truncated-log diff --git a/src/third_party/wiredtiger/test/suite/test_compact02.py b/src/third_party/wiredtiger/test/suite/test_compact02.py index fe31aa1cbe1..3a4ca6cbc7e 100644 --- a/src/third_party/wiredtiger/test/suite/test_compact02.py +++ b/src/third_party/wiredtiger/test/suite/test_compact02.py @@ -80,8 +80,11 @@ class test_compact02(wttest.WiredTigerTestCase): # Return the size of the file def getSize(self): + # To allow this to work on systems without ftruncate, + # get the portion of the file allocated, via 'statistics=(all)', + # not the physical file size, via 'statistics=(size)'. cstat = self.session.open_cursor( - 'statistics:' + self.uri, None, 'statistics=(size)') + 'statistics:' + self.uri, None, 'statistics=(all)') sz = cstat[stat.dsrc.block_size][2] cstat.close() return sz @@ -96,7 +99,7 @@ class test_compact02(wttest.WiredTigerTestCase): self.home = '.' conn_params = 'create,' + \ cacheSize + ',error_prefix="%s: ",' % self.shortid() + \ - 'statistics=(fast),eviction_dirty_target=99,eviction_dirty_trigger=99' + 'statistics=(all),eviction_dirty_target=99,eviction_dirty_trigger=99' try: self.conn = wiredtiger.wiredtiger_open(self.home, conn_params) except wiredtiger.WiredTigerError as e: diff --git a/src/third_party/wiredtiger/test/suite/test_reconfig04.py b/src/third_party/wiredtiger/test/suite/test_reconfig04.py new file mode 100644 index 00000000000..be5e6d3729e --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_reconfig04.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import fnmatch, os, time +import wiredtiger, wttest +from wtdataset import SimpleDataSet + +# test_reconfig04.py +# Test WT_SESSION::reconfigure +class test_reconfig04(wttest.WiredTigerTestCase): + def test_session_reconfigure(self): + self.session.reconfigure('ignore_cache_size=false') + + self.session.reconfigure('isolation=snapshot') + self.session.reconfigure('isolation=read-committed') + self.session.reconfigure('isolation=read-uncommitted') + + self.session.reconfigure('ignore_cache_size=true') + self.session.reconfigure('isolation=snapshot') + +if __name__ == '__main__': + wttest.run() |