From cde925294d785851a593f5f0a92c6b7c73e7d7ab Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Tue, 1 Jun 2021 19:22:08 +1000 Subject: Import wiredtiger: 0784f508f9ea28834c848ec36467fa9ad7a48f15 from branch mongodb-5.0 ref: a449c141a4..0784f508f9 for: 5.0.0-rc1 WT-7493 Add a new connection config to control the page eviction with update restore eviction --- src/third_party/wiredtiger/dist/api_data.py | 3 + src/third_party/wiredtiger/import.data | 2 +- src/third_party/wiredtiger/src/config/config_def.c | 53 ++++++++-------- src/third_party/wiredtiger/src/conn/conn_api.c | 5 ++ src/third_party/wiredtiger/src/evict/evict_lru.c | 5 ++ .../wiredtiger/src/include/connection.h | 1 + .../wiredtiger/src/include/wiredtiger.in | 5 ++ .../wiredtiger/src/reconcile/rec_visibility.c | 11 +++- .../wiredtiger/test/suite/test_debug_mode09.py | 73 ++++++++++++++++++++++ 9 files changed, 129 insertions(+), 29 deletions(-) create mode 100755 src/third_party/wiredtiger/test/suite/test_debug_mode09.py (limited to 'src/third_party') diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index e1c95a1c8bd..377fa72013a 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -596,6 +596,9 @@ connection_runtime_config = [ versions of WiredTiger. These operations are informational and skipped in recovery.''', type='boolean'), + Config('update_restore_evict', 'false', r''' + if true, control all dirty page evictions through forcing update restore eviction.''', + type='boolean'), ]), Config('error_prefix', '', r''' prefix string for error messages'''), diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 939d5e44e43..2d2fa53ff3f 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.0", - "commit": "a449c141a401dfdabeaec84f3a8f968a969e6002" + "commit": "0784f508f9ea28834c848ec36467fa9ad7a48f15" } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index ca7b6bca448..908e02e8688 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -48,7 +48,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_debug_mode_subconfigs[] = { {"realloc_exact", "boolean", NULL, NULL, NULL, 0}, {"rollback_error", "int", NULL, "min=0,max=10M", NULL, 0}, {"slow_checkpoint", "boolean", NULL, NULL, NULL, 0}, - {"table_logging", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + {"table_logging", "boolean", NULL, NULL, NULL, 0}, + {"update_restore_evict", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_wiredtiger_open_eviction_subconfigs[] = { {"threads_max", "int", NULL, "min=1,max=20", NULL, 0}, @@ -107,7 +108,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"compatibility", "category", NULL, NULL, confchk_WT_CONNECTION_reconfigure_compatibility_subconfigs, 1}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10}, {"error_prefix", "string", NULL, NULL, NULL, 0}, {"eviction", "category", NULL, NULL, confchk_wiredtiger_open_eviction_subconfigs, 2}, {"eviction_checkpoint_target", "int", NULL, "min=0,max=10TB", NULL, 0}, @@ -756,7 +757,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -834,7 +835,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -911,7 +912,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -984,7 +985,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -1069,11 +1070,11 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "debug_mode=(checkpoint_retention=0,corruption_abort=true," "cursor_copy=false,eviction=false,log_retention=0," "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),error_prefix=,eviction=(threads_max=8," - "threads_min=1),eviction_checkpoint_target=1," - "eviction_dirty_target=5,eviction_dirty_trigger=20," - "eviction_target=80,eviction_trigger=95,eviction_updates_target=0" - ",eviction_updates_trigger=0," + "table_logging=false,update_restore_evict=false),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," + "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" + ",eviction_updates_target=0,eviction_updates_trigger=0," "file_manager=(close_handle_minimum=250,close_idle_time=30," "close_scan_interval=10),history_store=(file_max=0)," "io_capacity=(total=0),log=(archive=true,os_cache_dirty_pct=0," @@ -1319,10 +1320,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "config_base=true,create=false,debug_mode=(checkpoint_retention=0" ",corruption_abort=true,cursor_copy=false,eviction=false," "log_retention=0,realloc_exact=false,rollback_error=0," - "slow_checkpoint=false,table_logging=false),direct_io=," - "encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "slow_checkpoint=false,table_logging=false," + "update_restore_evict=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "exclusive=false,extensions=,file_close_sync=true,file_extend=," @@ -1354,10 +1355,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "config_base=true,create=false,debug_mode=(checkpoint_retention=0" ",corruption_abort=true,cursor_copy=false,eviction=false," "log_retention=0,realloc_exact=false,rollback_error=0," - "slow_checkpoint=false,table_logging=false),direct_io=," - "encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "slow_checkpoint=false,table_logging=false," + "update_restore_evict=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "exclusive=false,extensions=,file_close_sync=true,file_extend=," @@ -1390,9 +1391,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "debug_mode=(checkpoint_retention=0,corruption_abort=true," "cursor_copy=false,eviction=false,log_retention=0," "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=1,eviction_dirty_target=5," + "table_logging=false,update_restore_evict=false),direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "extensions=,file_close_sync=true,file_extend=," @@ -1424,9 +1426,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "debug_mode=(checkpoint_retention=0,corruption_abort=true," "cursor_copy=false,eviction=false,log_retention=0," "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=1,eviction_dirty_target=5," + "table_logging=false,update_restore_evict=false),direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "extensions=,file_close_sync=true,file_extend=," diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 526fdbfba86..b35d198c3aa 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1962,6 +1962,11 @@ __wt_debug_mode_config(WT_SESSION_IMPL *session, const char *cfg[]) else FLD_CLR(conn->log_flags, WT_CONN_LOG_DEBUG_MODE); + WT_RET(__wt_config_gets(session, cfg, "debug_mode.update_restore_evict", &cval)); + if (cval.val) + FLD_SET(conn->debug_flags, WT_CONN_DEBUG_UPDATE_RESTORE_EVICT); + else + FLD_CLR(conn->debug_flags, WT_CONN_DEBUG_UPDATE_RESTORE_EVICT); return (0); } diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 8aa018bb063..26f572ce517 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -636,6 +636,11 @@ __evict_update_work(WT_SESSION_IMPL *session) } else LF_SET(WT_CACHE_EVICT_NOKEEP); + if (FLD_ISSET(conn->debug_flags, WT_CONN_DEBUG_UPDATE_RESTORE_EVICT)) { + LF_SET(WT_CACHE_EVICT_SCRUB); + LF_CLR(WT_CACHE_EVICT_NOKEEP); + } + /* * With an in-memory cache, we only do dirty eviction in order to scrub pages. */ diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 6a90cb5e9fc..b13ab2b911e 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -543,6 +543,7 @@ struct __wt_connection_impl { #define WT_CONN_DEBUG_CURSOR_COPY 0x04u #define WT_CONN_DEBUG_REALLOC_EXACT 0x08u #define WT_CONN_DEBUG_SLOW_CKPT 0x10u +#define WT_CONN_DEBUG_UPDATE_RESTORE_EVICT 0x20u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint64_t debug_flags; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 13cf882a45f..d0a4c0ad38e 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -2074,6 +2074,8 @@ struct __wt_connection { * operations\, even operations for tables with logging turned off. This setting introduces * a log format change that may break older versions of WiredTiger. These operations are * informational and skipped in recovery., a boolean flag; default \c false.} + * @config{    update_restore_evict, if true\, control all dirty page + * evictions through forcing update restore eviction., a boolean flag; default \c false.} * @config{ ),,} * @config{error_prefix, prefix string for error messages., a string; default empty.} * @config{eviction = (, eviction configuration options., a set of related configuration @@ -2738,6 +2740,9 @@ struct __wt_connection { * tables with logging turned off. This setting introduces a log format change that may break older * versions of WiredTiger. These operations are informational and skipped in recovery., a boolean * flag; default \c false.} + * @config{    update_restore_evict, if true\, control + * all dirty page evictions through forcing update restore eviction., a boolean flag; default \c + * false.} * @config{ ),,} * @config{direct_io, Use \c O_DIRECT on POSIX systems\, and \c FILE_FLAG_NO_BUFFERING on Windows to * access files. Options are given as a list\, such as "direct_io=[data]". Configuring diff --git a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c index d1e4d909b50..ef5c3246e30 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c @@ -255,7 +255,7 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v upd_memsize = 0; max_ts = WT_TS_NONE; max_txn = WT_TXN_NONE; - has_newer_updates = upd_saved = false; + has_newer_updates = supd_restore = upd_saved = false; is_hs_page = F_ISSET(session->dhandle, WT_DHANDLE_HS); session_txnid = WT_SESSION_TXN_SHARED(session)->id; @@ -615,8 +615,6 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v supd_restore = F_ISSET(r, WT_REC_EVICT) && (has_newer_updates || F_ISSET(S2C(session), WT_CONN_IN_MEMORY) || page->type == WT_PAGE_COL_FIX); - if (supd_restore) - r->cache_write_restore = true; WT_ERR(__rec_update_save(session, r, ins, ripcip, upd_select->upd != NULL && upd_select->upd->type == WT_UPDATE_TOMBSTONE ? NULL : upd_select->upd, @@ -633,6 +631,13 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v upd_saved = upd_select->upd_saved = true; } + /* + * Set statistics for update restore evictions. Update restore eviction debug mode forces update + * restores to both committed or uncommitted changes. + */ + if (supd_restore || F_ISSET(r, WT_REC_SCRUB)) + r->cache_write_restore = true; + /* * Paranoia: check that we didn't choose an update that has since been rolled back. */ diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode09.py b/src/third_party/wiredtiger/test/suite/test_debug_mode09.py new file mode 100755 index 00000000000..a44e25ca7ce --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode09.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from wiredtiger import stat +import wttest + +# test_debug_mode09.py +# Test the debug mode setting for update_restore_evict. +# Force update restore eviction, whenever we evict a page. The debug mode +# is only effective on high cache pressure as WiredTiger can potentially decide +# to do an update restore evict on a page, when the cache pressure requirements are not met. +# This means setting eviction target low and cache size high. +class test_debug_mode09(wttest.WiredTigerTestCase): + conn_config = 'cache_size=10MB,statistics=(all),eviction_target=10' + uri = "table:test_debug_mode09" + + # Insert a bunch of data to trigger eviction + def trigger_eviction(self, uri): + cursor = self.session.open_cursor(uri) + for i in range(0, 20000): + self.session.begin_transaction() + cursor[i] = 'b' * 500 + self.session.commit_transaction() + + def test_update_restore_evict(self): + self.session.create(self.uri, 'key_format=i,value_format=S') + + self.trigger_eviction(self.uri) + + # Read the statistics of pages that have been update restored without update_restore + stat_cursor = self.session.open_cursor('statistics:') + pages_update_restored = stat_cursor[stat.conn.cache_write_restore][2] + stat_cursor.close() + self.assertGreater(pages_update_restored, 0) + + # Restart the connection with update restore evict config and a clean table + self.close_conn() + self.conn_config += ",debug_mode=(update_restore_evict=true)" + self.reopen_conn(".", self.conn_config) + self.session.drop(self.uri) + self.session.create(self.uri, 'key_format=i,value_format=S') + + self.trigger_eviction(self.uri) + + stat_cursor = self.session.open_cursor('statistics:') + forced_pages_update_restore = stat_cursor[stat.conn.cache_write_restore][2] + stat_cursor.close() + self.assertGreater(forced_pages_update_restore, pages_update_restored) -- cgit v1.2.1