diff options
author | Luke Chen <luke.chen@mongodb.com> | 2021-05-25 16:05:07 +1000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-05-25 06:28:10 +0000 |
commit | 9529b4dbff677244a44b6e135abfb1858928ad48 (patch) | |
tree | 852cbfbfc60239bb836d07ab6317c36a5ec00dfc | |
parent | 61710dae9c72f603f15882f3005b729c7f84bcf6 (diff) | |
download | mongo-9529b4dbff677244a44b6e135abfb1858928ad48.tar.gz |
Import wiredtiger: cdeef9d4078d7bedaac368fe02953b4341b65228 from branch mongodb-5.0
ref: 4f12812afc..cdeef9d407
for: 5.0.0
WT-7493 Add a new connection config to control the page eviction with update restore eviction
9 files changed, 117 insertions, 31 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index e1c95a1c8bd..3a6abbbab5e 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -596,6 +596,9 @@ connection_runtime_config = [ versions of WiredTiger. These operations are informational and skipped in recovery.''', type='boolean'), + Config('update_restore_evict', 'false', r''' + if true, control the page eviction through forcing update restore eviction.''', + type='boolean'), ]), Config('error_prefix', '', r''' prefix string for error messages'''), diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index c70cc6bc94e..f7d71510134 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.0", - "commit": "4f12812afc32494be41c1a1c911c9f986a5bc5e9" + "commit": "cdeef9d4078d7bedaac368fe02953b4341b65228" } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index ca7b6bca448..908e02e8688 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -48,7 +48,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_debug_mode_subconfigs[] = { {"realloc_exact", "boolean", NULL, NULL, NULL, 0}, {"rollback_error", "int", NULL, "min=0,max=10M", NULL, 0}, {"slow_checkpoint", "boolean", NULL, NULL, NULL, 0}, - {"table_logging", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + {"table_logging", "boolean", NULL, NULL, NULL, 0}, + {"update_restore_evict", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_wiredtiger_open_eviction_subconfigs[] = { {"threads_max", "int", NULL, "min=1,max=20", NULL, 0}, @@ -107,7 +108,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"compatibility", "category", NULL, NULL, confchk_WT_CONNECTION_reconfigure_compatibility_subconfigs, 1}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10}, {"error_prefix", "string", NULL, NULL, NULL, 0}, {"eviction", "category", NULL, NULL, confchk_wiredtiger_open_eviction_subconfigs, 2}, {"eviction_checkpoint_target", "int", NULL, "min=0,max=10TB", NULL, 0}, @@ -756,7 +757,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -834,7 +835,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -911,7 +912,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -984,7 +985,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -1069,11 +1070,11 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "debug_mode=(checkpoint_retention=0,corruption_abort=true," "cursor_copy=false,eviction=false,log_retention=0," "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),error_prefix=,eviction=(threads_max=8," - "threads_min=1),eviction_checkpoint_target=1," - "eviction_dirty_target=5,eviction_dirty_trigger=20," - "eviction_target=80,eviction_trigger=95,eviction_updates_target=0" - ",eviction_updates_trigger=0," + "table_logging=false,update_restore_evict=false),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," + "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" + ",eviction_updates_target=0,eviction_updates_trigger=0," "file_manager=(close_handle_minimum=250,close_idle_time=30," "close_scan_interval=10),history_store=(file_max=0)," "io_capacity=(total=0),log=(archive=true,os_cache_dirty_pct=0," @@ -1319,10 +1320,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "config_base=true,create=false,debug_mode=(checkpoint_retention=0" ",corruption_abort=true,cursor_copy=false,eviction=false," "log_retention=0,realloc_exact=false,rollback_error=0," - "slow_checkpoint=false,table_logging=false),direct_io=," - "encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "slow_checkpoint=false,table_logging=false," + "update_restore_evict=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "exclusive=false,extensions=,file_close_sync=true,file_extend=," @@ -1354,10 +1355,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "config_base=true,create=false,debug_mode=(checkpoint_retention=0" ",corruption_abort=true,cursor_copy=false,eviction=false," "log_retention=0,realloc_exact=false,rollback_error=0," - "slow_checkpoint=false,table_logging=false),direct_io=," - "encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "slow_checkpoint=false,table_logging=false," + "update_restore_evict=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "exclusive=false,extensions=,file_close_sync=true,file_extend=," @@ -1390,9 +1391,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "debug_mode=(checkpoint_retention=0,corruption_abort=true," "cursor_copy=false,eviction=false,log_retention=0," "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=1,eviction_dirty_target=5," + "table_logging=false,update_restore_evict=false),direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "extensions=,file_close_sync=true,file_extend=," @@ -1424,9 +1426,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "debug_mode=(checkpoint_retention=0,corruption_abort=true," "cursor_copy=false,eviction=false,log_retention=0," "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=1,eviction_dirty_target=5," + "table_logging=false,update_restore_evict=false),direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "extensions=,file_close_sync=true,file_extend=," diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 3d7b5fed416..71e335cd25b 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1961,6 +1961,11 @@ __wt_debug_mode_config(WT_SESSION_IMPL *session, const char *cfg[]) else FLD_CLR(conn->log_flags, WT_CONN_LOG_DEBUG_MODE); + WT_RET(__wt_config_gets(session, cfg, "debug_mode.update_restore_evict", &cval)); + if (cval.val) + FLD_SET(conn->debug_flags, WT_CONN_DEBUG_UPDATE_RESTORE_EVICT); + else + FLD_CLR(conn->debug_flags, WT_CONN_DEBUG_UPDATE_RESTORE_EVICT); return (0); } diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 8aa018bb063..8c593076711 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -633,7 +633,9 @@ __evict_update_work(WT_SESSION_IMPL *session) if (bytes_dirty < (uint64_t)((dirty_target + dirty_trigger) * bytes_max) / 200 && bytes_updates < (uint64_t)((updates_target + updates_trigger) * bytes_max) / 200) LF_SET(WT_CACHE_EVICT_SCRUB); - } else + } else if (FLD_ISSET(conn->debug_flags, WT_CONN_DEBUG_UPDATE_RESTORE_EVICT)) + LF_SET(WT_CACHE_EVICT_SCRUB); + else LF_SET(WT_CACHE_EVICT_NOKEEP); /* diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index fc40f05e0d5..4d78b695ec8 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -539,6 +539,7 @@ struct __wt_connection_impl { #define WT_CONN_DEBUG_CURSOR_COPY 0x04u #define WT_CONN_DEBUG_REALLOC_EXACT 0x08u #define WT_CONN_DEBUG_SLOW_CKPT 0x10u +#define WT_CONN_DEBUG_UPDATE_RESTORE_EVICT 0x20u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint64_t debug_flags; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 14ee53f536a..8c626235ec9 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -2074,7 +2074,10 @@ struct __wt_connection { * operations\, even operations for tables with logging turned off. This setting introduces * a log format change that may break older versions of WiredTiger. These operations are * informational and skipped in recovery., a boolean flag; default \c false.} - * @config{ ),,} + * @config{ update_restore_evict, if true\, control the page eviction + * through forcing update restore eviction., a boolean flag; default \c false.} + * @config{ + * ),,} * @config{error_prefix, prefix string for error messages., a string; default empty.} * @config{eviction = (, eviction configuration options., a set of related configuration * options defined below.} @@ -2738,6 +2741,8 @@ struct __wt_connection { * tables with logging turned off. This setting introduces a log format change that may break older * versions of WiredTiger. These operations are informational and skipped in recovery., a boolean * flag; default \c false.} + * @config{ update_restore_evict, if true\, control + * the page eviction through forcing update restore eviction., a boolean flag; default \c false.} * @config{ ),,} * @config{direct_io, Use \c O_DIRECT on POSIX systems\, and \c FILE_FLAG_NO_BUFFERING on Windows to * access files. Options are given as a list\, such as <code>"direct_io=[data]"</code>. Configuring diff --git a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c index d1e4d909b50..ef5c3246e30 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c @@ -255,7 +255,7 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v upd_memsize = 0; max_ts = WT_TS_NONE; max_txn = WT_TXN_NONE; - has_newer_updates = upd_saved = false; + has_newer_updates = supd_restore = upd_saved = false; is_hs_page = F_ISSET(session->dhandle, WT_DHANDLE_HS); session_txnid = WT_SESSION_TXN_SHARED(session)->id; @@ -615,8 +615,6 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v supd_restore = F_ISSET(r, WT_REC_EVICT) && (has_newer_updates || F_ISSET(S2C(session), WT_CONN_IN_MEMORY) || page->type == WT_PAGE_COL_FIX); - if (supd_restore) - r->cache_write_restore = true; WT_ERR(__rec_update_save(session, r, ins, ripcip, upd_select->upd != NULL && upd_select->upd->type == WT_UPDATE_TOMBSTONE ? NULL : upd_select->upd, @@ -634,6 +632,13 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v } /* + * Set statistics for update restore evictions. Update restore eviction debug mode forces update + * restores to both committed or uncommitted changes. + */ + if (supd_restore || F_ISSET(r, WT_REC_SCRUB)) + r->cache_write_restore = true; + + /* * Paranoia: check that we didn't choose an update that has since been rolled back. */ WT_ASSERT(session, upd_select->upd == NULL || upd_select->upd->txnid != WT_TXN_ABORTED); diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode09.py b/src/third_party/wiredtiger/test/suite/test_debug_mode09.py new file mode 100755 index 00000000000..996bf0a2f0e --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode09.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from wiredtiger import stat +import wttest + +# test_debug_mode09.py +# Test the debug mode setting for update_restore_evict. +# Force update restore eviction, whenever we evict a page. +class test_debug_mode09(wttest.WiredTigerTestCase): + conn_config = 'cache_size=100MB,statistics=(all),debug_mode=(update_restore_evict=true)' + + def test_update_restore_evict(self): + uri = "table:test_debug_mode09" + self.session.create(uri, 'key_format=i,value_format=S') + + # Insert a bunch of content + cursor = self.session.open_cursor(uri) + for i in range(0, 100): + self.session.begin_transaction() + cursor[i] = 'a' * 500 + self.session.commit_transaction() + cursor.close() + + # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. + cursor = self.session.open_cursor(uri, None, "debug=(release_evict=true)") + for i in range(0, 100): + cursor.set_key(i) + self.assertEqual(cursor.search(), 0) + cursor.reset() + cursor.close() + + # Read the statistics of pages that have been update restored + stat_cursor = self.session.open_cursor('statistics:') + pages_update_restored = stat_cursor[stat.conn.cache_write_restore][2] + stat_cursor.close() + self.assertEqual(pages_update_restored, 1) |