diff options
author | Luke Chen <luke.chen@mongodb.com> | 2020-11-23 11:17:21 +1100 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-11-23 01:38:20 +0000 |
commit | 55ad85b6058c1c320dd811ac8c0d7e4aec7696f5 (patch) | |
tree | f420c0fad43453a9421371c3c2ebff7527d72d00 /src/third_party | |
parent | b4de2e9b4fce34587a3c7ddb1193c43d865cfe34 (diff) | |
download | mongo-55ad85b6058c1c320dd811ac8c0d7e4aec7696f5.tar.gz |
Import wiredtiger: f5c56624e1367fa4dd105d43ffc244084e9069f9 from branch mongodb-5.0
ref: b22e16b764..f5c56624e1
for: 4.9.0
Reverted ticket(s):
WT-6712 Allow RTS to operate when stable timestamp is not set
Diffstat (limited to 'src/third_party')
11 files changed, 47 insertions, 83 deletions
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 8dceff68b67..df0eeeee08c 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.0", - "commit": "b22e16b7643e0e07c784962899b3a45728536947" + "commit": "f5c56624e1367fa4dd105d43ffc244084e9069f9" } diff --git a/src/third_party/wiredtiger/src/block/block_open.c b/src/third_party/wiredtiger/src/block/block_open.c index d97f3a86f7d..61392492e96 100644 --- a/src/third_party/wiredtiger/src/block/block_open.c +++ b/src/third_party/wiredtiger/src/block/block_open.c @@ -391,12 +391,7 @@ __desc_read(WT_SESSION_IMPL *session, uint32_t allocsize, WT_BLOCK *block) */ if (F_ISSET(session, WT_SESSION_IMPORT_REPAIR)) goto err; - - if (F_ISSET(session, WT_SESSION_ROLLBACK_TO_STABLE)) - ret = ENOENT; - else - WT_ERR_MSG( - session, WT_ERROR, "%s does not appear to be a WiredTiger file", block->name); + WT_ERR_MSG(session, WT_ERROR, "%s does not appear to be a WiredTiger file", block->name); } if (desc->majorv > WT_BLOCK_MAJOR_VERSION || diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index c08c5d457b7..e049d277e32 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -929,9 +929,13 @@ done: * Perform rollback to stable only when the following conditions met. * 1. The connection is not read-only. A read-only connection expects that there shouldn't be * any changes that need to be done on the database other than reading. - * 2. The history store file was found in the metadata. + * 2. A valid recovery timestamp. The recovery timestamp is the stable timestamp retrieved + * from the metadata checkpoint information to indicate the stable timestamp when the + * checkpoint happened. Anything updates newer than this timestamp must rollback. + * 3. The history store file was found in the metadata. */ - if (hs_exists && !F_ISSET(conn, WT_CONN_READONLY)) { + if (hs_exists && !F_ISSET(conn, WT_CONN_READONLY) && + conn->txn_global.recovery_timestamp != WT_TS_NONE) { /* Start the eviction threads for rollback to stable if not already started. */ if (!eviction_started) { WT_ERR(__wt_evict_create(session)); @@ -958,10 +962,7 @@ done: * stable. */ conn->txn_global.stable_timestamp = conn->txn_global.recovery_timestamp; - conn->txn_global.has_stable_timestamp = false; - - if (conn->txn_global.recovery_timestamp != WT_TS_NONE) - conn->txn_global.has_stable_timestamp = true; + conn->txn_global.has_stable_timestamp = true; __wt_verbose(session, WT_VERB_RTS, "Performing recovery rollback_to_stable with stable timestamp: %s and oldest timestamp: " diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index ce904576651..852955ab317 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -987,9 +987,17 @@ __rollback_to_stable_btree(WT_SESSION_IMPL *session, wt_timestamp_t rollback_tim static int __rollback_to_stable_check(WT_SESSION_IMPL *session) { + WT_CONNECTION_IMPL *conn; WT_DECL_RET; + WT_TXN_GLOBAL *txn_global; bool txn_active; + conn = S2C(session); + txn_global = &conn->txn_global; + + if (!txn_global->has_stable_timestamp) + WT_RET_MSG(session, EINVAL, "rollback_to_stable requires a stable timestamp"); + /* * Help the user comply with the requirement that there are no concurrent operations. Protect * against spurious conflicts with the sweep server: we exclude it from running concurrent with @@ -1161,19 +1169,17 @@ static int __rollback_to_stable_btree_apply(WT_SESSION_IMPL *session) { WT_CONFIG ckptconf; - WT_CONFIG_ITEM cval, value, key; + WT_CONFIG_ITEM cval, durableval, key; WT_CURSOR *cursor; WT_DECL_RET; WT_TXN_GLOBAL *txn_global; wt_timestamp_t max_durable_ts, newest_start_durable_ts, newest_stop_durable_ts, rollback_timestamp; - size_t addr_size; char ts_string[2][WT_TS_INT_STRING_SIZE]; const char *config, *uri; bool durable_ts_found, prepared_updates; txn_global = &S2C(session)->txn_global; - addr_size = 0; /* * Copy the stable timestamp, otherwise we'd need to lock it each time it's accessed. Even @@ -1207,62 +1213,32 @@ __rollback_to_stable_btree_apply(WT_SESSION_IMPL *session) WT_ERR(__wt_config_getones(session, config, "checkpoint", &cval)); __wt_config_subinit(session, &ckptconf, &cval); for (; __wt_config_next(&ckptconf, &key, &cval) == 0;) { - ret = __wt_config_subgets(session, &cval, "newest_start_durable_ts", &value); + ret = __wt_config_subgets(session, &cval, "newest_start_durable_ts", &durableval); if (ret == 0) { newest_start_durable_ts = - WT_MAX(newest_start_durable_ts, (wt_timestamp_t)value.val); + WT_MAX(newest_start_durable_ts, (wt_timestamp_t)durableval.val); durable_ts_found = true; } WT_ERR_NOTFOUND_OK(ret, false); - ret = __wt_config_subgets(session, &cval, "newest_stop_durable_ts", &value); + ret = __wt_config_subgets(session, &cval, "newest_stop_durable_ts", &durableval); if (ret == 0) { - newest_stop_durable_ts = WT_MAX(newest_stop_durable_ts, (wt_timestamp_t)value.val); + newest_stop_durable_ts = + WT_MAX(newest_stop_durable_ts, (wt_timestamp_t)durableval.val); durable_ts_found = true; } WT_ERR_NOTFOUND_OK(ret, false); - ret = __wt_config_subgets(session, &cval, "prepare", &value); + ret = __wt_config_subgets(session, &cval, "prepare", &durableval); if (ret == 0) { - if (value.val) + if (durableval.val) prepared_updates = true; } WT_ERR_NOTFOUND_OK(ret, false); - ret = __wt_config_subgets(session, &cval, "addr", &value); - if (ret == 0) - addr_size = value.len; - WT_ERR_NOTFOUND_OK(ret, false); } max_durable_ts = WT_MAX(newest_start_durable_ts, newest_stop_durable_ts); - - /* - * The rollback to stable will skip the tables during recovery in the following conditions - * 1. Empty table - * 2. Table has timestamped updates without a stable timestamp. - */ - if (F_ISSET(S2C(session), WT_CONN_RECOVERING) && - (addr_size == 0 || - (txn_global->stable_timestamp == WT_TS_NONE && max_durable_ts != WT_TS_NONE))) { - __wt_verbose(session, WT_VERB_RTS, "Skip rollback to stable on file %s because %s", uri, - addr_size == 0 ? "its checkpoint address length is 0" : - "it has timestamped updates and the stable timestamp is 0"); - continue; - } - - /* Set this flag to return error instead of panic if file is corrupted. */ - F_SET(session, WT_SESSION_QUIET_CORRUPT_FILE); ret = __wt_session_get_dhandle(session, uri, NULL, NULL, 0); - F_CLR(session, WT_SESSION_QUIET_CORRUPT_FILE); - - /* - * Ignore performing rollback to stable on files that does not exist or the files where - * corruption is detected. - */ - if ((ret == ENOENT) || - (ret == WT_ERROR && F_ISSET(S2C(session), WT_CONN_DATA_CORRUPTION))) { - __wt_verbose(session, WT_VERB_RTS, - "Ignore performing rollback to stable on %s because the file %s", uri, - ret == ENOENT ? "does not exist" : "is corrupted."); + /* Ignore performing rollback to stable on files that don't exist. */ + if (ret == ENOENT) continue; - } WT_ERR(ret); /* diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py index afb96a09043..c5d3a9b7050 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py @@ -165,12 +165,8 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): # After simulating a crash, search for the keys inserted. txn_config = 'read_timestamp=' + timestamp_str(5) + ',ignore_prepare=false' - if self.commit == True: - # Search keys with timestamp 5, ignore_prepare=false and expect the cursor search to return WT_NOTFOUND. - self.search_keys_timestamp_and_ignore(ds, txn_config, None) - else: - # Search keys with timestamp 5, ignore_prepare=false and expect the cursor value to be commit_value. - self.search_keys_timestamp_and_ignore(ds, txn_config, commit_value) + # Search keys with timestamp 5, ignore_prepare=false and expect the cursor value to be commit_value. + self.search_keys_timestamp_and_ignore(ds, txn_config, commit_value) txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=true' # Search keys with timestamp 20, ignore_prepare=true and expect the cursor search to return WT_NOTFOUND. diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py index f3c93509d63..b16ee76a1f2 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py @@ -76,6 +76,10 @@ class test_rollback_to_stable05(test_rollback_to_stable_base): self, uri_2, 0, key_format="i", value_format="S", config='log=(enabled=false)') ds_2.populate() + # Pin oldest and stable to timestamp 1. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + + ',stable_timestamp=' + timestamp_str(1)) + valuea = "aaaaa" * 100 valueb = "bbbbb" * 100 valuec = "ccccc" * 100 @@ -109,6 +113,12 @@ class test_rollback_to_stable05(test_rollback_to_stable_base): self.large_updates(uri_2, valued, ds_2, nrows, 0) self.check(valued, uri_2, nrows, 0) + # Pin stable to timestamp 20 if prepare otherwise 10. + if self.prepare: + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) + else: + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) + # Checkpoint to ensure that all the data is flushed. if not self.in_memory: self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_schema08.py b/src/third_party/wiredtiger/test/suite/test_schema08.py index ceda29fdb30..91925ffc975 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema08.py +++ b/src/third_party/wiredtiger/test/suite/test_schema08.py @@ -140,13 +140,10 @@ class test_schema08(wttest.WiredTigerTestCase, suite_subprocess): # Make an initial copy as well as a copy for each LSN we save. # Truncate the log to the appropriate offset as we make each copy. olddir = "." - errfile="errfile.txt" for lsn in self.lsns: newdir = self.backup_pfx + str(lsn) outfile = newdir + '.txt' - self.runWt(['-R', '-h', newdir, 'list', '-v'], errfilename=errfile, outfilename=outfile) - if os.path.isfile(errfile) and os.path.getsize(errfile) > 0: - self.check_file_contains(errfile,'No such file or directory') + self.runWt(['-R', '-h', newdir, 'list', '-v'], outfilename=outfile) # Test that creating and dropping tables does not write individual # log records. diff --git a/src/third_party/wiredtiger/test/suite/test_txn16.py b/src/third_party/wiredtiger/test/suite/test_txn16.py index 6d7620218db..9e2829246e5 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn16.py +++ b/src/third_party/wiredtiger/test/suite/test_txn16.py @@ -92,10 +92,11 @@ class test_txn16(wttest.WiredTigerTestCase, suite_subprocess): cur_logs = fnmatch.filter(os.listdir(homedir), "*gerLog*") scur = set(cur_logs) sorig = set(orig_logs) - # There can be overlap with the log files that were - # there originally. Because some pages are rolled back - # as part of RTS. - self.assertEqual(scur.isdisjoint(sorig), False) + # There should never be overlap with the log files that + # were there originally. Mostly this checks that after + # opening with logging disabled and then re-enabled, we + # don't see log file 1. + self.assertEqual(scur.isdisjoint(sorig), True) if loop > 1: # We should be creating the same log files each time. for l in cur_logs: diff --git a/src/third_party/wiredtiger/test/suite/test_txn22.py b/src/third_party/wiredtiger/test/suite/test_txn22.py index 8cf10408ad4..31d569951a7 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn22.py +++ b/src/third_party/wiredtiger/test/suite/test_txn22.py @@ -170,9 +170,5 @@ class test_txn22(wttest.WiredTigerTestCase, suite_subprocess): lambda: self.reopen_conn(salvagedir, salvage_config), '/.*/') - # The test may output the following error message while opening a file that - # does not exist. Ignore that. - self.ignoreStderrPatternIfExists('No such file or directory') - if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_verify.py b/src/third_party/wiredtiger/test/suite/test_verify.py index 8279f772e3c..1fffcf66612 100755 --- a/src/third_party/wiredtiger/test/suite/test_verify.py +++ b/src/third_party/wiredtiger/test/suite/test_verify.py @@ -182,9 +182,7 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): f.truncate(0) self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out", failure=True) - # The test may output the following error message while opening a file that - # does not exist. Ignore that. - self.ignoreStderrPatternIfExists('No such file or directory') + self.check_non_empty_file("verifyerr.out") def test_verify_process_zero_length(self): """ @@ -197,9 +195,7 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): f.truncate(0) self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out", failure=True) - # The test may output the following error message while opening a file that - # does not exist. Ignore that. - self.ignoreStderrPatternIfExists('No such file or directory') + self.check_non_empty_file("verifyerr.out") if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 0e7bb3ae6f9..170e39a5b2c 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -527,10 +527,6 @@ class WiredTigerTestCase(unittest.TestCase): if self.captureout.hasUnexpectedOutput(self): self.captureout.checkAdditionalPattern(self, pat) - def ignoreStderrPatternIfExists(self, pat): - if self.captureerr.hasUnexpectedOutput(self): - self.captureerr.checkAdditionalPattern(self, pat) - def assertRaisesWithMessage(self, exceptionType, expr, message): """ Like TestCase.assertRaises(), but also checks to see |