diff options
author | Gregory Wlodarek <gregory.wlodarek@mongodb.com> | 2022-08-26 08:07:46 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-08-26 09:49:33 +0000 |
commit | 4b504aaf5c9166e0ce1ff08755ef93c62666beec (patch) | |
tree | f3dcadf4804178d2e1dca44a445f607d03e190cd | |
parent | 117f34a9133927b903f50e04ca3754e6db374f4a (diff) | |
download | mongo-4b504aaf5c9166e0ce1ff08755ef93c62666beec.tar.gz |
SERVER-68925 Reintroduce check table logging settings at startup
(cherry picked from commit 994562a9c82fd89549ac8ee0861eb9f10509d32c)
-rw-r--r-- | jstests/disk/wt_table_checks.js | 164 | ||||
-rw-r--r-- | jstests/disk/wt_table_checks_read_only.js | 51 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp | 1 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_standard_index_test.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp | 158 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_util.h | 14 |
6 files changed, 82 insertions, 310 deletions
diff --git a/jstests/disk/wt_table_checks.js b/jstests/disk/wt_table_checks.js index 6d0cf003aab..2222a808ab5 100644 --- a/jstests/disk/wt_table_checks.js +++ b/jstests/disk/wt_table_checks.js @@ -8,6 +8,43 @@ load('jstests/disk/libs/wt_file_helper.js'); +function checkTableLogSettings(conn, enabled) { + conn.getDBNames().forEach(function(d) { + let collNames = + conn.getDB(d) + .runCommand({listCollections: 1, nameOnly: true, filter: {type: "collection"}}) + .cursor.firstBatch; + + collNames.forEach(function(c) { + let stats = conn.getDB(d).runCommand({collStats: c.name}); + + let logStr = "log=(enabled=" + (enabled ? "true" : "false") + ")"; + if (d == "local") { + if (c.name == "replset.minvalid" && !enabled) { + // This collection is never logged in a replica set. + logStr = "log=(enabled=false)"; + } else { + // All other collections and indexes in the 'local' database have table + // logging enabled always. + logStr = "log=(enabled=true)"; + } + } + + assert.eq(true, stats.wiredTiger.creationString.includes(logStr)); + Object.keys(stats.indexDetails).forEach(function(i) { + assert.eq(true, stats.indexDetails[i].creationString.includes(logStr)); + }); + }); + }); +} + +function checkTableChecksFileRemoved(dbpath) { + let files = listFiles(dbpath); + for (file of files) { + assert.eq(false, file.name.includes("_wt_table_checks")); + } +} + // Create a bunch of collections under various database names. let conn = MongoRunner.runMongod({}); const dbpath = conn.dbpath; @@ -16,114 +53,75 @@ for (let i = 0; i < 10; i++) { assert.commandWorked(conn.getDB(i.toString()).createCollection(i.toString())); } +checkTableLogSettings(conn, /*enabled=*/true); MongoRunner.stopMongod(conn); /** - * Test 1. The regular case, where no table logging setting modifications are needed. + * Test 1. Change into a single node replica set, which requires all of the table logging settings + * to be updated. Write the '_wt_table_checks' file and check that it gets removed. */ jsTest.log("Test 1."); -conn = startMongodOnExistingPath(dbpath, {}); -checkLog.containsJson(conn, 4366408, {loggingEnabled: true}); -MongoRunner.stopMongod(conn); - -/** - * Test 2. Repair checks all of the table logging settings. - */ -jsTest.log("Test 2."); - -assertRepairSucceeds(dbpath, conn.port, {}); - -// Cannot use checkLog here as the server is no longer running. -let logContents = rawMongoProgramOutput(); -assert(logContents.indexOf( - "Modifying the table logging settings for all existing WiredTiger tables") > 0); - -/** - * Test 3. Explicitly create the '_wt_table_checks' file to force all of the table logging setting - * modifications to be made. - */ -jsTest.log("Test 3."); - -let files = listFiles(dbpath); -for (f in files) { - assert(!files[f].name.includes("_wt_table_checks")); -} - writeFile(dbpath + "/_wt_table_checks", ""); +conn = startMongodOnExistingPath( + dbpath, {replSet: "mySet", setParameter: {logComponentVerbosity: tojson({verbosity: 1})}}); +checkTableChecksFileRemoved(dbpath); -// Cannot skip table logging checks on startup when there are previously incomplete table checks. -conn = startMongodOnExistingPath(dbpath, - {setParameter: "wiredTigerSkipTableLoggingChecksOnStartup=true"}); -assert.eq(conn, null); - -conn = startMongodOnExistingPath(dbpath, {}); -checkLog.containsJson( - conn, 4366405, {loggingEnabled: true, repair: false, hasPreviouslyIncompleteTableChecks: true}); +// Changing table logging settings. +checkLog.containsJson(conn, 22432); MongoRunner.stopMongod(conn); /** - * Test 4. Change into a single replica set, which requires all of the table logging settings to be - * updated. But simulate an interruption/crash while starting up during the table logging check - * phase. - * - * The next start up will detect an unclean shutdown causing all of the table logging settings to be - * updated. + * Test 2. Restart in standalone mode with wiredTigerSkipTableLoggingChecksOnStartup. No table log + * settings are updated. Write the '_wt_table_checks' file and check that it gets removed. */ -jsTest.log("Test 4."); - +jsTest.log("Test 2."); +writeFile(dbpath + "/_wt_table_checks", ""); conn = startMongodOnExistingPath(dbpath, { - replSet: "mySet", - setParameter: - "failpoint.crashAfterUpdatingFirstTableLoggingSettings=" + tojson({"mode": "alwaysOn"}) + setParameter: { + wiredTigerSkipTableLoggingChecksOnStartup: true, + logComponentVerbosity: tojson({verbosity: 1}) + } }); -assert(!conn); +checkTableChecksFileRemoved(dbpath); -// Cannot use checkLog here as the server is no longer running. -logContents = rawMongoProgramOutput(); -assert(logContents.indexOf( - "Crashing due to 'crashAfterUpdatingFirstTableLoggingSettings' fail point") > 0); +// Skipping table logging checks. +checkLog.containsJson(conn, 5548302); -// The '_wt_table_checks' still exists, so all table logging settings should be modified. -conn = startMongodOnExistingPath(dbpath, {}); -checkLog.containsJson( - conn, 4366405, {loggingEnabled: true, repair: false, hasPreviouslyIncompleteTableChecks: true}); +// Changing table logging settings. +assert(checkLog.checkContainsWithCountJson(conn, 22432, undefined, 0)); +checkTableLogSettings(conn, /*enabled=*/false); MongoRunner.stopMongod(conn); /** - * Test 5. Change into a single node replica set, which requires all of the table logging settings - * to be updated as the node was successfully started up as a standalone the last time. + * Test 3. Change into a single node replica set again. Table log settings are checked but none are + * changed. Write the '_wt_table_checks' file and check that it gets removed. */ -jsTest.log("Test 5."); +jsTestLog("Test 3."); +writeFile(dbpath + "/_wt_table_checks", ""); +conn = startMongodOnExistingPath( + dbpath, {replSet: "mySet", setParameter: {logComponentVerbosity: tojson({verbosity: 1})}}); +checkTableChecksFileRemoved(dbpath); -conn = startMongodOnExistingPath(dbpath, {replSet: "mySet"}); -checkLog.containsJson(conn, 4366406, {loggingEnabled: false}); +// Changing table logging settings. +assert(checkLog.checkContainsWithCountJson(conn, 22432, undefined, 0)); MongoRunner.stopMongod(conn); /** - * Test 6. Restart as a standalone and skip table logging checks on startup. Verify that restarting - * as a replica set again does not require any table logging modifications. + * Test 4. Back to standalone. Check that the table log settings are enabled. Write the + * '_wt_table_checks' file and check that it gets removed. */ -jsTest.log("Test 6."); - -conn = startMongodOnExistingPath(dbpath, { - setParameter: { - wiredTigerSkipTableLoggingChecksOnStartup: true, - logComponentVerbosity: tojson({verbosity: 1}) - } -}); - -// Skipping table logging checks for all existing tables. -checkLog.containsJson(conn, 5548301, {wiredTigerSkipTableLoggingChecksOnStartup: true}); - -// Log level 1 prints each individual table it skips table logging checks for. -checkLog.containsJson(conn, 5548302); - -MongoRunner.stopMongod(conn); +jsTest.log("Test 4."); +writeFile(dbpath + "/_wt_table_checks", ""); +conn = startMongodOnExistingPath(dbpath, + {setParameter: {logComponentVerbosity: tojson({verbosity: 1})}}); +checkTableChecksFileRemoved(dbpath); -conn = startMongodOnExistingPath(dbpath, {replSet: "mySet"}); +// Changing table logging settings. +checkLog.containsJson(conn, 22432); -// No table logging settings modifications are required. -checkLog.containsJson(conn, 4366408); +// Skipping table logging checks. +assert(checkLog.checkContainsWithCountJson(conn, 5548302, undefined, 0)); +checkTableLogSettings(conn, /*enabled=*/true); MongoRunner.stopMongod(conn); -}()); +}());
\ No newline at end of file diff --git a/jstests/disk/wt_table_checks_read_only.js b/jstests/disk/wt_table_checks_read_only.js deleted file mode 100644 index 6d6519f2c85..00000000000 --- a/jstests/disk/wt_table_checks_read_only.js +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Tests that the table logging settings are not changed during read only mode. - * - * @tags: [requires_wiredtiger] - */ -(function() { - -load('jstests/disk/libs/wt_file_helper.js'); - -// Create a bunch of collections under various database names. -let conn = MongoRunner.runMongod({}); -const dbpath = conn.dbpath; - -for (let i = 0; i < 10; i++) { - assert.commandWorked(conn.getDB(i.toString()).createCollection(i.toString())); -} - -MongoRunner.stopMongod(conn); - -// Option for read only mode. -let options = {queryableBackupMode: ""}; - -// Verifies that setTableLogging() does not get called in read only mode, otherwise the invariant -// would fire. -conn = startMongodOnExistingPath(dbpath, options); -assert(conn); -MongoRunner.stopMongod(conn); - -// Create the '_wt_table_checks' file in the dbpath and ensure it doesn't get removed while in read -// only mode. -let files = listFiles(dbpath); -for (f in files) { - assert(!files[f].name.includes("_wt_table_checks")); -} - -writeFile(dbpath + "/_wt_table_checks", ""); - -conn = startMongodOnExistingPath(dbpath, options); -assert(conn); -MongoRunner.stopMongod(conn); - -let hasWTTableChecksFile = false; -files = listFiles(dbpath); -for (f in files) { - if (files[f].name.includes("_wt_table_checks")) { - hasWTTableChecksFile = true; - } -} - -assert(hasWTTableChecksFile); -}()); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index 63e679ac7ec..821149484f5 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -704,7 +704,6 @@ void WiredTigerKVEngine::_openWiredTiger(const std::string& path, const std::str void WiredTigerKVEngine::cleanShutdown() { LOGV2(22317, "WiredTigerKVEngine shutting down"); - WiredTigerUtil::resetTableLoggingInfo(); if (!_conn) { return; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_standard_index_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_standard_index_test.cpp index b050a4a2f88..f3b2f963b53 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_standard_index_test.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_standard_index_test.cpp @@ -62,15 +62,11 @@ public: _fastClockSource = std::make_unique<SystemClockSource>(); _sessionCache = new WiredTigerSessionCache(_conn, _fastClockSource.get()); - - WiredTigerUtil::notifyStartupComplete(); } ~WiredTigerIndexHarnessHelper() final { delete _sessionCache; _conn->close(_conn, nullptr); - - WiredTigerUtil::resetTableLoggingInfo(); } std::unique_ptr<SortedDataInterface> newIdIndexSortedDataInterface() final { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp index 9fcdd78858f..8615a9a9def 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp @@ -49,55 +49,17 @@ #include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h" #include "mongo/logv2/log.h" #include "mongo/util/assert_util.h" -#include "mongo/util/fail_point.h" #include "mongo/util/processinfo.h" #include "mongo/util/scopeguard.h" #include "mongo/util/str.h" namespace mongo { -MONGO_FAIL_POINT_DEFINE(crashAfterUpdatingFirstTableLoggingSettings); - namespace { const std::string kTableChecksFileName = "_wt_table_checks"; /** - * Returns true if the 'kTableChecksFileName' file exists in the dbpath. - * - * Must be called before createTableChecksFile() or removeTableChecksFile() to get accurate results. - */ -bool hasPreviouslyIncompleteTableChecks() { - auto path = boost::filesystem::path(storageGlobalParams.dbpath) / - boost::filesystem::path(kTableChecksFileName); - - return boost::filesystem::exists(path); -} - -/** - * Creates the 'kTableChecksFileName' file in the dbpath. - */ -void createTableChecksFile() { - auto path = boost::filesystem::path(storageGlobalParams.dbpath) / - boost::filesystem::path(kTableChecksFileName); - - boost::filesystem::ofstream fileStream(path); - fileStream << "This file indicates that a WiredTiger table check operation is in progress or " - "incomplete." - << std::endl; - if (fileStream.fail()) { - LOGV2_FATAL_NOTRACE(4366400, - "Failed to write to file", - "file"_attr = path.generic_string(), - "error"_attr = errnoWithDescription()); - } - fileStream.close(); - - fassertNoTrace(4366401, fsyncFile(path)); - fassertNoTrace(4366402, fsyncParentDirectory(path)); -} - -/** * Removes the 'kTableChecksFileName' file in the dbpath, if it exists. */ void removeTableChecksFile() { @@ -123,10 +85,6 @@ void removeTableChecksFile() { using std::string; -Mutex WiredTigerUtil::_tableLoggingInfoMutex = - MONGO_MAKE_LATCH("WiredTigerUtil::_tableLoggingInfoMutex"); -WiredTigerUtil::TableLoggingInfo WiredTigerUtil::_tableLoggingInfo; - Status wtRCToStatus_slow(int retCode, const char* prefix) { if (retCode == 0) return Status::OK(); @@ -659,20 +617,7 @@ int WiredTigerUtil::verifyTable(OperationContext* opCtx, } void WiredTigerUtil::notifyStartupComplete() { - { - stdx::lock_guard<Latch> lk(_tableLoggingInfoMutex); - invariant(_tableLoggingInfo.isInitializing); - _tableLoggingInfo.isInitializing = false; - } - - if (!storageGlobalParams.readOnly) { - removeTableChecksFile(); - } -} - -void WiredTigerUtil::resetTableLoggingInfo() { - stdx::lock_guard<Latch> lk(_tableLoggingInfoMutex); - _tableLoggingInfo = TableLoggingInfo(); + removeTableChecksFile(); } bool WiredTigerUtil::useTableLogging(NamespaceString ns, bool replEnabled) { @@ -710,112 +655,11 @@ Status WiredTigerUtil::setTableLogging(OperationContext* opCtx, const std::strin Status WiredTigerUtil::setTableLogging(WT_SESSION* session, const std::string& uri, bool on) { invariant(!storageGlobalParams.readOnly); - stdx::lock_guard<Latch> lk(_tableLoggingInfoMutex); - - // Update the table logging settings regardless if we're no longer starting up the process. - if (!_tableLoggingInfo.isInitializing) { - return _setTableLogging(session, uri, on); - } - - // During the start up process, the table logging settings are checked for each table to verify - // that they are set appropriately. We can speed this process up by assuming that the logging - // setting is identical for each table. - // We cross reference the logging settings for the first table and if it isn't correctly set, we - // change the logging settings for all tables during start up. - // In the event that the server wasn't shutdown cleanly, the logging settings will be modified - // for all tables as a safety precaution, or if repair mode is running. - if (_tableLoggingInfo.isFirstTable && hasPreviouslyIncompleteTableChecks()) { - _tableLoggingInfo.hasPreviouslyIncompleteTableChecks = true; - } if (gWiredTigerSkipTableLoggingChecksOnStartup) { - if (_tableLoggingInfo.hasPreviouslyIncompleteTableChecks) { - LOGV2_FATAL_NOTRACE( - 5548300, - "Cannot use the 'wiredTigerSkipTableLoggingChecksOnStartup' startup parameter when " - "there are previously incomplete table checks"); - } - - // Only log this warning once. - if (_tableLoggingInfo.isFirstTable) { - _tableLoggingInfo.isFirstTable = false; - LOGV2_WARNING_OPTIONS( - 5548301, - {logv2::LogTag::kStartupWarnings}, - "Skipping table logging checks for all existing WiredTiger tables on startup", - "wiredTigerSkipTableLoggingChecksOnStartup"_attr = - gWiredTigerSkipTableLoggingChecksOnStartup); - } - LOGV2_DEBUG(5548302, 1, "Skipping table logging check", "uri"_attr = uri); return Status::OK(); } - - if (storageGlobalParams.repair || _tableLoggingInfo.hasPreviouslyIncompleteTableChecks) { - if (_tableLoggingInfo.isFirstTable) { - _tableLoggingInfo.isFirstTable = false; - if (!_tableLoggingInfo.hasPreviouslyIncompleteTableChecks) { - createTableChecksFile(); - } - - LOGV2(4366405, - "Modifying the table logging settings for all existing WiredTiger tables", - "loggingEnabled"_attr = on, - "repair"_attr = storageGlobalParams.repair, - "hasPreviouslyIncompleteTableChecks"_attr = - _tableLoggingInfo.hasPreviouslyIncompleteTableChecks); - } - - return _setTableLogging(session, uri, on); - } - - if (!_tableLoggingInfo.isFirstTable) { - if (_tableLoggingInfo.changeTableLogging) { - return _setTableLogging(session, uri, on); - } - - // The table logging settings do not need to be modified. - return Status::OK(); - } - - invariant(_tableLoggingInfo.isFirstTable); - invariant(!_tableLoggingInfo.hasPreviouslyIncompleteTableChecks); - - // When repair or a forced modification to the table logging settings isn't running, check that - // the first table is the catalog. - invariant(uri == "table:_mdb_catalog", str::stream() << "First table checked was: " << uri); - _tableLoggingInfo.isFirstTable = false; - - // Check if the first tables logging settings need to be modified. - const std::string setting = on ? "log=(enabled=true)" : "log=(enabled=false)"; - const std::string existingMetadata = getMetadataCreate(session, uri).getValue(); - if (existingMetadata.find(setting) != std::string::npos) { - // The table is running with the expected logging settings. - LOGV2(4366408, - "No table logging settings modifications are required for existing WiredTiger tables", - "loggingEnabled"_attr = on); - return Status::OK(); - } - - // The first table is running with the incorrect logging settings. All tables will need to have - // their logging settings modified. - _tableLoggingInfo.changeTableLogging = true; - createTableChecksFile(); - - LOGV2(4366406, - "Modifying the table logging settings for all existing WiredTiger tables", - "loggingEnabled"_attr = on); - - Status status = _setTableLogging(session, uri, on); - - if (MONGO_unlikely(crashAfterUpdatingFirstTableLoggingSettings.shouldFail())) { - LOGV2_FATAL_NOTRACE( - 4366407, "Crashing due to 'crashAfterUpdatingFirstTableLoggingSettings' fail point"); - } - return status; -} - -Status WiredTigerUtil::_setTableLogging(WT_SESSION* session, const std::string& uri, bool on) { const std::string setting = on ? "log=(enabled=true)" : "log=(enabled=false)"; // This method does some "weak" parsing to see if the table is in the expected logging diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h index c7097e8b508..04ab25365e6 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h @@ -285,8 +285,6 @@ public: static void notifyStartupComplete(); - static void resetTableLoggingInfo(); - static bool useTableLogging(NamespaceString ns, bool replEnabled); static Status setTableLogging(OperationContext* opCtx, const std::string& uri, bool on); @@ -307,18 +305,6 @@ private: */ template <typename T> static T _castStatisticsValue(uint64_t statisticsValue, T maximumResultType); - - static Status _setTableLogging(WT_SESSION* session, const std::string& uri, bool on); - - // Used to keep track of the table logging setting modifications during start up. The mutex must - // be held prior to accessing any of the member variables in the struct. - static Mutex _tableLoggingInfoMutex; - static struct TableLoggingInfo { - bool isInitializing = true; - bool isFirstTable = true; - bool changeTableLogging = false; - bool hasPreviouslyIncompleteTableChecks = false; - } _tableLoggingInfo; }; class WiredTigerConfigParser { |