diff options
author | Gregory Wlodarek <gregory.wlodarek@mongodb.com> | 2022-08-26 08:49:02 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-08-26 23:08:22 +0000 |
commit | 8dd7f66d46cead919bbe13554c3543d16b16a385 (patch) | |
tree | 74d56d20b04e84b6a05a458b2f9444c4fec7536a | |
parent | eb3adbe5a061ecb3c7fb123f42241bbae03f88bd (diff) | |
download | mongo-8dd7f66d46cead919bbe13554c3543d16b16a385.tar.gz |
SERVER-68925 Reintroduce check table logging settings at startup
(cherry picked from commit 994562a9c82fd89549ac8ee0861eb9f10509d32c)
-rw-r--r-- | jstests/disk/wt_table_checks.js | 170 | ||||
-rw-r--r-- | jstests/disk/wt_table_checks_read_only.js | 51 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp | 1 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp | 144 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_util.h | 14 |
5 files changed, 73 insertions, 307 deletions
diff --git a/jstests/disk/wt_table_checks.js b/jstests/disk/wt_table_checks.js index 17c511a2828..4e1770f5c80 100644 --- a/jstests/disk/wt_table_checks.js +++ b/jstests/disk/wt_table_checks.js @@ -8,6 +8,43 @@ load('jstests/disk/libs/wt_file_helper.js'); +function checkTableLogSettings(conn, enabled) { + conn.getDBNames().forEach(function(d) { + let collNames = + conn.getDB(d) + .runCommand({listCollections: 1, nameOnly: true, filter: {type: "collection"}}) + .cursor.firstBatch; + + collNames.forEach(function(c) { + let stats = conn.getDB(d).runCommand({collStats: c.name}); + + let logStr = "log=(enabled=" + (enabled ? "true" : "false") + ")"; + if (d == "local") { + if (c.name == "replset.minvalid" && !enabled) { + // This collection is never logged in a replica set. + logStr = "log=(enabled=false)"; + } else { + // All other collections and indexes in the 'local' database have table + // logging enabled always. + logStr = "log=(enabled=true)"; + } + } + + assert.eq(true, stats.wiredTiger.creationString.includes(logStr)); + Object.keys(stats.indexDetails).forEach(function(i) { + assert.eq(true, stats.indexDetails[i].creationString.includes(logStr)); + }); + }); + }); +} + +function checkTableChecksFileRemoved(dbpath) { + let files = listFiles(dbpath); + for (file of files) { + assert.eq(false, file.name.includes("_wt_table_checks")); + } +} + // Create a bunch of collections under various database names. let conn = MongoRunner.runMongod({}); const dbpath = conn.dbpath; @@ -16,126 +53,63 @@ for (let i = 0; i < 10; i++) { assert.commandWorked(conn.getDB(i.toString()).createCollection(i.toString())); } +checkTableLogSettings(conn, /*enabled=*/true); MongoRunner.stopMongod(conn); /** - * Test 1. The regular case, where no table logging setting modifications are needed. + * Test 1. Change into a single node replica set, which requires all of the table logging settings + * to be updated. Write the '_wt_table_checks' file and check that it gets removed. */ jsTest.log("Test 1."); -conn = startMongodOnExistingPath(dbpath, {}); -checkLog.contains( - conn, - "No table logging settings modifications are required for existing WiredTiger tables. Logging enabled? 1"); -MongoRunner.stopMongod(conn); - -/** - * Test 2. Repair checks all of the table logging settings. - */ -jsTest.log("Test 2."); - -assertRepairSucceeds(dbpath, conn.port, {}); - -// Cannot use checkLog here as the server is no longer running. -let logContents = rawMongoProgramOutput(); -assert( - logContents.indexOf( - "Modifying the table logging settings to 1 for all existing WiredTiger tables. Repair? 1, has previously incomplete table checks? 0") > - 0); - -/** - * Test 3. Explicitly create the '_wt_table_checks' file to force all of the table logging setting - * modifications to be made. - */ -jsTest.log("Test 3."); - -let files = listFiles(dbpath); -for (f in files) { - assert(!files[f].name.includes("_wt_table_checks")); -} - writeFile(dbpath + "/_wt_table_checks", ""); +conn = startMongodOnExistingPath( + dbpath, {replSet: "mySet", setParameter: {logComponentVerbosity: tojson({verbosity: 1})}}); +checkTableChecksFileRemoved(dbpath); -// Cannot skip table logging checks on startup when there are previously incomplete table checks. -conn = startMongodOnExistingPath(dbpath, - {setParameter: "wiredTigerSkipTableLoggingChecksOnStartup=true"}); -assert.eq(conn, null); - -conn = startMongodOnExistingPath(dbpath, {}); -checkLog.contains( - conn, - "Modifying the table logging settings to 1 for all existing WiredTiger tables. Repair? 0, has previously incomplete table checks? 1"); +checkLog.contains(conn, "Changing table logging settings"); MongoRunner.stopMongod(conn); /** - * Test 4. Change into a single replica set, which requires all of the table logging settings to be - * updated. But simulate an interruption/crash while starting up during the table logging check - * phase. - * - * The next start up will detect an unclean shutdown causing all of the table logging settings to be - * updated. + * Test 2. Restart in standalone mode with wiredTigerSkipTableLoggingChecksOnStartup. No table log + * settings are updated. Write the '_wt_table_checks' file and check that it gets removed. */ -jsTest.log("Test 4."); - -conn = startMongodOnExistingPath(dbpath, { - replSet: "mySet", - setParameter: - "failpoint.crashAfterUpdatingFirstTableLoggingSettings=" + tojson({"mode": "alwaysOn"}) -}); -assert(!conn); - -// Cannot use checkLog here as the server is no longer running. -logContents = rawMongoProgramOutput(); -assert(logContents.indexOf( - "Crashing due to 'crashAfterUpdatingFirstTableLoggingSettings' fail point") > 0); - -// The '_wt_table_checks' still exists, so all table logging settings should be modified. -conn = startMongodOnExistingPath(dbpath, {}); -checkLog.contains( - conn, - "Modifying the table logging settings to 1 for all existing WiredTiger tables. Repair? 0, has previously incomplete table checks? 1"); -MongoRunner.stopMongod(conn); - -/** - * Test 5. Change into a single node replica set, which requires all of the table logging settings - * to be updated as the node was successfully started up as a standalone the last time. - */ -jsTest.log("Test 5."); - -conn = startMongodOnExistingPath(dbpath, {replSet: "mySet"}); -checkLog.contains( - conn, - "Modifying the table logging settings for all existing WiredTiger tables. Logging enabled? 0"); -MongoRunner.stopMongod(conn); - -/** - * Test 6. Restart as a standalone and skip table logging checks on startup. Verify that restarting - * as a replica set again does not require any table logging modifications. - */ -jsTest.log("Test 6."); - +jsTest.log("Test 2."); +writeFile(dbpath + "/_wt_table_checks", ""); conn = startMongodOnExistingPath(dbpath, { setParameter: { wiredTigerSkipTableLoggingChecksOnStartup: true, logComponentVerbosity: tojson({verbosity: 1}) } }); +checkTableChecksFileRemoved(dbpath); -// Skipping table logging checks for all existing tables. -checkLog.contains(conn, - "Skipping table logging checks for all existing WiredTiger" + - " tables on startup. wiredTigerSkipTableLoggingChecksOnStartup=1"); - -// Log level 1 prints each individual table it skips table logging checks for. checkLog.contains(conn, "Skipping table logging check"); +checkTableLogSettings(conn, /*enabled=*/false); +MongoRunner.stopMongod(conn); +/** + * Test 3. Change into a single node replica set again. Table log settings are checked but none are + * changed. Write the '_wt_table_checks' file and check that it gets removed. + */ +jsTestLog("Test 3."); +writeFile(dbpath + "/_wt_table_checks", ""); +conn = startMongodOnExistingPath( + dbpath, {replSet: "mySet", setParameter: {logComponentVerbosity: tojson({verbosity: 1})}}); +checkTableChecksFileRemoved(dbpath); MongoRunner.stopMongod(conn); -conn = startMongodOnExistingPath(dbpath, {replSet: "mySet"}); +/** + * Test 4. Back to standalone. Check that the table log settings are enabled. Write the + * '_wt_table_checks' file and check that it gets removed. + */ +jsTest.log("Test 4."); +writeFile(dbpath + "/_wt_table_checks", ""); +conn = startMongodOnExistingPath(dbpath, + {setParameter: {logComponentVerbosity: tojson({verbosity: 1})}}); +checkTableChecksFileRemoved(dbpath); -// No table logging settings modifications are required. -checkLog.contains(conn, - "No table logging settings modifications" + - " are required for existing WiredTiger tables"); +checkLog.contains(conn, "Changing table logging settings"); +checkTableLogSettings(conn, /*enabled=*/true); MongoRunner.stopMongod(conn); -}()); +}());
\ No newline at end of file diff --git a/jstests/disk/wt_table_checks_read_only.js b/jstests/disk/wt_table_checks_read_only.js deleted file mode 100644 index 6d6519f2c85..00000000000 --- a/jstests/disk/wt_table_checks_read_only.js +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Tests that the table logging settings are not changed during read only mode. - * - * @tags: [requires_wiredtiger] - */ -(function() { - -load('jstests/disk/libs/wt_file_helper.js'); - -// Create a bunch of collections under various database names. -let conn = MongoRunner.runMongod({}); -const dbpath = conn.dbpath; - -for (let i = 0; i < 10; i++) { - assert.commandWorked(conn.getDB(i.toString()).createCollection(i.toString())); -} - -MongoRunner.stopMongod(conn); - -// Option for read only mode. -let options = {queryableBackupMode: ""}; - -// Verifies that setTableLogging() does not get called in read only mode, otherwise the invariant -// would fire. -conn = startMongodOnExistingPath(dbpath, options); -assert(conn); -MongoRunner.stopMongod(conn); - -// Create the '_wt_table_checks' file in the dbpath and ensure it doesn't get removed while in read -// only mode. -let files = listFiles(dbpath); -for (f in files) { - assert(!files[f].name.includes("_wt_table_checks")); -} - -writeFile(dbpath + "/_wt_table_checks", ""); - -conn = startMongodOnExistingPath(dbpath, options); -assert(conn); -MongoRunner.stopMongod(conn); - -let hasWTTableChecksFile = false; -files = listFiles(dbpath); -for (f in files) { - if (files[f].name.includes("_wt_table_checks")) { - hasWTTableChecksFile = true; - } -} - -assert(hasWTTableChecksFile); -}()); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index 2e3f0768003..ec2f78429c9 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -945,7 +945,6 @@ void WiredTigerKVEngine::_openWiredTiger(const std::string& path, const std::str void WiredTigerKVEngine::cleanShutdown() { log() << "WiredTigerKVEngine shutting down"; - WiredTigerUtil::resetTableLoggingInfo(); if (!_readOnly) syncSizeInfo(true); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp index fcebf78d238..799621ccd00 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp @@ -56,47 +56,11 @@ namespace mongo { -MONGO_FAIL_POINT_DEFINE(crashAfterUpdatingFirstTableLoggingSettings); - namespace { const std::string kTableChecksFileName = "_wt_table_checks"; /** - * Returns true if the 'kTableChecksFileName' file exists in the dbpath. - * - * Must be called before createTableChecksFile() or removeTableChecksFile() to get accurate results. - */ -bool hasPreviouslyIncompleteTableChecks() { - auto path = boost::filesystem::path(storageGlobalParams.dbpath) / - boost::filesystem::path(kTableChecksFileName); - - return boost::filesystem::exists(path); -} - -/** - * Creates the 'kTableChecksFileName' file in the dbpath. - */ -void createTableChecksFile() { - auto path = boost::filesystem::path(storageGlobalParams.dbpath) / - boost::filesystem::path(kTableChecksFileName); - - boost::filesystem::ofstream fileStream(path); - fileStream << "This file indicates that a WiredTiger table check operation is in progress or " - "incomplete." - << std::endl; - if (fileStream.fail()) { - log() << "Failed to write to file " << path.generic_string() << " because " - << errnoWithDescription(); - fassertFailedNoTrace(4366400); - } - fileStream.close(); - - fassertNoTrace(4366401, fsyncFile(path)); - fassertNoTrace(4366402, fsyncParentDirectory(path)); -} - -/** * Removes the 'kTableChecksFileName' file in the dbpath, if it exists. */ void removeTableChecksFile() { @@ -121,10 +85,6 @@ void removeTableChecksFile() { using std::string; -Mutex WiredTigerUtil::_tableLoggingInfoMutex = - MONGO_MAKE_LATCH("WiredTigerUtil::_tableLoggingInfoMutex"); -WiredTigerUtil::TableLoggingInfo WiredTigerUtil::_tableLoggingInfo; - Status wtRCToStatus_slow(int retCode, const char* prefix) { if (retCode == 0) return Status::OK(); @@ -601,20 +561,7 @@ int WiredTigerUtil::verifyTable(OperationContext* opCtx, } void WiredTigerUtil::notifyStartupComplete() { - { - stdx::lock_guard<Latch> lk(_tableLoggingInfoMutex); - invariant(_tableLoggingInfo.isInitializing); - _tableLoggingInfo.isInitializing = false; - } - - if (!storageGlobalParams.readOnly) { - removeTableChecksFile(); - } -} - -void WiredTigerUtil::resetTableLoggingInfo() { - stdx::lock_guard<Latch> lk(_tableLoggingInfoMutex); - _tableLoggingInfo = TableLoggingInfo(); + removeTableChecksFile(); } bool WiredTigerUtil::useTableLogging(NamespaceString ns, bool replEnabled) { @@ -652,101 +599,12 @@ Status WiredTigerUtil::setTableLogging(OperationContext* opCtx, const std::strin Status WiredTigerUtil::setTableLogging(WT_SESSION* session, const std::string& uri, bool on) { invariant(!storageGlobalParams.readOnly); - stdx::lock_guard<Latch> lk(_tableLoggingInfoMutex); - - // Update the table logging settings regardless if we're no longer starting up the process. - if (!_tableLoggingInfo.isInitializing) { - return _setTableLogging(session, uri, on); - } - - // During the start up process, the table logging settings are checked for each table to verify - // that they are set appropriately. We can speed this process up by assuming that the logging - // setting is identical for each table. - // We cross reference the logging settings for the first table and if it isn't correctly set, we - // change the logging settings for all tables during start up. - // In the event that the server wasn't shutdown cleanly, the logging settings will be modified - // for all tables as a safety precaution, or if repair mode is running. - if (_tableLoggingInfo.isFirstTable && hasPreviouslyIncompleteTableChecks()) { - _tableLoggingInfo.hasPreviouslyIncompleteTableChecks = true; - } if (gWiredTigerSkipTableLoggingChecksOnStartup) { - if (_tableLoggingInfo.hasPreviouslyIncompleteTableChecks) { - log() << "Cannot use the 'wiredTigerSkipTableLoggingChecksOnStartup' startup parameter " - "when there are previously incomplete table checks"; - fassertFailedNoTrace(5548300); - } - - // Only log this warning once. - if (_tableLoggingInfo.isFirstTable) { - _tableLoggingInfo.isFirstTable = false; - log() << "Skipping table logging checks for all existing WiredTiger tables on startup. " - "wiredTigerSkipTableLoggingChecksOnStartup=" - << gWiredTigerSkipTableLoggingChecksOnStartup; - } - LOG(1) << "Skipping table logging check for " << uri; return Status::OK(); } - if (storageGlobalParams.repair || _tableLoggingInfo.hasPreviouslyIncompleteTableChecks) { - if (_tableLoggingInfo.isFirstTable) { - _tableLoggingInfo.isFirstTable = false; - if (!_tableLoggingInfo.hasPreviouslyIncompleteTableChecks) { - createTableChecksFile(); - } - - log() << "Modifying the table logging settings to " << on - << " for all existing WiredTiger tables. Repair? " << storageGlobalParams.repair - << ", has previously incomplete table checks? " - << _tableLoggingInfo.hasPreviouslyIncompleteTableChecks; - } - - return _setTableLogging(session, uri, on); - } - - if (!_tableLoggingInfo.isFirstTable) { - if (_tableLoggingInfo.changeTableLogging) { - return _setTableLogging(session, uri, on); - } - - // The table logging settings do not need to be modified. - return Status::OK(); - } - - invariant(_tableLoggingInfo.isFirstTable); - invariant(!_tableLoggingInfo.hasPreviouslyIncompleteTableChecks); - _tableLoggingInfo.isFirstTable = false; - - // Check if the first tables logging settings need to be modified. - const std::string setting = on ? "log=(enabled=true)" : "log=(enabled=false)"; - const std::string existingMetadata = getMetadataCreate(session, uri).getValue(); - if (existingMetadata.find(setting) != std::string::npos) { - // The table is running with the expected logging settings. - log() << "No table logging settings modifications are required for existing WiredTiger " - "tables." - << " Logging enabled? " << on; - return Status::OK(); - } - - // The first table is running with the incorrect logging settings. All tables will need to have - // their logging settings modified. - _tableLoggingInfo.changeTableLogging = true; - createTableChecksFile(); - - log() << "Modifying the table logging settings for all existing WiredTiger tables." - << " Logging enabled? " << on; - - Status status = _setTableLogging(session, uri, on); - - if (MONGO_unlikely(crashAfterUpdatingFirstTableLoggingSettings.shouldFail())) { - log() << "Crashing due to 'crashAfterUpdatingFirstTableLoggingSettings' fail point"; - fassertFailedNoTrace(4366407); - } - return status; -} - -Status WiredTigerUtil::_setTableLogging(WT_SESSION* session, const std::string& uri, bool on) { const std::string setting = on ? "log=(enabled=true)" : "log=(enabled=false)"; // This method does some "weak" parsing to see if the table is in the expected logging diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h index 72582ab7b89..9e68e596a5d 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h @@ -279,8 +279,6 @@ public: static void notifyStartupComplete(); - static void resetTableLoggingInfo(); - static bool useTableLogging(NamespaceString ns, bool replEnabled); static Status setTableLogging(OperationContext* opCtx, const std::string& uri, bool on); @@ -301,18 +299,6 @@ private: */ template <typename T> static T _castStatisticsValue(uint64_t statisticsValue, T maximumResultType); - - static Status _setTableLogging(WT_SESSION* session, const std::string& uri, bool on); - - // Used to keep track of the table logging setting modifications during start up. The mutex must - // be held prior to accessing any of the member variables in the struct. - static Mutex _tableLoggingInfoMutex; - static struct TableLoggingInfo { - bool isInitializing = true; - bool isFirstTable = true; - bool changeTableLogging = false; - bool hasPreviouslyIncompleteTableChecks = false; - } _tableLoggingInfo; }; class WiredTigerConfigParser { |