SERVER-68925 Reintroduce check table logging settings at startupr4.4.17-rc0

(cherry picked from commit 994562a9c82fd89549ac8ee0861eb9f10509d32c)

5 files changed, 82 insertions, 301 deletions

diff --git a/jstests/disk/wt_table_checks.js b/jstests/disk/wt_table_checks.js
index 6d0cf003aab..2222a808ab5 100644
--- a/jstests/disk/wt_table_checks.js
+++ b/jstests/disk/wt_table_checks.js
@@ -8,6 +8,43 @@
 
 load('jstests/disk/libs/wt_file_helper.js');
 
+function checkTableLogSettings(conn, enabled) {
+    conn.getDBNames().forEach(function(d) {
+        let collNames =
+            conn.getDB(d)
+                .runCommand({listCollections: 1, nameOnly: true, filter: {type: "collection"}})
+                .cursor.firstBatch;
+
+        collNames.forEach(function(c) {
+            let stats = conn.getDB(d).runCommand({collStats: c.name});
+
+            let logStr = "log=(enabled=" + (enabled ? "true" : "false") + ")";
+            if (d == "local") {
+                if (c.name == "replset.minvalid" && !enabled) {
+                    // This collection is never logged in a replica set.
+                    logStr = "log=(enabled=false)";
+                } else {
+                    // All other collections and indexes in the 'local' database have table
+                    // logging enabled always.
+                    logStr = "log=(enabled=true)";
+                }
+            }
+
+            assert.eq(true, stats.wiredTiger.creationString.includes(logStr));
+            Object.keys(stats.indexDetails).forEach(function(i) {
+                assert.eq(true, stats.indexDetails[i].creationString.includes(logStr));
+            });
+        });
+    });
+}
+
+function checkTableChecksFileRemoved(dbpath) {
+    let files = listFiles(dbpath);
+    for (file of files) {
+        assert.eq(false, file.name.includes("_wt_table_checks"));
+    }
+}
+
 // Create a bunch of collections under various database names.
 let conn = MongoRunner.runMongod({});
 const dbpath = conn.dbpath;
@@ -16,114 +53,75 @@ for (let i = 0; i < 10; i++) {
     assert.commandWorked(conn.getDB(i.toString()).createCollection(i.toString()));
 }
 
+checkTableLogSettings(conn, /*enabled=*/true);
 MongoRunner.stopMongod(conn);
 
 /**
- * Test 1. The regular case, where no table logging setting modifications are needed.
+ * Test 1. Change into a single node replica set, which requires all of the table logging settings
+ * to be updated. Write the '_wt_table_checks' file and check that it gets removed.
  */
 jsTest.log("Test 1.");
 
-conn = startMongodOnExistingPath(dbpath, {});
-checkLog.containsJson(conn, 4366408, {loggingEnabled: true});
-MongoRunner.stopMongod(conn);
-
-/**
- * Test 2. Repair checks all of the table logging settings.
- */
-jsTest.log("Test 2.");
-
-assertRepairSucceeds(dbpath, conn.port, {});
-
-// Cannot use checkLog here as the server is no longer running.
-let logContents = rawMongoProgramOutput();
-assert(logContents.indexOf(
-           "Modifying the table logging settings for all existing WiredTiger tables") > 0);
-
-/**
- * Test 3. Explicitly create the '_wt_table_checks' file to force all of the table logging setting
- * modifications to be made.
- */
-jsTest.log("Test 3.");
-
-let files = listFiles(dbpath);
-for (f in files) {
-    assert(!files[f].name.includes("_wt_table_checks"));
-}
-
 writeFile(dbpath + "/_wt_table_checks", "");
+conn = startMongodOnExistingPath(
+    dbpath, {replSet: "mySet", setParameter: {logComponentVerbosity: tojson({verbosity: 1})}});
+checkTableChecksFileRemoved(dbpath);
 
-// Cannot skip table logging checks on startup when there are previously incomplete table checks.
-conn = startMongodOnExistingPath(dbpath,
-                                 {setParameter: "wiredTigerSkipTableLoggingChecksOnStartup=true"});
-assert.eq(conn, null);
-
-conn = startMongodOnExistingPath(dbpath, {});
-checkLog.containsJson(
-    conn, 4366405, {loggingEnabled: true, repair: false, hasPreviouslyIncompleteTableChecks: true});
+// Changing table logging settings.
+checkLog.containsJson(conn, 22432);
 MongoRunner.stopMongod(conn);
 
 /**
- * Test 4. Change into a single replica set, which requires all of the table logging settings to be
- * updated. But simulate an interruption/crash while starting up during the table logging check
- * phase.
- *
- * The next start up will detect an unclean shutdown causing all of the table logging settings to be
- * updated.
+ * Test 2. Restart in standalone mode with wiredTigerSkipTableLoggingChecksOnStartup. No table log
+ * settings are updated.  Write the '_wt_table_checks' file and check that it gets removed.
  */
-jsTest.log("Test 4.");
-
+jsTest.log("Test 2.");
+writeFile(dbpath + "/_wt_table_checks", "");
 conn = startMongodOnExistingPath(dbpath, {
-    replSet: "mySet",
-    setParameter:
-        "failpoint.crashAfterUpdatingFirstTableLoggingSettings=" + tojson({"mode": "alwaysOn"})
+    setParameter: {
+        wiredTigerSkipTableLoggingChecksOnStartup: true,
+        logComponentVerbosity: tojson({verbosity: 1})
+    }
 });
-assert(!conn);
+checkTableChecksFileRemoved(dbpath);
 
-// Cannot use checkLog here as the server is no longer running.
-logContents = rawMongoProgramOutput();
-assert(logContents.indexOf(
-           "Crashing due to 'crashAfterUpdatingFirstTableLoggingSettings' fail point") > 0);
+// Skipping table logging checks.
+checkLog.containsJson(conn, 5548302);
 
-// The '_wt_table_checks' still exists, so all table logging settings should be modified.
-conn = startMongodOnExistingPath(dbpath, {});
-checkLog.containsJson(
-    conn, 4366405, {loggingEnabled: true, repair: false, hasPreviouslyIncompleteTableChecks: true});
+// Changing table logging settings.
+assert(checkLog.checkContainsWithCountJson(conn, 22432, undefined, 0));
+checkTableLogSettings(conn, /*enabled=*/false);
 MongoRunner.stopMongod(conn);
 
 /**
- * Test 5. Change into a single node replica set, which requires all of the table logging settings
- * to be updated as the node was successfully started up as a standalone the last time.
+ * Test 3. Change into a single node replica set again. Table log settings are checked but none are
+ * changed.  Write the '_wt_table_checks' file and check that it gets removed.
  */
-jsTest.log("Test 5.");
+jsTestLog("Test 3.");
+writeFile(dbpath + "/_wt_table_checks", "");
+conn = startMongodOnExistingPath(
+    dbpath, {replSet: "mySet", setParameter: {logComponentVerbosity: tojson({verbosity: 1})}});
+checkTableChecksFileRemoved(dbpath);
 
-conn = startMongodOnExistingPath(dbpath, {replSet: "mySet"});
-checkLog.containsJson(conn, 4366406, {loggingEnabled: false});
+// Changing table logging settings.
+assert(checkLog.checkContainsWithCountJson(conn, 22432, undefined, 0));
 MongoRunner.stopMongod(conn);
 
 /**
- * Test 6. Restart as a standalone and skip table logging checks on startup. Verify that restarting
- * as a replica set again does not require any table logging modifications.
+ * Test 4. Back to standalone. Check that the table log settings are enabled. Write the
+ * '_wt_table_checks' file and check that it gets removed.
  */
-jsTest.log("Test 6.");
-
-conn = startMongodOnExistingPath(dbpath, {
-    setParameter: {
-        wiredTigerSkipTableLoggingChecksOnStartup: true,
-        logComponentVerbosity: tojson({verbosity: 1})
-    }
-});
-
-// Skipping table logging checks for all existing tables.
-checkLog.containsJson(conn, 5548301, {wiredTigerSkipTableLoggingChecksOnStartup: true});
-
-// Log level 1 prints each individual table it skips table logging checks for.
-checkLog.containsJson(conn, 5548302);
-
-MongoRunner.stopMongod(conn);
+jsTest.log("Test 4.");
+writeFile(dbpath + "/_wt_table_checks", "");
+conn = startMongodOnExistingPath(dbpath,
+                                 {setParameter: {logComponentVerbosity: tojson({verbosity: 1})}});
+checkTableChecksFileRemoved(dbpath);
 
-conn = startMongodOnExistingPath(dbpath, {replSet: "mySet"});
+// Changing table logging settings.
+checkLog.containsJson(conn, 22432);
 
-// No table logging settings modifications are required.
-checkLog.containsJson(conn, 4366408);
+// Skipping table logging checks.
+assert(checkLog.checkContainsWithCountJson(conn, 5548302, undefined, 0));
+checkTableLogSettings(conn, /*enabled=*/true);
 MongoRunner.stopMongod(conn);
-}());
+}());
\ No newline at end of file
diff --git a/jstests/disk/wt_table_checks_read_only.js b/jstests/disk/wt_table_checks_read_only.js
deleted file mode 100644
index 6d6519f2c85..00000000000
--- a/jstests/disk/wt_table_checks_read_only.js
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Tests that the table logging settings are not changed during read only mode.
- *
- * @tags: [requires_wiredtiger]
- */
-(function() {
-
-load('jstests/disk/libs/wt_file_helper.js');
-
-// Create a bunch of collections under various database names.
-let conn = MongoRunner.runMongod({});
-const dbpath = conn.dbpath;
-
-for (let i = 0; i < 10; i++) {
-    assert.commandWorked(conn.getDB(i.toString()).createCollection(i.toString()));
-}
-
-MongoRunner.stopMongod(conn);
-
-// Option for read only mode.
-let options = {queryableBackupMode: ""};
-
-// Verifies that setTableLogging() does not get called in read only mode, otherwise the invariant
-// would fire.
-conn = startMongodOnExistingPath(dbpath, options);
-assert(conn);
-MongoRunner.stopMongod(conn);
-
-// Create the '_wt_table_checks' file in the dbpath and ensure it doesn't get removed while in read
-// only mode.
-let files = listFiles(dbpath);
-for (f in files) {
-    assert(!files[f].name.includes("_wt_table_checks"));
-}
-
-writeFile(dbpath + "/_wt_table_checks", "");
-
-conn = startMongodOnExistingPath(dbpath, options);
-assert(conn);
-MongoRunner.stopMongod(conn);
-
-let hasWTTableChecksFile = false;
-files = listFiles(dbpath);
-for (f in files) {
-    if (files[f].name.includes("_wt_table_checks")) {
-        hasWTTableChecksFile = true;
-    }
-}
-
-assert(hasWTTableChecksFile);
-}());
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index eaf1ea32e06..9c62481e373 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -975,7 +975,6 @@ void WiredTigerKVEngine::_openWiredTiger(const std::string& path, const std::str
 
 void WiredTigerKVEngine::cleanShutdown() {
     LOGV2(22317, "WiredTigerKVEngine shutting down");
-    WiredTigerUtil::resetTableLoggingInfo();
 
     if (!_conn) {
         return;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
index 85121941458..9b954ebf4ea 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
@@ -49,55 +49,17 @@
 #include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h"
 #include "mongo/logv2/log.h"
 #include "mongo/util/assert_util.h"
-#include "mongo/util/fail_point.h"
 #include "mongo/util/processinfo.h"
 #include "mongo/util/scopeguard.h"
 #include "mongo/util/str.h"
 
 namespace mongo {
 
-MONGO_FAIL_POINT_DEFINE(crashAfterUpdatingFirstTableLoggingSettings);
-
 namespace {
 
 const std::string kTableChecksFileName = "_wt_table_checks";
 
 /**
- * Returns true if the 'kTableChecksFileName' file exists in the dbpath.
- *
- * Must be called before createTableChecksFile() or removeTableChecksFile() to get accurate results.
- */
-bool hasPreviouslyIncompleteTableChecks() {
-    auto path = boost::filesystem::path(storageGlobalParams.dbpath) /
-        boost::filesystem::path(kTableChecksFileName);
-
-    return boost::filesystem::exists(path);
-}
-
-/**
- * Creates the 'kTableChecksFileName' file in the dbpath.
- */
-void createTableChecksFile() {
-    auto path = boost::filesystem::path(storageGlobalParams.dbpath) /
-        boost::filesystem::path(kTableChecksFileName);
-
-    boost::filesystem::ofstream fileStream(path);
-    fileStream << "This file indicates that a WiredTiger table check operation is in progress or "
-                  "incomplete."
-               << std::endl;
-    if (fileStream.fail()) {
-        LOGV2_FATAL_NOTRACE(4366400,
-                            "Failed to write to file",
-                            "file"_attr = path.generic_string(),
-                            "error"_attr = errnoWithDescription());
-    }
-    fileStream.close();
-
-    fassertNoTrace(4366401, fsyncFile(path));
-    fassertNoTrace(4366402, fsyncParentDirectory(path));
-}
-
-/**
  * Removes the 'kTableChecksFileName' file in the dbpath, if it exists.
  */
 void removeTableChecksFile() {
@@ -123,10 +85,6 @@ void removeTableChecksFile() {
 
 using std::string;
 
-Mutex WiredTigerUtil::_tableLoggingInfoMutex =
-    MONGO_MAKE_LATCH("WiredTigerUtil::_tableLoggingInfoMutex");
-WiredTigerUtil::TableLoggingInfo WiredTigerUtil::_tableLoggingInfo;
-
 Status wtRCToStatus_slow(int retCode, const char* prefix) {
     if (retCode == 0)
         return Status::OK();
@@ -642,20 +600,7 @@ int WiredTigerUtil::verifyTable(OperationContext* opCtx,
 }
 
 void WiredTigerUtil::notifyStartupComplete() {
-    {
-        stdx::lock_guard<Latch> lk(_tableLoggingInfoMutex);
-        invariant(_tableLoggingInfo.isInitializing);
-        _tableLoggingInfo.isInitializing = false;
-    }
-
-    if (!storageGlobalParams.readOnly) {
-        removeTableChecksFile();
-    }
-}
-
-void WiredTigerUtil::resetTableLoggingInfo() {
-    stdx::lock_guard<Latch> lk(_tableLoggingInfoMutex);
-    _tableLoggingInfo = TableLoggingInfo();
+    removeTableChecksFile();
 }
 
 bool WiredTigerUtil::useTableLogging(NamespaceString ns, bool replEnabled) {
@@ -693,108 +638,12 @@ Status WiredTigerUtil::setTableLogging(OperationContext* opCtx, const std::strin
 
 Status WiredTigerUtil::setTableLogging(WT_SESSION* session, const std::string& uri, bool on) {
     invariant(!storageGlobalParams.readOnly);
-    stdx::lock_guard<Latch> lk(_tableLoggingInfoMutex);
-
-    // Update the table logging settings regardless if we're no longer starting up the process.
-    if (!_tableLoggingInfo.isInitializing) {
-        return _setTableLogging(session, uri, on);
-    }
-
-    // During the start up process, the table logging settings are checked for each table to verify
-    // that they are set appropriately. We can speed this process up by assuming that the logging
-    // setting is identical for each table.
-    // We cross reference the logging settings for the first table and if it isn't correctly set, we
-    // change the logging settings for all tables during start up.
-    // In the event that the server wasn't shutdown cleanly, the logging settings will be modified
-    // for all tables as a safety precaution, or if repair mode is running.
-    if (_tableLoggingInfo.isFirstTable && hasPreviouslyIncompleteTableChecks()) {
-        _tableLoggingInfo.hasPreviouslyIncompleteTableChecks = true;
-    }
 
     if (gWiredTigerSkipTableLoggingChecksOnStartup) {
-        if (_tableLoggingInfo.hasPreviouslyIncompleteTableChecks) {
-            LOGV2_FATAL_NOTRACE(
-                5548300,
-                "Cannot use the 'wiredTigerSkipTableLoggingChecksOnStartup' startup parameter when "
-                "there are previously incomplete table checks");
-        }
-
-        // Only log this warning once.
-        if (_tableLoggingInfo.isFirstTable) {
-            _tableLoggingInfo.isFirstTable = false;
-            LOGV2_WARNING_OPTIONS(
-                5548301,
-                {logv2::LogTag::kStartupWarnings},
-                "Skipping table logging checks for all existing WiredTiger tables on startup",
-                "wiredTigerSkipTableLoggingChecksOnStartup"_attr =
-                    gWiredTigerSkipTableLoggingChecksOnStartup);
-        }
-
         LOGV2_DEBUG(5548302, 1, "Skipping table logging check", "uri"_attr = uri);
         return Status::OK();
     }
 
-    if (storageGlobalParams.repair || _tableLoggingInfo.hasPreviouslyIncompleteTableChecks) {
-        if (_tableLoggingInfo.isFirstTable) {
-            _tableLoggingInfo.isFirstTable = false;
-            if (!_tableLoggingInfo.hasPreviouslyIncompleteTableChecks) {
-                createTableChecksFile();
-            }
-
-            LOGV2(4366405,
-                  "Modifying the table logging settings for all existing WiredTiger tables",
-                  "loggingEnabled"_attr = on,
-                  "repair"_attr = storageGlobalParams.repair,
-                  "hasPreviouslyIncompleteTableChecks"_attr =
-                      _tableLoggingInfo.hasPreviouslyIncompleteTableChecks);
-        }
-
-        return _setTableLogging(session, uri, on);
-    }
-
-    if (!_tableLoggingInfo.isFirstTable) {
-        if (_tableLoggingInfo.changeTableLogging) {
-            return _setTableLogging(session, uri, on);
-        }
-
-        // The table logging settings do not need to be modified.
-        return Status::OK();
-    }
-
-    invariant(_tableLoggingInfo.isFirstTable);
-    invariant(!_tableLoggingInfo.hasPreviouslyIncompleteTableChecks);
-    _tableLoggingInfo.isFirstTable = false;
-
-    // Check if the first tables logging settings need to be modified.
-    const std::string setting = on ? "log=(enabled=true)" : "log=(enabled=false)";
-    const std::string existingMetadata = getMetadataCreate(session, uri).getValue();
-    if (existingMetadata.find(setting) != std::string::npos) {
-        // The table is running with the expected logging settings.
-        LOGV2(4366408,
-              "No table logging settings modifications are required for existing WiredTiger tables",
-              "loggingEnabled"_attr = on);
-        return Status::OK();
-    }
-
-    // The first table is running with the incorrect logging settings. All tables will need to have
-    // their logging settings modified.
-    _tableLoggingInfo.changeTableLogging = true;
-    createTableChecksFile();
-
-    LOGV2(4366406,
-          "Modifying the table logging settings for all existing WiredTiger tables",
-          "loggingEnabled"_attr = on);
-
-    Status status = _setTableLogging(session, uri, on);
-
-    if (MONGO_unlikely(crashAfterUpdatingFirstTableLoggingSettings.shouldFail())) {
-        LOGV2_FATAL_NOTRACE(
-            4366407, "Crashing due to 'crashAfterUpdatingFirstTableLoggingSettings' fail point");
-    }
-    return status;
-}
-
-Status WiredTigerUtil::_setTableLogging(WT_SESSION* session, const std::string& uri, bool on) {
     const std::string setting = on ? "log=(enabled=true)" : "log=(enabled=false)";
 
     // This method does some "weak" parsing to see if the table is in the expected logging
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
index cb3338abbb9..c6b5caf1068 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
@@ -258,8 +258,6 @@ public:
 
     static void notifyStartupComplete();
 
-    static void resetTableLoggingInfo();
-
     static bool useTableLogging(NamespaceString ns, bool replEnabled);
 
     static Status setTableLogging(OperationContext* opCtx, const std::string& uri, bool on);
@@ -280,18 +278,6 @@ private:
      */
     template <typename T>
     static T _castStatisticsValue(uint64_t statisticsValue, T maximumResultType);
-
-    static Status _setTableLogging(WT_SESSION* session, const std::string& uri, bool on);
-
-    // Used to keep track of the table logging setting modifications during start up. The mutex must
-    // be held prior to accessing any of the member variables in the struct.
-    static Mutex _tableLoggingInfoMutex;
-    static struct TableLoggingInfo {
-        bool isInitializing = true;
-        bool isFirstTable = true;
-        bool changeTableLogging = false;
-        bool hasPreviouslyIncompleteTableChecks = false;
-    } _tableLoggingInfo;
 };
 
 class WiredTigerConfigParser {