SERVER-43664 Speedup WiredTiger storage engine startup for many tables by optimizing WiredTigerUtil::setTableLogging()

(cherry picked from commit 8e64b07e3bb363347ee2c11a56aba873365ed74a)

16 files changed, 392 insertions, 5 deletions

diff --git a/jstests/disk/wt_table_checks.js b/jstests/disk/wt_table_checks.js
new file mode 100644
index 00000000000..a97344a19df
--- /dev/null
+++ b/jstests/disk/wt_table_checks.js
@@ -0,0 +1,105 @@
+/**
+ * Tests that MongoDB sets the WiredTiger table logging settings correctly under different
+ * circumstances.
+ *
+ * @tags: [requires_wiredtiger, requires_journaling]
+ */
+(function() {
+
+load('jstests/disk/libs/wt_file_helper.js');
+
+// Create a bunch of collections under various database names.
+let conn = MongoRunner.runMongod({});
+const dbpath = conn.dbpath;
+
+for (let i = 0; i < 10; i++) {
+    assert.commandWorked(conn.getDB(i.toString()).createCollection(i.toString()));
+}
+
+MongoRunner.stopMongod(conn);
+
+/**
+ * Test 1. The regular case, where no table logging setting modifications are needed.
+ */
+jsTest.log("Test 1.");
+
+conn = startMongodOnExistingPath(dbpath, {});
+checkLog.contains(
+    conn,
+    "No table logging settings modifications are required for existing WiredTiger tables. Logging enabled? 1");
+MongoRunner.stopMongod(conn);
+
+/**
+ * Test 2. Repair checks all of the table logging settings.
+ */
+jsTest.log("Test 2.");
+
+assertRepairSucceeds(dbpath, conn.port, {});
+
+// Cannot use checkLog here as the server is no longer running.
+let logContents = rawMongoProgramOutput();
+assert(
+    logContents.indexOf(
+        "Modifying the table logging settings to 1 for all existing WiredTiger tables. Repair? 1, has previously incomplete table checks? 0") >
+    0);
+
+/**
+ * Test 3. Explicitly create the '_wt_table_checks' file to force all of the table logging setting
+ * modifications to be made.
+ */
+jsTest.log("Test 3.");
+
+let files = listFiles(dbpath);
+for (f in files) {
+    assert(!files[f].name.includes("_wt_table_checks"));
+}
+
+writeFile(dbpath + "/_wt_table_checks", "");
+
+conn = startMongodOnExistingPath(dbpath, {});
+checkLog.contains(
+    conn,
+    "Modifying the table logging settings to 1 for all existing WiredTiger tables. Repair? 0, has previously incomplete table checks? 1");
+MongoRunner.stopMongod(conn);
+
+/**
+ * Test 4. Change into a single replica set, which requires all of the table logging settings to be
+ * updated. But simulate an interruption/crash while starting up during the table logging check
+ * phase.
+ *
+ * The next start up will detect an unclean shutdown causing all of the table logging settings to be
+ * updated.
+ */
+jsTest.log("Test 4.");
+
+conn = startMongodOnExistingPath(dbpath, {
+    replSet: "mySet",
+    setParameter:
+        "failpoint.crashAfterUpdatingFirstTableLoggingSettings=" + tojson({"mode": "alwaysOn"})
+});
+assert(!conn);
+
+// Cannot use checkLog here as the server is no longer running.
+logContents = rawMongoProgramOutput();
+assert(logContents.indexOf(
+           "Crashing due to 'crashAfterUpdatingFirstTableLoggingSettings' fail point") > 0);
+
+// The '_wt_table_checks' still exists, so all table logging settings should be modified.
+conn = startMongodOnExistingPath(dbpath, {});
+checkLog.contains(
+    conn,
+    "Modifying the table logging settings to 1 for all existing WiredTiger tables. Repair? 0, has previously incomplete table checks? 1");
+MongoRunner.stopMongod(conn);
+
+/**
+ * Test 5. Change into a single node replica set, which requires all of the table logging settings
+ * to be updated as the node was successfully started up as a standalone the last time.
+ */
+jsTest.log("Test 5.");
+
+conn = startMongodOnExistingPath(dbpath, {replSet: "mySet"});
+checkLog.contains(
+    conn,
+    "Modifying the table logging settings for all existing WiredTiger tables. Logging enabled? 0");
+MongoRunner.stopMongod(conn);
+}());
diff --git a/jstests/disk/wt_table_checks_read_only.js b/jstests/disk/wt_table_checks_read_only.js
new file mode 100644
index 00000000000..6d6519f2c85
--- /dev/null
+++ b/jstests/disk/wt_table_checks_read_only.js
@@ -0,0 +1,51 @@
+/**
+ * Tests that the table logging settings are not changed during read only mode.
+ *
+ * @tags: [requires_wiredtiger]
+ */
+(function() {
+
+load('jstests/disk/libs/wt_file_helper.js');
+
+// Create a bunch of collections under various database names.
+let conn = MongoRunner.runMongod({});
+const dbpath = conn.dbpath;
+
+for (let i = 0; i < 10; i++) {
+    assert.commandWorked(conn.getDB(i.toString()).createCollection(i.toString()));
+}
+
+MongoRunner.stopMongod(conn);
+
+// Option for read only mode.
+let options = {queryableBackupMode: ""};
+
+// Verifies that setTableLogging() does not get called in read only mode, otherwise the invariant
+// would fire.
+conn = startMongodOnExistingPath(dbpath, options);
+assert(conn);
+MongoRunner.stopMongod(conn);
+
+// Create the '_wt_table_checks' file in the dbpath and ensure it doesn't get removed while in read
+// only mode.
+let files = listFiles(dbpath);
+for (f in files) {
+    assert(!files[f].name.includes("_wt_table_checks"));
+}
+
+writeFile(dbpath + "/_wt_table_checks", "");
+
+conn = startMongodOnExistingPath(dbpath, options);
+assert(conn);
+MongoRunner.stopMongod(conn);
+
+let hasWTTableChecksFile = false;
+files = listFiles(dbpath);
+for (f in files) {
+    if (files[f].name.includes("_wt_table_checks")) {
+        hasWTTableChecksFile = true;
+    }
+}
+
+assert(hasWTTableChecksFile);
+}());
diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 8c81f4fa6a1..27442f9b257 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -448,6 +448,10 @@ ExitCode _initAndListen(int listenPort) {
         log() << "Flow Control is enabled on this deployment.";
     }
 
+    // Notify the storage engine that startup is completed before repair exits below, as repair sets
+    // the upgrade flag to true.
+    serviceContext->getStorageEngine()->notifyStartupComplete();
+
     if (storageGlobalParams.upgrade) {
         log() << "finished checking dbs";
         exitCleanly(EXIT_CLEAN);
diff --git a/src/mongo/db/service_context_d_test_fixture.cpp b/src/mongo/db/service_context_d_test_fixture.cpp
index 992e8a86967..6bae2b41670 100644
--- a/src/mongo/db/service_context_d_test_fixture.cpp
+++ b/src/mongo/db/service_context_d_test_fixture.cpp
@@ -83,6 +83,7 @@ ServiceContextMongoDTest::ServiceContextMongoDTest(std::string engine, RepairAct
     IndexAccessMethodFactory::set(serviceContext, std::make_unique<IndexAccessMethodFactoryImpl>());
     Collection::Factory::set(serviceContext, std::make_unique<CollectionImpl::FactoryImpl>());
     IndexBuildsCoordinator::set(serviceContext, std::make_unique<IndexBuildsCoordinatorMongod>());
+    getServiceContext()->getStorageEngine()->notifyStartupComplete();
 }
 
 ServiceContextMongoDTest::~ServiceContextMongoDTest() {
diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h
index 136adb07718..63c19b477e8 100644
--- a/src/mongo/db/storage/kv/kv_engine.h
+++ b/src/mongo/db/storage/kv/kv_engine.h
@@ -52,6 +52,20 @@ class SnapshotManager;
 
 class KVEngine {
 public:
+    /**
+     * During the startup process, the storage engine is one of the first components to be started
+     * up and fully initialized. But that fully initialized storage engine may not be recognized as
+     * the end for the remaining storage startup tasks that still need to be performed.
+     *
+     * For example, after the storage engine has been fully initialized, we need to access it in
+     * order to set up all of the collections and indexes based on the metadata, or perform some
+     * corrective measures on the data files, etc.
+     *
+     * When all of the storage startup tasks are completed as a whole, then this function is called
+     * by the external force managing the startup process.
+     */
+    virtual void notifyStartupComplete() {}
+
     virtual RecoveryUnit* newRecoveryUnit() = 0;
 
     // ---------
diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h
index 6c4231322bc..5b24a659c31 100644
--- a/src/mongo/db/storage/storage_engine.h
+++ b/src/mongo/db/storage/storage_engine.h
@@ -192,6 +192,20 @@ public:
     virtual void finishInit() {}
 
     /**
+     * During the startup process, the storage engine is one of the first components to be started
+     * up and fully initialized. But that fully initialized storage engine may not be recognized as
+     * the end for the remaining storage startup tasks that still need to be performed.
+     *
+     * For example, after the storage engine has been fully initialized, we need to access it in
+     * order to set up all of the collections and indexes based on the metadata, or perform some
+     * corrective measures on the data files, etc.
+     *
+     * When all of the storage startup tasks are completed as a whole, then this function is called
+     * by the external force managing the startup process.
+     */
+    virtual void notifyStartupComplete() {}
+
+    /**
      * Returns a new interface to the storage engine's recovery unit.  The recovery
      * unit is the durability interface.  For details, see recovery_unit.h
      *
diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp
index c33ffc01366..8913244e922 100644
--- a/src/mongo/db/storage/storage_engine_impl.cpp
+++ b/src/mongo/db/storage/storage_engine_impl.cpp
@@ -546,6 +546,10 @@ void StorageEngineImpl::finishInit() {
     }
 }
 
+void StorageEngineImpl::notifyStartupComplete() {
+    _engine->notifyStartupComplete();
+}
+
 RecoveryUnit* StorageEngineImpl::newRecoveryUnit() {
     if (!_engine) {
         // shutdown
diff --git a/src/mongo/db/storage/storage_engine_impl.h b/src/mongo/db/storage/storage_engine_impl.h
index 2422b4b0f64..15f18307737 100644
--- a/src/mongo/db/storage/storage_engine_impl.h
+++ b/src/mongo/db/storage/storage_engine_impl.h
@@ -73,6 +73,8 @@ public:
 
     virtual RecoveryUnit* newRecoveryUnit();
 
+    virtual void notifyStartupComplete() override;
+
     virtual std::vector<std::string> listDatabases() const;
 
     virtual bool supportsDocLocking() const {
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index f40d0f80006..a015e747028 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -810,6 +810,10 @@ WiredTigerKVEngine::~WiredTigerKVEngine() {
     _sessionCache.reset(NULL);
 }
 
+void WiredTigerKVEngine::notifyStartupComplete() {
+    WiredTigerUtil::notifyStartupComplete();
+}
+
 void WiredTigerKVEngine::appendGlobalStats(BSONObjBuilder& b) {
     BSONObjBuilder bb(b.subobjStart("concurrentTransactions"));
     {
@@ -938,6 +942,8 @@ void WiredTigerKVEngine::_openWiredTiger(const std::string& path, const std::str
 
 void WiredTigerKVEngine::cleanShutdown() {
     log() << "WiredTigerKVEngine shutting down";
+    WiredTigerUtil::resetTableLoggingInfo();
+
     if (!_readOnly)
         syncSizeInfo(true);
     if (!_conn) {
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index b4149e9c1d6..7d42c8b52fa 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -85,6 +85,8 @@ public:
 
     ~WiredTigerKVEngine();
 
+    void notifyStartupComplete() override;
+
     void setRecordStoreExtraOptions(const std::string& options);
     void setSortedDataInterfaceExtraOptions(const std::string& options);
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp
index 6cbf18a756b..9b0bb9d2979 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp
@@ -62,6 +62,7 @@ public:
             getGlobalServiceContext(),
             std::unique_ptr<repl::ReplicationCoordinator>(new repl::ReplicationCoordinatorMock(
                 getGlobalServiceContext(), repl::ReplSettings())));
+        _engine->notifyStartupComplete();
     }
 
     virtual KVEngine* restartEngine() override {
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp
index 88e274f8a7f..363665ed1e7 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp
@@ -87,6 +87,7 @@ public:
             getGlobalServiceContext(),
             std::unique_ptr<repl::ReplicationCoordinator>(new repl::ReplicationCoordinatorMock(
                 getGlobalServiceContext(), repl::ReplSettings())));
+        _engine->notifyStartupComplete();
     }
 
     PrefixedWiredTigerHarnessHelper(StringData extraStrings) : _dbpath("wt_test") {}
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_standard_record_store_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_standard_record_store_test.cpp
index 771fc314c8a..2da84b493cc 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_standard_record_store_test.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_standard_record_store_test.cpp
@@ -87,6 +87,7 @@ public:
         repl::ReplicationCoordinator::set(serviceContext(),
                                           std::make_unique<repl::ReplicationCoordinatorMock>(
                                               serviceContext(), repl::ReplSettings()));
+        _engine.notifyStartupComplete();
     }
 
     ~WiredTigerHarnessHelper() {}
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
index d9e327163ff..db82f29378c 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
@@ -35,14 +35,19 @@
 
 #include <limits>
 
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/path.hpp>
+
 #include "mongo/base/simple_string_data_comparator.h"
 #include "mongo/bson/bsonobjbuilder.h"
 #include "mongo/db/concurrency/write_conflict_exception.h"
 #include "mongo/db/snapshot_window_options.h"
+#include "mongo/db/storage/storage_file_util.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h"
 #include "mongo/util/assert_util.h"
+#include "mongo/util/fail_point.h"
 #include "mongo/util/log.h"
 #include "mongo/util/processinfo.h"
 #include "mongo/util/scopeguard.h"
@@ -50,8 +55,75 @@
 
 namespace mongo {
 
+MONGO_FAIL_POINT_DEFINE(crashAfterUpdatingFirstTableLoggingSettings);
+
+namespace {
+
+const std::string kTableChecksFileName = "_wt_table_checks";
+
+/**
+ * Returns true if the 'kTableChecksFileName' file exists in the dbpath.
+ *
+ * Must be called before createTableChecksFile() or removeTableChecksFile() to get accurate results.
+ */
+bool hasPreviouslyIncompleteTableChecks() {
+    auto path = boost::filesystem::path(storageGlobalParams.dbpath) /
+        boost::filesystem::path(kTableChecksFileName);
+
+    return boost::filesystem::exists(path);
+}
+
+/**
+ * Creates the 'kTableChecksFileName' file in the dbpath.
+ */
+void createTableChecksFile() {
+    auto path = boost::filesystem::path(storageGlobalParams.dbpath) /
+        boost::filesystem::path(kTableChecksFileName);
+
+    boost::filesystem::ofstream fileStream(path);
+    fileStream << "This file indicates that a WiredTiger table check operation is in progress or "
+                  "incomplete."
+               << std::endl;
+    if (fileStream.fail()) {
+        log() << "Failed to write to file " << path.generic_string() << " because "
+              << errnoWithDescription();
+        fassertFailedNoTrace(4366400);
+    }
+    fileStream.close();
+
+    fassertNoTrace(4366401, fsyncFile(path));
+    fassertNoTrace(4366402, fsyncParentDirectory(path));
+}
+
+/**
+ * Removes the 'kTableChecksFileName' file in the dbpath, if it exists.
+ */
+void removeTableChecksFile() {
+    auto path = boost::filesystem::path(storageGlobalParams.dbpath) /
+        boost::filesystem::path(kTableChecksFileName);
+
+    if (!boost::filesystem::exists(path)) {
+        return;
+    }
+
+    boost::system::error_code errorCode;
+    boost::filesystem::remove(path, errorCode);
+
+    if (errorCode) {
+        log() << "Failed to remove file " << path.generic_string() << " because "
+              << errorCode.message();
+        fassertFailedNoTrace(4366403);
+    }
+}
+
+}  // namespace
+
 using std::string;
 
+Mutex WiredTigerUtil::_tableLoggingInfoMutex =
+    MONGO_MAKE_LATCH("WiredTigerUtil::_tableLoggingInfoMutex");
+WiredTigerUtil::TableLoggingInfo WiredTigerUtil::_tableLoggingInfo;
+
 Status wtRCToStatus_slow(int retCode, const char* prefix) {
     if (retCode == 0)
         return Status::OK();
@@ -527,6 +599,23 @@ int WiredTigerUtil::verifyTable(OperationContext* opCtx,
     return (session->verify)(session, uri.c_str(), NULL);
 }
 
+void WiredTigerUtil::notifyStartupComplete() {
+    {
+        stdx::lock_guard<Latch> lk(_tableLoggingInfoMutex);
+        invariant(_tableLoggingInfo.isInitializing);
+        _tableLoggingInfo.isInitializing = false;
+    }
+
+    if (!storageGlobalParams.readOnly) {
+        removeTableChecksFile();
+    }
+}
+
+void WiredTigerUtil::resetTableLoggingInfo() {
+    stdx::lock_guard<Latch> lk(_tableLoggingInfoMutex);
+    _tableLoggingInfo = TableLoggingInfo();
+}
+
 bool WiredTigerUtil::useTableLogging(NamespaceString ns, bool replEnabled) {
     if (!replEnabled) {
         // All tables on standalones are logged.
@@ -561,13 +650,85 @@ Status WiredTigerUtil::setTableLogging(OperationContext* opCtx, const std::strin
 }
 
 Status WiredTigerUtil::setTableLogging(WT_SESSION* session, const std::string& uri, bool on) {
-    std::string setting;
-    if (on) {
-        setting = "log=(enabled=true)";
-    } else {
-        setting = "log=(enabled=false)";
+    invariant(!storageGlobalParams.readOnly);
+    stdx::lock_guard<Latch> lk(_tableLoggingInfoMutex);
+
+    // Update the table logging settings regardless if we're no longer starting up the process.
+    if (!_tableLoggingInfo.isInitializing) {
+        return _setTableLogging(session, uri, on);
+    }
+
+    // During the start up process, the table logging settings are checked for each table to verify
+    // that they are set appropriately. We can speed this process up by assuming that the logging
+    // setting is identical for each table.
+    // We cross reference the logging settings for the first table and if it isn't correctly set, we
+    // change the logging settings for all tables during start up.
+    // In the event that the server wasn't shutdown cleanly, the logging settings will be modified
+    // for all tables as a safety precaution, or if repair mode is running.
+    if (_tableLoggingInfo.isFirstTable && hasPreviouslyIncompleteTableChecks()) {
+        _tableLoggingInfo.hasPreviouslyIncompleteTableChecks = true;
+    }
+
+    if (storageGlobalParams.repair || _tableLoggingInfo.hasPreviouslyIncompleteTableChecks) {
+        if (_tableLoggingInfo.isFirstTable) {
+            _tableLoggingInfo.isFirstTable = false;
+            if (!_tableLoggingInfo.hasPreviouslyIncompleteTableChecks) {
+                createTableChecksFile();
+            }
+
+            log() << "Modifying the table logging settings to " << on
+                  << " for all existing WiredTiger tables. Repair? " << storageGlobalParams.repair
+                  << ", has previously incomplete table checks? "
+                  << _tableLoggingInfo.hasPreviouslyIncompleteTableChecks;
+        }
+
+        return _setTableLogging(session, uri, on);
+    }
+
+    if (!_tableLoggingInfo.isFirstTable) {
+        if (_tableLoggingInfo.changeTableLogging) {
+            return _setTableLogging(session, uri, on);
+        }
+
+        // The table logging settings do not need to be modified.
+        return Status::OK();
+    }
+
+    invariant(_tableLoggingInfo.isFirstTable);
+    invariant(!_tableLoggingInfo.hasPreviouslyIncompleteTableChecks);
+    _tableLoggingInfo.isFirstTable = false;
+
+    // Check if the first tables logging settings need to be modified.
+    const std::string setting = on ? "log=(enabled=true)" : "log=(enabled=false)";
+    const std::string existingMetadata = getMetadataCreate(session, uri).getValue();
+    if (existingMetadata.find(setting) != std::string::npos) {
+        // The table is running with the expected logging settings.
+        log() << "No table logging settings modifications are required for existing WiredTiger "
+                 "tables."
+              << " Logging enabled? " << on;
+        return Status::OK();
     }
 
+    // The first table is running with the incorrect logging settings. All tables will need to have
+    // their logging settings modified.
+    _tableLoggingInfo.changeTableLogging = true;
+    createTableChecksFile();
+
+    log() << "Modifying the table logging settings for all existing WiredTiger tables."
+          << " Logging enabled? " << on;
+
+    Status status = _setTableLogging(session, uri, on);
+
+    if (MONGO_unlikely(crashAfterUpdatingFirstTableLoggingSettings.shouldFail())) {
+        log() << "Crashing due to 'crashAfterUpdatingFirstTableLoggingSettings' fail point";
+        fassertFailedNoTrace(4366407);
+    }
+    return status;
+}
+
+Status WiredTigerUtil::_setTableLogging(WT_SESSION* session, const std::string& uri, bool on) {
+    const std::string setting = on ? "log=(enabled=true)" : "log=(enabled=false)";
+
     // This method does some "weak" parsing to see if the table is in the expected logging
     // state. Only attempt to alter the table when a change is needed. This avoids grabbing heavy
     // locks in WT when creating new tables for collections and indexes. Those tables are created
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
index 1e7e5d9c113..72582ab7b89 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
@@ -277,6 +277,10 @@ public:
                            const std::string& uri,
                            std::vector<std::string>* errors = NULL);
 
+    static void notifyStartupComplete();
+
+    static void resetTableLoggingInfo();
+
     static bool useTableLogging(NamespaceString ns, bool replEnabled);
 
     static Status setTableLogging(OperationContext* opCtx, const std::string& uri, bool on);
@@ -297,6 +301,18 @@ private:
      */
     template <typename T>
     static T _castStatisticsValue(uint64_t statisticsValue, T maximumResultType);
+
+    static Status _setTableLogging(WT_SESSION* session, const std::string& uri, bool on);
+
+    // Used to keep track of the table logging setting modifications during start up. The mutex must
+    // be held prior to accessing any of the member variables in the struct.
+    static Mutex _tableLoggingInfoMutex;
+    static struct TableLoggingInfo {
+        bool isInitializing = true;
+        bool isFirstTable = true;
+        bool changeTableLogging = false;
+        bool hasPreviouslyIncompleteTableChecks = false;
+    } _tableLoggingInfo;
 };
 
 class WiredTigerConfigParser {
diff --git a/src/mongo/embedded/embedded.cpp b/src/mongo/embedded/embedded.cpp
index 6a58f70004a..d4ea2e65436 100644
--- a/src/mongo/embedded/embedded.cpp
+++ b/src/mongo/embedded/embedded.cpp
@@ -295,6 +295,10 @@ ServiceContext* initialize(const char* yaml_config) {
         invariant(serverGlobalParams.featureCompatibility.isVersionInitialized());
     }
 
+    // Notify the storage engine that startup is completed before repair exits below, as repair sets
+    // the upgrade flag to true.
+    serviceContext->getStorageEngine()->notifyStartupComplete();
+
     if (storageGlobalParams.upgrade) {
         log() << "finished checking dbs";
         exitCleanly(EXIT_CLEAN);