SERVER-48707 Move replication recovery invariant behind test only flag

6 files changed, 42 insertions, 5 deletions

diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py
index e4e64bf21a2..eebd51c68bc 100644
--- a/buildscripts/resmokelib/core/programs.py
+++ b/buildscripts/resmokelib/core/programs.py
@@ -182,6 +182,11 @@ def mongod_program(  # pylint: disable=too-many-branches,too-many-statements
             "mode": "alwaysOn", "data": {"numTickets": config.FLOW_CONTROL_TICKETS}
         }
 
+    # Always enable this invariant for testing.
+    if "assertStableTimestampEqualsAppliedThroughOnRecovery" not in suite_set_parameters and \
+            executable != LAST_STABLE_MONGOD_BINARY:
+        suite_set_parameters["assertStableTimestampEqualsAppliedThroughOnRecovery"] = True
+
     _apply_set_parameters(args, suite_set_parameters)
 
     shortcut_opts = {
@@ -343,6 +348,9 @@ def mongo_shell_program(  # pylint: disable=too-many-branches,too-many-locals,to
     if config.FLOW_CONTROL is not None:
         mongod_set_parameters.setdefault("enableFlowControl", config.FLOW_CONTROL == "on")
 
+    # Always enable this invariant in tests.
+    mongod_set_parameters.setdefault("assertStableTimestampEqualsAppliedThroughOnRecovery", True)
+
     # If the 'logComponentVerbosity' setParameter for mongos was not already specified, we set its
     # value to a default.
     mongos_set_parameters.setdefault("logComponentVerbosity",
diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript
index f4a43da5dd7..c0d4a3cbc31 100644
--- a/src/mongo/db/repl/SConscript
+++ b/src/mongo/db/repl/SConscript
@@ -269,6 +269,7 @@ env.Library(
     LIBDEPS_PRIVATE=[
         'oplog',
         'oplog_application',
+        'repl_server_parameters',
         '$BUILD_DIR/mongo/base',
         '$BUILD_DIR/mongo/db/storage/storage_options',
     ],
diff --git a/src/mongo/db/repl/repl_server_parameters.idl b/src/mongo/db/repl/repl_server_parameters.idl
index ba55667b7b9..6033dc3a5f9 100644
--- a/src/mongo/db/repl/repl_server_parameters.idl
+++ b/src/mongo/db/repl/repl_server_parameters.idl
@@ -288,3 +288,11 @@ server_parameters:
         # and if it is not, readPreference is 'nearest'.
         default: ""
         validator: { callback: 'validateReadPreferenceMode' }
+
+    assertStableTimestampEqualsAppliedThroughOnRecovery:
+        description: Enables invariant to check stable timestamp equals appliedThrough on recovery.
+        set_at: startup
+        test_only: true
+        cpp_vartype: bool
+        cpp_varname: assertStableTimestampEqualsAppliedThroughOnRecovery
+        default: false
\ No newline at end of file
diff --git a/src/mongo/db/repl/replication_recovery.cpp b/src/mongo/db/repl/replication_recovery.cpp
index 9259013bcc8..2d363d6a579 100644
--- a/src/mongo/db/repl/replication_recovery.cpp
+++ b/src/mongo/db/repl/replication_recovery.cpp
@@ -43,6 +43,7 @@
 #include "mongo/db/repl/apply_ops.h"
 #include "mongo/db/repl/oplog_applier_impl.h"
 #include "mongo/db/repl/oplog_buffer.h"
+#include "mongo/db/repl/repl_server_parameters_gen.h"
 #include "mongo/db/repl/replication_consistency_markers_impl.h"
 #include "mongo/db/repl/storage_interface.h"
 #include "mongo/db/repl/transaction_oplog_application.h"
@@ -427,11 +428,14 @@ void ReplicationRecoveryImpl::recoverFromOplog(OperationContext* opCtx,
     }
 
     const auto appliedThrough = _consistencyMarkers->getAppliedThrough(opCtx);
-    invariant(!stableTimestamp || stableTimestamp->isNull() || appliedThrough.isNull() ||
-                  *stableTimestamp == appliedThrough.getTimestamp(),
-              str::stream() << "Stable timestamp " << stableTimestamp->toString()
-                            << " does not equal appliedThrough timestamp "
-                            << appliedThrough.toString());
+    if (assertStableTimestampEqualsAppliedThroughOnRecovery) {
+        invariant(!stableTimestamp || stableTimestamp->isNull() || appliedThrough.isNull() ||
+                      *stableTimestamp == appliedThrough.getTimestamp(),
+                  str::stream() << "Stable timestamp " << stableTimestamp->toString()
+                                << " does not equal appliedThrough timestamp "
+                                << appliedThrough.toString());
+    }
+
 
     // This may take an IS lock on the oplog collection.
     _truncateOplogIfNeededAndThenClearOplogTruncateAfterPoint(opCtx, stableTimestamp);
diff --git a/src/mongo/db/repl/replication_recovery_test.cpp b/src/mongo/db/repl/replication_recovery_test.cpp
index 7eb258c5a8d..d80e7022582 100644
--- a/src/mongo/db/repl/replication_recovery_test.cpp
+++ b/src/mongo/db/repl/replication_recovery_test.cpp
@@ -695,6 +695,10 @@ TEST_F(ReplicationRecoveryTest,
 DEATH_TEST_F(ReplicationRecoveryTest,
              RecoveryFailsWithUnmatchedAppliedThrough,
              "Invariant failure") {
+    ASSERT_OK(ServerParameterSet::getGlobal()
+                  ->getMap()
+                  .find("assertStableTimestampEqualsAppliedThroughOnRecovery")
+                  ->second->setFromString("true"));
     ReplicationRecoveryImpl recovery(getStorageInterface(), getConsistencyMarkers());
     auto opCtx = getOperationContext();
 
@@ -834,6 +838,10 @@ TEST_F(ReplicationRecoveryTest, RecoveryDoesNotApplyOperationsIfAppliedThroughIs
 DEATH_TEST_F(ReplicationRecoveryTest,
              RecoveryInvariantsWithUnequalStableTimestampAndAppliedThrough,
              "Invariant failure") {
+    ASSERT_OK(ServerParameterSet::getGlobal()
+                  ->getMap()
+                  .find("assertStableTimestampEqualsAppliedThroughOnRecovery")
+                  ->second->setFromString("true"));
     ReplicationRecoveryImpl recovery(getStorageInterface(), getConsistencyMarkers());
     auto opCtx = getOperationContext();
 
diff --git a/src/mongo/shell/servers.js b/src/mongo/shell/servers.js
index 5c4ac4ee515..0f0e1057835 100644
--- a/src/mongo/shell/servers.js
+++ b/src/mongo/shell/servers.js
@@ -1265,6 +1265,14 @@ function appendSetParameterArgs(argArray) {
                             continue;
                         }
 
+                        // Do not set this if test commands are not enabled or we are not on a new
+                        // enough version.
+                        if (paramName === "assertStableTimestampEqualsAppliedThroughOnRecovery" &&
+                            (!jsTest.options().enableTestCommands ||
+                             programMajorMinorVersion < 404)) {
+                            continue;
+                        }
+
                         const paramVal = ((param) => {
                             if (typeof param === "object") {
                                 return JSON.stringify(param);