SERVER-47568 Disable clusterTime gossiping for nodes in unreadable states

(cherry picked from commit 024b130c5e66bafd99cf7f899cdef8d23284ef81)

6 files changed, 91 insertions, 7 deletions

diff --git a/buildscripts/resmokeconfig/suites/replica_sets_auth.yml b/buildscripts/resmokeconfig/suites/replica_sets_auth.yml
index 8dbde6e9005..080497ce17d 100644
--- a/buildscripts/resmokeconfig/suites/replica_sets_auth.yml
+++ b/buildscripts/resmokeconfig/suites/replica_sets_auth.yml
@@ -11,6 +11,7 @@ selector:
   exclude_files:
   # Skip any tests that run with auth explicitly.
   - jstests/replsets/*[aA]uth*.js
+  - jstests/replsets/disable_cluster_time_gossiping_in_unreadable_state.js
   # Also skip tests that require a ScopedThread, because ScopedThreads don't inherit credentials.
   - jstests/replsets/interrupted_batch_insert.js
   - jstests/replsets/transactions_reaped_with_tickets_exhausted.js
diff --git a/jstests/replsets/disable_cluster_time_gossiping_in_unreadable_state.js b/jstests/replsets/disable_cluster_time_gossiping_in_unreadable_state.js
new file mode 100644
index 00000000000..e43ea6f59b8
--- /dev/null
+++ b/jstests/replsets/disable_cluster_time_gossiping_in_unreadable_state.js
@@ -0,0 +1,73 @@
+/**
+ * Verifies cluster time metadata is not gossiped or processed by nodes in an unreadable state.
+ */
+(function() {
+    "use strict";
+
+    function setUpUsers(rst) {
+        const primaryAdminDB = rst.getPrimary().getDB("admin");
+        assert.commandWorked(
+            primaryAdminDB.runCommand({createUser: "admin", pwd: "admin", roles: ["root"]}));
+        assert.eq(1, primaryAdminDB.auth("admin", "admin"));
+
+        assert.commandWorked(primaryAdminDB.getSiblingDB("test").runCommand(
+            {createUser: "NotTrusted", pwd: "pwd", roles: ["readWrite"]}));
+        primaryAdminDB.logout();
+
+        authutil.asCluster(rst.nodes, "jstests/libs/key1", () => {
+            rst.awaitLastOpCommitted();
+        });
+    }
+
+    // Start with auth enabled so cluster times are validated.
+    const rst = new ReplSetTest({nodes: 2, keyFile: "jstests/libs/key1"});
+    rst.startSet();
+    rst.initiate();
+
+    setUpUsers(rst);
+
+    const secondaryAdminDB = new Mongo(rst.getSecondary().host).getDB("admin");
+    secondaryAdminDB.auth("admin", "admin");
+
+    const secondaryTestDB = rst.getSecondary().getDB("test");
+    secondaryTestDB.auth("NotTrusted", "pwd");
+
+    // Cluster time should be gossipped in the steady state.
+    let res = assert.commandWorked(secondaryTestDB.runCommand({find: "foo", filter: {}}));
+    assert.hasFields(res, ["$clusterTime", "operationTime"]);
+    const validClusterTimeMetadata = res.$clusterTime;
+
+    // After entering maintenance mode, cluster time should no longer be gossipped, in or out.
+    assert.commandWorked(secondaryAdminDB.adminCommand({replSetMaintenance: 1}));
+
+    // The find should fail because the node is unreadable.
+    res = assert.commandFailedWithCode(secondaryTestDB.runCommand({find: "foo", filter: {}}),
+                                       ErrorCodes.NotMasterOrSecondary);
+    assert(!res.hasOwnProperty("$clusterTime"), tojson(res));
+    assert(!res.hasOwnProperty("operationTime"), tojson(res));
+
+    // A request with $clusterTime should be ignored. This is verified by sending an invalid
+    // $clusterTime to emulate situations where valid cluster times would be unable to be verified,
+    // e.g. when the signing keys have not been cached but cannot be read from admin.system.keys
+    // because the node is in an unreadable state.
+    const invalidClusterTimeMetadata = Object.assign(
+        validClusterTimeMetadata,
+        {clusterTime: new Timestamp(validClusterTimeMetadata.clusterTime.getTime() + 100, 0)});
+    res = assert.commandWorked(
+        secondaryTestDB.runCommand({hello: 1, $clusterTime: invalidClusterTimeMetadata}));
+
+    assert.commandWorked(secondaryAdminDB.adminCommand({replSetMaintenance: 0}));
+
+    res = assert.commandWorked(secondaryTestDB.runCommand({find: "foo", filter: {}}));
+    assert.hasFields(res, ["$clusterTime", "operationTime"]);
+
+    // A request with invalid cluster time metadata should now be rejected.
+    assert.commandFailedWithCode(
+        secondaryTestDB.runCommand({hello: 1, $clusterTime: invalidClusterTimeMetadata}),
+        ErrorCodes.TimeProofMismatch);
+
+    secondaryAdminDB.logout();
+    secondaryTestDB.logout();
+
+    rst.stopSet();
+})();
diff --git a/jstests/replsets/read_concern_uninitated_set.js b/jstests/replsets/read_concern_uninitated_set.js
index 0737b88e229..4856992d9f1 100644
--- a/jstests/replsets/read_concern_uninitated_set.js
+++ b/jstests/replsets/read_concern_uninitated_set.js
@@ -37,16 +37,18 @@
             {find: "test", filter: {}, maxTimeMS: 60000, readConcern: {level: "majority"}}),
         ErrorCodes.NotYetInitialized);
 
-    jsTestLog("afterClusterTime readConcern should fail with NotYetInitialized.");
+    // Nodes don't process $clusterTime metadata when in an unreadable state, so this read will fail
+    // because the logical clock's latest value is less than the given afterClusterTime timestamp.
+    jsTestLog("afterClusterTime readConcern should fail with InvalidOptions.");
     assert.commandFailedWithCode(localDB.runCommand({
         find: "test",
         filter: {},
         maxTimeMS: 60000,
         readConcern: {afterClusterTime: Timestamp(1, 1)}
     }),
-                                 ErrorCodes.NotYetInitialized);
+                                 ErrorCodes.InvalidOptions);
 
-    jsTestLog("oplog query should fail with NotYetInitialized.");
+    jsTestLog("oplog query should fail with InvalidOptions.");
     assert.commandFailedWithCode(localDB.runCommand({
         find: "oplog.rs",
         filter: {ts: {$gte: Timestamp(1520004466, 2)}},
@@ -58,6 +60,6 @@
         term: 1,
         readConcern: {afterClusterTime: Timestamp(1, 1)}
     }),
-                                 ErrorCodes.NotYetInitialized);
+                                 ErrorCodes.InvalidOptions);
     rst.stopSet();
 }());
diff --git a/src/mongo/db/service_entry_point_common.cpp b/src/mongo/db/service_entry_point_common.cpp
index 2d5b25061f0..b29c38a6f26 100644
--- a/src/mongo/db/service_entry_point_common.cpp
+++ b/src/mongo/db/service_entry_point_common.cpp
@@ -437,8 +437,9 @@ void appendClusterAndOperationTime(OperationContext* opCtx,
                                    BSONObjBuilder* commandBodyFieldsBob,
                                    BSONObjBuilder* metadataBob,
                                    LogicalTime startTime) {
-    if (repl::ReplicationCoordinator::get(opCtx)->getReplicationMode() !=
-            repl::ReplicationCoordinator::modeReplSet ||
+    auto replicationCoordinator = repl::ReplicationCoordinator::get(opCtx);
+    if (replicationCoordinator->getReplicationMode() != repl::ReplicationCoordinator::modeReplSet ||
+        !replicationCoordinator->getMemberState().readable() ||
         !LogicalClock::get(opCtx)->isEnabled()) {
         return;
     }
diff --git a/src/mongo/rpc/SConscript b/src/mongo/rpc/SConscript
index fff0a749384..a2a628b26ce 100644
--- a/src/mongo/rpc/SConscript
+++ b/src/mongo/rpc/SConscript
@@ -155,6 +155,7 @@ env.Library(
         '$BUILD_DIR/mongo/client/read_preference',
         '$BUILD_DIR/mongo/db/logical_time_validator',
         '$BUILD_DIR/mongo/db/repl/optime',
+        '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface',
         '$BUILD_DIR/mongo/db/signed_logical_time',
     ],
 )
diff --git a/src/mongo/rpc/metadata.cpp b/src/mongo/rpc/metadata.cpp
index 5bbda5af5b3..13e0fcf43d7 100644
--- a/src/mongo/rpc/metadata.cpp
+++ b/src/mongo/rpc/metadata.cpp
@@ -37,6 +37,7 @@
 #include "mongo/db/jsobj.h"
 #include "mongo/db/logical_clock.h"
 #include "mongo/db/logical_time_validator.h"
+#include "mongo/db/repl/replication_coordinator.h"
 #include "mongo/rpc/metadata/audit_metadata.h"
 #include "mongo/rpc/metadata/client_metadata_ismaster.h"
 #include "mongo/rpc/metadata/config_server_metadata.h"
@@ -93,7 +94,12 @@ void readRequestMetadata(OperationContext* opCtx, const BSONObj& metadataObj, bo
         uassertStatusOK(TrackingMetadata::readFromMetadata(trackingElem));
 
     auto logicalClock = LogicalClock::get(opCtx);
-    if (logicalClock && logicalClock->isEnabled()) {
+    auto replicationCoordinator = repl::ReplicationCoordinator::get(opCtx);
+    if (logicalClock && logicalClock->isEnabled() &&
+        (!replicationCoordinator || (replicationCoordinator->getReplicationMode() ==
+                                         repl::ReplicationCoordinator::modeReplSet &&
+                                     replicationCoordinator->getMemberState().readable()))) {
+
         auto logicalTimeMetadata =
             uassertStatusOK(rpc::LogicalTimeMetadata::readFromMetadata(logicalTimeElem));