SERVER-47568 Disable clusterTime gossiping for nodes in unreadable statesr4.2.13-rc2r4.2.13

(cherry picked from commit 024b130c5e66bafd99cf7f899cdef8d23284ef81)

6 files changed, 95 insertions, 7 deletions

diff --git a/buildscripts/resmokeconfig/suites/replica_sets_auth.yml b/buildscripts/resmokeconfig/suites/replica_sets_auth.yml
index 03f48dd72b5..b040a1c9da7 100644
--- a/buildscripts/resmokeconfig/suites/replica_sets_auth.yml
+++ b/buildscripts/resmokeconfig/suites/replica_sets_auth.yml
@@ -11,6 +11,7 @@ selector:
   exclude_files:
   # Skip any tests that run with auth explicitly.
   - jstests/replsets/*[aA]uth*.js
+  - jstests/replsets/disable_cluster_time_gossiping_in_unreadable_state.js
   # Also skip tests that require a ScopedThread, because ScopedThreads don't inherit credentials.
   - jstests/replsets/interrupted_batch_insert.js
   - jstests/replsets/transactions_reaped_with_tickets_exhausted.js
diff --git a/etc/backports_required_for_multiversion_tests.yml b/etc/backports_required_for_multiversion_tests.yml
index 3421495d9d6..a22586a0777 100644
--- a/etc/backports_required_for_multiversion_tests.yml
+++ b/etc/backports_required_for_multiversion_tests.yml
@@ -123,6 +123,10 @@ suites:
     test_file: jstests/replsets/invalidate_sessions_on_stepdown.js
   - ticket: SERVER-35649
     test_file: jstests/replsets/disallow_adding_initialized_node1.js
+  - ticket: SERVER-47568
+    test_file: jstests/replsets/read_concern_uninitated_set.js
+  - ticket: SERVER-47568
+    test_file: jstests/replsets/disable_cluster_time_gossiping_in_unreadable_state.js
 
   sharding_multiversion:
 
diff --git a/jstests/replsets/disable_cluster_time_gossiping_in_unreadable_state.js b/jstests/replsets/disable_cluster_time_gossiping_in_unreadable_state.js
new file mode 100644
index 00000000000..3febc4fef54
--- /dev/null
+++ b/jstests/replsets/disable_cluster_time_gossiping_in_unreadable_state.js
@@ -0,0 +1,73 @@
+/**
+ * Verifies cluster time metadata is not gossiped or processed by nodes in an unreadable state.
+ */
+(function() {
+"use strict";
+
+function setUpUsers(rst) {
+    const primaryAdminDB = rst.getPrimary().getDB("admin");
+    assert.commandWorked(
+        primaryAdminDB.runCommand({createUser: "admin", pwd: "admin", roles: ["root"]}));
+    assert.eq(1, primaryAdminDB.auth("admin", "admin"));
+
+    assert.commandWorked(primaryAdminDB.getSiblingDB("test").runCommand(
+        {createUser: "NotTrusted", pwd: "pwd", roles: ["readWrite"]}));
+    primaryAdminDB.logout();
+
+    authutil.asCluster(rst.nodes, "jstests/libs/key1", () => {
+        rst.awaitLastOpCommitted();
+    });
+}
+
+// Start with auth enabled so cluster times are validated.
+const rst = new ReplSetTest({nodes: 2, keyFile: "jstests/libs/key1"});
+rst.startSet();
+rst.initiate();
+
+setUpUsers(rst);
+
+const secondaryAdminDB = new Mongo(rst.getSecondary().host).getDB("admin");
+secondaryAdminDB.auth("admin", "admin");
+
+const secondaryTestDB = rst.getSecondary().getDB("test");
+secondaryTestDB.auth("NotTrusted", "pwd");
+
+// Cluster time should be gossipped in the steady state.
+let res = assert.commandWorked(secondaryTestDB.runCommand({find: "foo", filter: {}}));
+assert.hasFields(res, ["$clusterTime", "operationTime"]);
+const validClusterTimeMetadata = res.$clusterTime;
+
+// After entering maintenance mode, cluster time should no longer be gossipped, in or out.
+assert.commandWorked(secondaryAdminDB.adminCommand({replSetMaintenance: 1}));
+
+// The find should fail because the node is unreadable.
+res = assert.commandFailedWithCode(secondaryTestDB.runCommand({find: "foo", filter: {}}),
+                                   ErrorCodes.NotPrimaryOrSecondary);
+assert(!res.hasOwnProperty("$clusterTime"), tojson(res));
+assert(!res.hasOwnProperty("operationTime"), tojson(res));
+
+// A request with $clusterTime should be ignored. This is verified by sending an invalid
+// $clusterTime to emulate situations where valid cluster times would be unable to be verified, e.g.
+// when the signing keys have not been cached but cannot be read from admin.system.keys because the
+// node is in an unreadable state.
+const invalidClusterTimeMetadata = Object.assign(
+    validClusterTimeMetadata,
+    {clusterTime: new Timestamp(validClusterTimeMetadata.clusterTime.getTime() + 100, 0)});
+res = assert.commandWorked(
+    secondaryTestDB.runCommand({hello: 1, $clusterTime: invalidClusterTimeMetadata}));
+
+assert.commandWorked(secondaryAdminDB.adminCommand({replSetMaintenance: 0}));
+
+res = assert.commandWorked(secondaryTestDB.runCommand({find: "foo", filter: {}}));
+assert.hasFields(res, ["$clusterTime", "operationTime"]);
+
+// A request with invalid cluster time metadata should now be rejected.
+assert.commandFailedWithCode(
+    secondaryTestDB.runCommand({hello: 1, $clusterTime: invalidClusterTimeMetadata}),
+    ErrorCodes.TimeProofMismatch);
+
+secondaryAdminDB.logout();
+secondaryTestDB.logout();
+
+rst.stopSet();
+})();
diff --git a/jstests/replsets/read_concern_uninitated_set.js b/jstests/replsets/read_concern_uninitated_set.js
index 71f9b2c6956..209c7c5b826 100644
--- a/jstests/replsets/read_concern_uninitated_set.js
+++ b/jstests/replsets/read_concern_uninitated_set.js
@@ -36,16 +36,18 @@ assert.commandFailedWithCode(
         {find: "test", filter: {}, maxTimeMS: 60000, readConcern: {level: "majority"}}),
     ErrorCodes.NotYetInitialized);
 
-jsTestLog("afterClusterTime readConcern should fail with NotYetInitialized.");
+// Nodes don't process $clusterTime metadata when in an unreadable state, so this read will fail
+// because the logical clock's latest value is less than the given afterClusterTime timestamp.
+jsTestLog("afterClusterTime readConcern should fail with InvalidOptions.");
 assert.commandFailedWithCode(localDB.runCommand({
     find: "test",
     filter: {},
     maxTimeMS: 60000,
     readConcern: {afterClusterTime: Timestamp(1, 1)}
 }),
-                             ErrorCodes.NotYetInitialized);
+                             ErrorCodes.InvalidOptions);
 
-jsTestLog("oplog query should fail with NotYetInitialized.");
+jsTestLog("oplog query should fail with InvalidOptions.");
 assert.commandFailedWithCode(localDB.runCommand({
     find: "oplog.rs",
     filter: {ts: {$gte: Timestamp(1520004466, 2)}},
@@ -57,6 +59,6 @@ assert.commandFailedWithCode(localDB.runCommand({
     term: 1,
     readConcern: {afterClusterTime: Timestamp(1, 1)}
 }),
-                             ErrorCodes.NotYetInitialized);
+                             ErrorCodes.InvalidOptions);
 rst.stopSet();
 }());
diff --git a/src/mongo/db/service_entry_point_common.cpp b/src/mongo/db/service_entry_point_common.cpp
index a5068b83d58..9a4b200d37a 100644
--- a/src/mongo/db/service_entry_point_common.cpp
+++ b/src/mongo/db/service_entry_point_common.cpp
@@ -332,8 +332,9 @@ void appendClusterAndOperationTime(OperationContext* opCtx,
                                    BSONObjBuilder* commandBodyFieldsBob,
                                    BSONObjBuilder* metadataBob,
                                    LogicalTime startTime) {
-    if (repl::ReplicationCoordinator::get(opCtx)->getReplicationMode() !=
-            repl::ReplicationCoordinator::modeReplSet ||
+    auto replicationCoordinator = repl::ReplicationCoordinator::get(opCtx);
+    if (replicationCoordinator->getReplicationMode() != repl::ReplicationCoordinator::modeReplSet ||
+        !replicationCoordinator->getMemberState().readable() ||
         !LogicalClock::get(opCtx)->isEnabled()) {
         return;
     }
diff --git a/src/mongo/rpc/metadata.cpp b/src/mongo/rpc/metadata.cpp
index e3ed093a693..6d9e7581698 100644
--- a/src/mongo/rpc/metadata.cpp
+++ b/src/mongo/rpc/metadata.cpp
@@ -37,6 +37,7 @@
 #include "mongo/db/jsobj.h"
 #include "mongo/db/logical_clock.h"
 #include "mongo/db/logical_time_validator.h"
+#include "mongo/db/repl/replication_coordinator.h"
 #include "mongo/rpc/metadata/client_metadata_ismaster.h"
 #include "mongo/rpc/metadata/config_server_metadata.h"
 #include "mongo/rpc/metadata/impersonated_user_metadata.h"
@@ -93,7 +94,13 @@ void readRequestMetadata(OperationContext* opCtx, const BSONObj& metadataObj, bo
         uassertStatusOK(TrackingMetadata::readFromMetadata(trackingElem));
 
     auto logicalClock = LogicalClock::get(opCtx);
-    if (logicalClock && logicalClock->isEnabled()) {
+    auto replicationCoordinator = repl::ReplicationCoordinator::get(opCtx);
+    if (logicalClock && logicalClock->isEnabled() &&
+        (!replicationCoordinator ||
+         (replicationCoordinator->getReplicationMode() ==
+              repl::ReplicationCoordinator::modeReplSet &&
+          replicationCoordinator->getMemberState().readable()))) {
+
         auto logicalTimeMetadata =
             uassertStatusOK(rpc::LogicalTimeMetadata::readFromMetadata(logicalTimeElem));