SERVER-40010 Make only FCV 4.0 secondaries ask the primary to flush the database cache

3 files changed, 57 insertions, 4 deletions

diff --git a/jstests/multiVersion/secondary_reads_in_mixed_version_shard_replica_set.js b/jstests/multiVersion/secondary_reads_in_mixed_version_shard_replica_set.js
new file mode 100644
index 00000000000..5a1fc9d86b3
--- /dev/null
+++ b/jstests/multiVersion/secondary_reads_in_mixed_version_shard_replica_set.js
@@ -0,0 +1,42 @@
+//
+// Testing migrations between latest and last-stable mongod versions, where the
+// donor is the latest version and the recipient the last-stable, and vice versa.
+// Migrations should be successful.
+//
+
+// Checking UUID consistency involves talking to a shard node, which in this test is shutdown
+TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+
+(function() {
+    "use strict";
+
+    var mixedShard = new ReplSetTest({
+        name: 'MixedVersionShard',
+        nodes: [
+            {binVersion: "last-stable", rsConfig: {votes: 1}},
+            {binVersion: "latest", rsConfig: {priority: 0, votes: 0}}
+        ]
+    });
+    mixedShard.startSet();
+    mixedShard.initiate();
+    mixedShard.restart(0, {'shardsvr': ""});
+    mixedShard.restart(1, {'shardsvr': ""});
+
+    var st = new ShardingTest(
+        {mongosOptions: {binVersion: "last-stable"}, shards: 0, manualAddShard: true});
+    assert.commandWorked(st.addShard(mixedShard.getURL()));
+
+    // Ensure the secondary node also knows that it is sharded by waiting for the identity document
+    // write to propagate
+    mixedShard.awaitReplication();
+
+    assert.commandWorked(st.s0.adminCommand({enableSharding: 'TestDB'}));
+    assert.commandWorked(st.s0.adminCommand({shardCollection: 'TestDB.TestColl', key: {x: 1}}));
+
+    let db = st.s0.getDB('TestDB');
+    assert.commandWorked(db.runCommand(
+        {find: 'TestColl', $readPreference: {mode: 'secondary'}, readConcern: {'level': 'local'}}));
+
+    st.stop();
+    mixedShard.stopSet();
+})();
diff --git a/jstests/sharding/database_versioning_upgrade_downgrade.js b/jstests/sharding/database_versioning_upgrade_downgrade.js
index b6840d7c73a..d832bd3ca35 100644
--- a/jstests/sharding/database_versioning_upgrade_downgrade.js
+++ b/jstests/sharding/database_versioning_upgrade_downgrade.js
@@ -45,14 +45,14 @@
     // TODO (SERVER-34431): Remove this once the DatabaseShardingState is in a standalone map.
     assert.commandWorked(st.s.getDB(db1Name).runCommand({create: collName}));
 
-    // Force the shard database to refresh and ensure it writes a cache entry but does not cache the
-    // version in memory.
+    // Force the shard database to refresh and ensure it neither writes a cache entry, nor does it
+    // cache the version in memory
     assert.commandWorked(st.rs0.getPrimary().adminCommand({_flushDatabaseCacheUpdates: db1Name}));
     checkInMemoryDatabaseVersion(st.rs0.getPrimary(), db1Name, {});
     checkInMemoryDatabaseVersion(st.rs0.getSecondary(), db1Name, {});
     checkInMemoryDatabaseVersion(st.rs0.getPrimary(), db2Name, {});
     checkInMemoryDatabaseVersion(st.rs0.getSecondary(), db2Name, {});
-    checkOnDiskDatabaseVersion(st.shard0, db1Name, db1EntryOriginal);
+    checkOnDiskDatabaseVersion(st.shard0, db1Name, undefined);
     checkOnDiskDatabaseVersion(st.shard0, db2Name, undefined);
 
     //
@@ -74,7 +74,7 @@
     checkInMemoryDatabaseVersion(st.rs0.getSecondary(), db1Name, {});
     checkInMemoryDatabaseVersion(st.rs0.getPrimary(), db2Name, {});
     checkInMemoryDatabaseVersion(st.rs0.getSecondary(), db2Name, {});
-    checkOnDiskDatabaseVersion(st.shard0, db1Name, db1EntryOriginal);
+    checkOnDiskDatabaseVersion(st.shard0, db1Name, undefined);
     checkOnDiskDatabaseVersion(st.shard0, db2Name, undefined);
 
     // After receiving a versioned request for a database, the shard should refresh its in-memory
diff --git a/src/mongo/db/s/shard_server_catalog_cache_loader.cpp b/src/mongo/db/s/shard_server_catalog_cache_loader.cpp
index 52bba0249c9..880ae109ddf 100644
--- a/src/mongo/db/s/shard_server_catalog_cache_loader.cpp
+++ b/src/mongo/db/s/shard_server_catalog_cache_loader.cpp
@@ -41,6 +41,7 @@
 #include "mongo/db/repl/replication_coordinator.h"
 #include "mongo/db/s/shard_metadata_util.h"
 #include "mongo/db/s/sharding_state.h"
+#include "mongo/db/server_options.h"
 #include "mongo/s/catalog/type_shard_collection.h"
 #include "mongo/s/catalog/type_shard_database.h"
 #include "mongo/s/client/shard_registry.h"
@@ -440,6 +441,16 @@ std::shared_ptr<Notification<void>> ShardServerCatalogCacheLoader::getChunksSinc
 void ShardServerCatalogCacheLoader::getDatabase(
     StringData dbName,
     stdx::function<void(OperationContext*, StatusWith<DatabaseType>)> callbackFn) {
+    // If the FCV version has not been fully upgraded to 4.0, fetch the database entries directly
+    // from the config server, regardless of whether the shard is running as a primary or secondary.
+    // This matches the behaviour of v3.6 and ensures that v4.0 secondaries do not ask a potentially
+    // v3.6 primary to flush its database cache (which is not supported)
+    if (serverGlobalParams.featureCompatibility.getVersion() <
+        ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo40) {
+        _configServerLoader->getDatabase(dbName, callbackFn);
+        return;
+    }
+
     long long currentTerm;
     bool isPrimary;