SERVER-34163 Make refreshing a database entry in the CatalogCache asynchronous

6 files changed, 372 insertions, 44 deletions

diff --git a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
index c6bef066976..37bbedbadfd 100644
--- a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
@@ -68,6 +68,7 @@ selector:
   - jstests/sharding/split_large_key.js
   - jstests/sharding/balancer_window.js
   # No retries on direct writes to the config/admin databases on the config servers
+  - jstests/sharding/database_versioning_safe_secondary_reads.js
   - jstests/sharding/listDatabases.js
   - jstests/sharding/bulk_insert.js
   - jstests/sharding/printShardingStatus.js
diff --git a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
index a6de5231bf4..4b291847bf0 100644
--- a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
@@ -13,6 +13,7 @@ selector:
   - jstests/sharding/change_streams_unsharded_becomes_sharded.js
   - jstests/sharding/create_database.js
   - jstests/sharding/database_and_shard_versioning_all_commands.js
+  - jstests/sharding/database_versioning_safe_secondary_reads.js
   - jstests/sharding/dump_coll_metadata.js
   - jstests/sharding/geo_near_random1.js
   - jstests/sharding/geo_near_random2.js
diff --git a/jstests/sharding/database_versioning_safe_secondary_reads.js b/jstests/sharding/database_versioning_safe_secondary_reads.js
new file mode 100644
index 00000000000..9123cc91c8a
--- /dev/null
+++ b/jstests/sharding/database_versioning_safe_secondary_reads.js
@@ -0,0 +1,243 @@
+/**
+ * Tests that shards' cached in-memory and on-disk database versions are updated on primary and
+ * secondary nodes when:
+ * - the node does not have a cached in-memory version
+ * - the node's cached in-memory version is lower than the version sent by a client
+ * - the movePrimary critical section is entered on the primary node
+ */
+(function() {
+    "use strict";
+
+    function checkInMemoryDatabaseVersion(conn, dbName, expectedVersion) {
+        const res = conn.adminCommand({getDatabaseVersion: dbName});
+        assert.commandWorked(res);
+        assert.docEq(res.dbVersion,
+                     expectedVersion,
+                     conn + " did not have expected in-memory database version for " + dbName);
+    }
+
+    function checkOnDiskDatabaseVersion(conn, dbName, authoritativeEntry) {
+        const res =
+            conn.getDB("config").runCommand({find: "cache.databases", filter: {_id: dbName}});
+        assert.commandWorked(res);
+        const cacheEntry = res.cursor.firstBatch[0];
+
+        if (authoritativeEntry === undefined) {
+            assert.eq(cacheEntry, undefined);
+        } else {
+            // Remove the 'enterCriticalSectionCounter' field, which is the only field the cache
+            // entry should have but the authoritative entry does not.
+            delete cacheEntry["enterCriticalSectionCounter"];
+            assert.docEq(cacheEntry,
+                         authoritativeEntry,
+                         conn + " did not have expected on-disk database version for " + dbName);
+        }
+    }
+
+    const dbName = "test";
+
+    const st = new ShardingTest({
+        mongos: 2,
+        rs0: {nodes: [{rsConfig: {votes: 1}}, {rsConfig: {priority: 0, votes: 0}}]},
+        rs1: {nodes: [{rsConfig: {votes: 1}}, {rsConfig: {priority: 0, votes: 0}}]},
+        verbose: 2
+    });
+
+    // Before creating the database, none of the nodes have a cached entry for the database either
+    // in memory or on disk.
+    checkInMemoryDatabaseVersion(st.rs0.getPrimary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs1.getPrimary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs0.getSecondary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs1.getSecondary(), dbName, {});
+    checkOnDiskDatabaseVersion(st.rs0.getPrimary(), dbName, undefined);
+    checkOnDiskDatabaseVersion(st.rs1.getPrimary(), dbName, undefined);
+    checkOnDiskDatabaseVersion(st.rs0.getSecondary(), dbName, undefined);
+    checkOnDiskDatabaseVersion(st.rs1.getSecondary(), dbName, undefined);
+
+    // Use 'enableSharding' to create the database only in the sharding catalog (the database will
+    // not exist on any shards).
+    assert.commandWorked(st.s.adminCommand({enableSharding: dbName}));
+
+    // Check that a command that attaches databaseVersion returns empty results, even though the
+    // database does not actually exist on any shard (because the version won't be checked).
+    assert.commandWorked(st.s.getDB(dbName).runCommand({listCollections: 1}));
+
+    // Once SERVER-34431 goes in, this should have caused the primary shard's primary to refresh its
+    // in-memory and on-disk caches.
+    checkInMemoryDatabaseVersion(st.rs0.getPrimary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs1.getPrimary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs0.getSecondary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs1.getSecondary(), dbName, {});
+    checkOnDiskDatabaseVersion(st.rs0.getPrimary(), dbName, undefined);
+    checkOnDiskDatabaseVersion(st.rs1.getPrimary(), dbName, undefined);
+    checkOnDiskDatabaseVersion(st.rs0.getSecondary(), dbName, undefined);
+    checkOnDiskDatabaseVersion(st.rs1.getSecondary(), dbName, undefined);
+
+    assert.commandWorked(st.s.getDB(dbName).runCommand(
+        {listCollections: 1, $readPreference: {mode: "secondary"}, readConcern: {level: "local"}}));
+
+    // Once SERVER-34431 goes in, this should have caused the primary shard's secondary to refresh
+    // its in-memory cache (its on-disk cache was updated when the primary refreshed, above).
+    checkInMemoryDatabaseVersion(st.rs0.getPrimary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs1.getPrimary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs0.getSecondary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs1.getSecondary(), dbName, {});
+    checkOnDiskDatabaseVersion(st.rs0.getPrimary(), dbName, undefined);
+    checkOnDiskDatabaseVersion(st.rs1.getPrimary(), dbName, undefined);
+    checkOnDiskDatabaseVersion(st.rs0.getSecondary(), dbName, undefined);
+    checkOnDiskDatabaseVersion(st.rs1.getSecondary(), dbName, undefined);
+
+    // Use 'movePrimary' to ensure shard0 is the primary shard. This will create the database on the
+    // shards only if shard0 was not already the primary shard.
+    st.ensurePrimaryShard(dbName, st.shard0.shardName);
+    const dbEntry1 = st.s.getDB("config").getCollection("databases").findOne({_id: dbName});
+
+    // Ensure the database actually gets created on the primary shard by creating a collection in
+    // it.
+    assert.commandWorked(st.s.getDB(dbName).runCommand({create: "foo"}));
+
+    // Run a command that attaches databaseVersion to cause the current primary shard's primary to
+    // refresh its in-memory cached database version.
+    jsTest.log("About to do listCollections with readPref=primary");
+    assert.commandWorked(st.s.getDB(dbName).runCommand({listCollections: 1}));
+
+    // Ensure the current primary shard's primary has written the new database entry to disk.
+    st.rs0.getPrimary().adminCommand({_flushDatabaseCacheUpdates: dbName, syncFromConfig: false});
+
+    // Ensure the database entry on the current primary shard has replicated to the secondary.
+    st.rs0.awaitReplication();
+
+    // The primary shard's primary should have refreshed.
+    checkInMemoryDatabaseVersion(st.rs0.getPrimary(), dbName, dbEntry1.version);
+    checkInMemoryDatabaseVersion(st.rs1.getPrimary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs0.getSecondary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs1.getSecondary(), dbName, {});
+    checkOnDiskDatabaseVersion(st.rs0.getPrimary(), dbName, dbEntry1);
+    checkOnDiskDatabaseVersion(st.rs1.getPrimary(), dbName, undefined);
+    checkOnDiskDatabaseVersion(st.rs0.getSecondary(), dbName, dbEntry1);
+    checkOnDiskDatabaseVersion(st.rs1.getSecondary(), dbName, undefined);
+
+    // Now run a command that attaches databaseVersion with readPref=secondary to make the current
+    // primary shard's secondary refresh its in-memory database version from its on-disk entry.
+    jsTest.log("About to do listCollections with readPref=secondary");
+    assert.commandWorked(st.s.getDB(dbName).runCommand(
+        {listCollections: 1, $readPreference: {mode: "secondary"}, readConcern: {level: "local"}}));
+
+    // The primary shard's secondary should have refreshed.
+    checkInMemoryDatabaseVersion(st.rs0.getPrimary(), dbName, dbEntry1.version);
+    checkInMemoryDatabaseVersion(st.rs1.getPrimary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs0.getSecondary(), dbName, dbEntry1.version);
+    checkInMemoryDatabaseVersion(st.rs1.getSecondary(), dbName, {});
+    checkOnDiskDatabaseVersion(st.rs0.getPrimary(), dbName, dbEntry1);
+    checkOnDiskDatabaseVersion(st.rs1.getPrimary(), dbName, undefined);
+    checkOnDiskDatabaseVersion(st.rs0.getSecondary(), dbName, dbEntry1);
+    checkOnDiskDatabaseVersion(st.rs1.getSecondary(), dbName, undefined);
+
+    // Make "staleMongos" load the stale database info into memory.
+    const freshMongos = st.s0;
+    const staleMongos = st.s1;
+    staleMongos.getDB(dbName).runCommand({listCollections: 1});
+
+    // Run movePrimary to ensure the movePrimary critical section clears the in-memory cache on the
+    // old primary shard.
+    jsTest.log("About to do movePrimary");
+    assert.commandWorked(freshMongos.adminCommand({movePrimary: dbName, to: st.shard1.shardName}));
+    const dbEntry2 = freshMongos.getDB("config").getCollection("databases").findOne({_id: dbName});
+    assert.eq(dbEntry2.version.uuid, dbEntry1.version.uuid);
+    assert.eq(dbEntry2.version.lastMod, dbEntry1.version.lastMod + 1);
+
+    // Ensure the old primary shard's primary has written the 'enterCriticalSectionSignal' flag to
+    // its on-disk database entry.
+    st.rs0.getPrimary().adminCommand({_flushDatabaseCacheUpdates: dbName, syncFromConfig: false});
+
+    // Ensure 'enterCriticalSectionSignal' flag has replicated to the secondary.
+    st.rs0.awaitReplication();
+
+    // The in-memory cached version should have been cleared on the old primary shard's primary and
+    // secondary nodes.
+    checkInMemoryDatabaseVersion(st.rs0.getPrimary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs1.getPrimary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs0.getSecondary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs1.getSecondary(), dbName, {});
+    checkOnDiskDatabaseVersion(st.rs0.getPrimary(), dbName, dbEntry1);
+    checkOnDiskDatabaseVersion(st.rs1.getPrimary(), dbName, undefined);
+    checkOnDiskDatabaseVersion(st.rs0.getSecondary(), dbName, dbEntry1);
+    checkOnDiskDatabaseVersion(st.rs1.getSecondary(), dbName, undefined);
+
+    // Run listCollections with readPref=secondary from the stale mongos. First, this should cause
+    // the old primary shard's secondary to provoke the old primary shard's primary to refresh. Then
+    // once the stale mongos refreshes, it should cause the new primary shard's secondary to provoke
+    // the new primary shard's primary to refresh.
+    jsTest.log("About to do listCollections with readPref=secondary after movePrimary");
+    assert.commandWorked(staleMongos.getDB(dbName).runCommand(
+        {listCollections: 1, $readPreference: {mode: "secondary"}, readConcern: {level: "local"}}));
+
+    // All nodes should have refreshed.
+    checkInMemoryDatabaseVersion(st.rs0.getPrimary(), dbName, dbEntry2.version);
+    checkInMemoryDatabaseVersion(st.rs1.getPrimary(), dbName, dbEntry2.version);
+    checkInMemoryDatabaseVersion(st.rs0.getSecondary(), dbName, dbEntry2.version);
+    checkInMemoryDatabaseVersion(st.rs1.getSecondary(), dbName, dbEntry2.version);
+    checkOnDiskDatabaseVersion(st.rs0.getPrimary(), dbName, dbEntry2);
+    checkOnDiskDatabaseVersion(st.rs1.getPrimary(), dbName, dbEntry2);
+    checkOnDiskDatabaseVersion(st.rs0.getSecondary(), dbName, dbEntry2);
+    checkOnDiskDatabaseVersion(st.rs1.getSecondary(), dbName, dbEntry2);
+
+    // Ensure that dropping the database drops it from all shards, which clears their in-memory
+    // caches but not their on-disk caches.
+    jsTest.log("About to drop database from the cluster");
+    assert.commandWorked(freshMongos.getDB(dbName).runCommand({dropDatabase: 1}));
+
+    // Once SERVER-34431 goes in, this should not have caused the in-memory versions to be cleared.
+    checkInMemoryDatabaseVersion(st.rs0.getPrimary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs1.getPrimary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs0.getSecondary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs1.getSecondary(), dbName, {});
+    checkOnDiskDatabaseVersion(st.rs0.getPrimary(), dbName, dbEntry2);
+    checkOnDiskDatabaseVersion(st.rs1.getPrimary(), dbName, dbEntry2);
+    checkOnDiskDatabaseVersion(st.rs0.getSecondary(), dbName, dbEntry2);
+    checkOnDiskDatabaseVersion(st.rs1.getSecondary(), dbName, dbEntry2);
+
+    // Confirm that we have a bug (SERVER-34431), where if a database is dropped and recreated on a
+    // different shard, a stale mongos that has cached the old database's primary shard will *not*
+    // be routed to the new database's primary shard (and will see an empty database).
+
+    // Use 'enableSharding' to create the database only in the sharding catalog (the database will
+    // not exist on any shards).
+    assert.commandWorked(st.s.adminCommand({enableSharding: dbName}));
+
+    // Simulate that the database was created on 'shard0' by directly modifying the database entry
+    // (we cannot use movePrimary, since movePrimary creates the database on the shards).
+    const dbEntry = st.s.getDB("config").getCollection("databases").findOne({_id: dbName}).version;
+    assert.writeOK(st.s.getDB("config").getCollection("databases").update({_id: dbName}, {
+        $set: {primary: st.shard0.shardName}
+    }));
+
+    assert.commandWorked(st.s.getDB(dbName).runCommand({listCollections: 1}));
+
+    // Once SERVER-34431 goes in, this should have caused the primary shard's primary to refresh its
+    // in-memory and on-disk caches.
+    checkInMemoryDatabaseVersion(st.rs0.getPrimary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs1.getPrimary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs0.getSecondary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs1.getSecondary(), dbName, {});
+    checkOnDiskDatabaseVersion(st.rs0.getPrimary(), dbName, dbEntry2);
+    checkOnDiskDatabaseVersion(st.rs1.getPrimary(), dbName, dbEntry2);
+    checkOnDiskDatabaseVersion(st.rs0.getSecondary(), dbName, dbEntry2);
+    checkOnDiskDatabaseVersion(st.rs1.getSecondary(), dbName, dbEntry2);
+
+    assert.commandWorked(st.s.getDB(dbName).runCommand(
+        {listCollections: 1, $readPreference: {mode: "secondary"}, readConcern: {level: "local"}}));
+
+    // Once SERVER-34431 goes in, this should have caused the primary shard's secondary to refresh
+    // its in-memory cache (its on-disk cache was already updated when the primary refreshed).
+    checkInMemoryDatabaseVersion(st.rs0.getPrimary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs1.getPrimary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs0.getSecondary(), dbName, {});
+    checkInMemoryDatabaseVersion(st.rs1.getSecondary(), dbName, {});
+    checkOnDiskDatabaseVersion(st.rs0.getPrimary(), dbName, dbEntry2);
+    checkOnDiskDatabaseVersion(st.rs1.getPrimary(), dbName, dbEntry2);
+    checkOnDiskDatabaseVersion(st.rs0.getSecondary(), dbName, dbEntry2);
+    checkOnDiskDatabaseVersion(st.rs1.getSecondary(), dbName, dbEntry2);
+
+    st.stop();
+})();
diff --git a/src/mongo/s/catalog_cache.cpp b/src/mongo/s/catalog_cache.cpp
index 3980317ea00..3d2825198b6 100644
--- a/src/mongo/s/catalog_cache.cpp
+++ b/src/mongo/s/catalog_cache.cpp
@@ -122,34 +122,40 @@ CatalogCache::~CatalogCache() = default;
 StatusWith<CachedDatabaseInfo> CatalogCache::getDatabase(OperationContext* opCtx,
                                                          StringData dbName) {
     try {
-        stdx::lock_guard<stdx::mutex> lg(_mutex);
+        while (true) {
+            stdx::unique_lock<stdx::mutex> ul(_mutex);
 
-        auto& dbEntry = _databases[dbName];
-        if (!dbEntry) {
-            dbEntry = std::make_shared<DatabaseInfoEntry>();
-        }
+            auto& dbEntry = _databases[dbName];
+            if (!dbEntry) {
+                dbEntry = std::make_shared<DatabaseInfoEntry>();
+            }
 
-        if (dbEntry->needsRefresh) {
-            const auto catalogClient = Grid::get(opCtx)->catalogClient();
+            if (dbEntry->needsRefresh) {
+                auto refreshNotification = dbEntry->refreshCompletionNotification;
+                if (!refreshNotification) {
+                    refreshNotification = (dbEntry->refreshCompletionNotification =
+                                               std::make_shared<Notification<Status>>());
+                    _scheduleDatabaseRefresh(ul, dbName, dbEntry);
+                }
 
-            const auto dbNameCopy = dbName.toString();
+                // Wait on the notification outside of the mutex.
+                ul.unlock();
+                uassertStatusOK(refreshNotification->get(opCtx));
 
-            // Load the database entry
-            const auto opTimeWithDb = uassertStatusOK(catalogClient->getDatabase(
-                opCtx, dbNameCopy, repl::ReadConcernLevel::kMajorityReadConcern));
-            const auto& dbDesc = opTimeWithDb.value;
+                // Once the refresh is complete, loop around to get the refreshed cache entry.
+                continue;
+            }
 
-            if (!dbEntry->dbt) {
+            if (dbEntry->mustLoadShardedCollections) {
                 // If this is the first time we are loading info for this database, also load the
                 // sharded collections.
-                // TODO (SERVER-34061): Stop loading sharded collections here.
-                repl::OpTime collLoadConfigOptime;
-                const std::vector<CollectionType> collections = uassertStatusOK(
-                    catalogClient->getCollections(opCtx, &dbNameCopy, &collLoadConfigOptime));
+                // TODO (SERVER-34061): Stop loading sharded collections when loading a database.
 
-                const auto refreshCallbackFn = [](OperationContext* opCtx,
-                                                  StatusWith<DatabaseType>) {};
-                _cacheLoader.getDatabase(dbName, refreshCallbackFn);
+                const auto dbNameCopy = dbName.toString();
+                repl::OpTime collLoadConfigOptime;
+                const std::vector<CollectionType> collections =
+                    uassertStatusOK(Grid::get(opCtx)->catalogClient()->getCollections(
+                        opCtx, &dbNameCopy, &collLoadConfigOptime));
 
                 CollectionInfoMap collectionEntries;
                 for (const auto& coll : collections) {
@@ -160,15 +166,13 @@ StatusWith<CachedDatabaseInfo> CatalogCache::getDatabase(OperationContext* opCtx
                         std::make_shared<CollectionRoutingInfoEntry>();
                 }
                 _collectionsByDb[dbName] = std::move(collectionEntries);
+                dbEntry->mustLoadShardedCollections = false;
             }
 
-            dbEntry->needsRefresh = false;
-            dbEntry->dbt = std::move(dbDesc);
+            auto primaryShard = uassertStatusOK(
+                Grid::get(opCtx)->shardRegistry()->getShard(opCtx, dbEntry->dbt->getPrimary()));
+            return {CachedDatabaseInfo(*dbEntry->dbt, std::move(primaryShard))};
         }
-
-        auto primaryShard = uassertStatusOK(
-            Grid::get(opCtx)->shardRegistry()->getShard(opCtx, dbEntry->dbt->getPrimary()));
-        return {CachedDatabaseInfo(*dbEntry->dbt, std::move(primaryShard))};
     } catch (const DBException& ex) {
         return ex.toStatus();
     }
@@ -400,6 +404,70 @@ void CatalogCache::report(BSONObjBuilder* builder) const {
     _stats.report(&cacheStatsBuilder);
 }
 
+void CatalogCache::_scheduleDatabaseRefresh(WithLock,
+                                            const StringData dbName,
+                                            std::shared_ptr<DatabaseInfoEntry> dbEntry) {
+
+    log() << "Refreshing cached database entry for " << dbName
+          << "; current cached database info is "
+          << (dbEntry->dbt ? dbEntry->dbt->toBSON() : BSONObj());
+
+    const auto onRefreshCompleted =
+        [ this, t = Timer(), dbName ](const StatusWith<DatabaseType>& swDbt) {
+        // TODO (SERVER-34164): Track and increment stats for database refreshes.
+        if (!swDbt.isOK()) {
+            log() << "Refresh for database " << dbName << " took " << t.millis() << " ms and failed"
+                  << causedBy(redact(swDbt.getStatus()));
+            return;
+        }
+        log() << "Refresh for database " << dbName << " took " << t.millis() << " ms and found "
+              << swDbt.getValue().toBSON();
+    };
+
+    const auto onRefreshFailed =
+        [ this, dbName, dbEntry ](WithLock lk, const Status& status) noexcept {
+        // Clear the notification so the next 'getDatabase' kicks off a new refresh attempt.
+        dbEntry->refreshCompletionNotification->set(status);
+        dbEntry->refreshCompletionNotification = nullptr;
+
+        if (status == ErrorCodes::NamespaceNotFound) {
+            // The refresh found that the database was dropped, so remove its entry from the cache.
+            _databases.erase(dbName);
+            _collectionsByDb.erase(dbName);
+            return;
+        }
+    };
+
+    const auto onRefreshSucceeded = [this, dbName, dbEntry](WithLock lk, DatabaseType dbt) {
+        // Update the cached entry with the refreshed metadata and mark the entry as fresh.
+        dbEntry->dbt = std::move(dbt);
+        dbEntry->needsRefresh = false;
+        dbEntry->refreshCompletionNotification->set(Status::OK());
+        dbEntry->refreshCompletionNotification = nullptr;
+    };
+
+    const auto updateCatalogCacheFn =
+        [ this, dbName, dbEntry, onRefreshFailed, onRefreshSucceeded, onRefreshCompleted ](
+            OperationContext * opCtx, StatusWith<DatabaseType> swDbt) noexcept {
+        onRefreshCompleted(swDbt);
+        stdx::lock_guard<stdx::mutex> lg(_mutex);
+        if (!swDbt.isOK()) {
+            onRefreshFailed(lg, swDbt.getStatus());
+            return;
+        }
+        onRefreshSucceeded(lg, std::move(swDbt.getValue()));
+    };
+
+    try {
+        _cacheLoader.getDatabase(dbName, updateCatalogCacheFn);
+    } catch (const DBException& ex) {
+        const auto status = ex.toStatus();
+        stdx::lock_guard<stdx::mutex> lg(_mutex);
+        onRefreshCompleted(status);
+        onRefreshFailed(lg, status);
+    }
+}
+
 void CatalogCache::_scheduleCollectionRefresh(WithLock lk,
                                               std::shared_ptr<CollectionRoutingInfoEntry> collEntry,
                                               NamespaceString const& nss,
diff --git a/src/mongo/s/catalog_cache.h b/src/mongo/s/catalog_cache.h
index 9903f19b3ec..35fecda1081 100644
--- a/src/mongo/s/catalog_cache.h
+++ b/src/mongo/s/catalog_cache.h
@@ -198,11 +198,28 @@ private:
         // unset if the cache entry has never been loaded, or should not be relied on).
         bool needsRefresh{true};
 
+        // Contains a notification to be waited on for the refresh to complete (only available if
+        // needsRefresh is true)
+        std::shared_ptr<Notification<Status>> refreshCompletionNotification;
+
+        // Until SERVER-34061 goes in, after a database refresh, one thread should also load the
+        // sharded collections. In case multiple threads were queued up on the refresh, this bool
+        // ensures only the first loads the collections.
+        bool mustLoadShardedCollections{true};
+
         // Contains the cached info about the database (only available if needsRefresh is false)
         boost::optional<DatabaseType> dbt;
     };
 
     /**
+     * Non-blocking call which schedules an asynchronous refresh for the specified database. The
+     * database entry must be in the 'needsRefresh' state.
+     */
+    void _scheduleDatabaseRefresh(WithLock,
+                                  const StringData dbName,
+                                  std::shared_ptr<DatabaseInfoEntry> dbEntry);
+
+    /**
      * Non-blocking call which schedules an asynchronous refresh for the specified namespace. The
      * namespace must be in the 'needRefresh' state.
      */
diff --git a/src/mongo/s/config_server_catalog_cache_loader.cpp b/src/mongo/s/config_server_catalog_cache_loader.cpp
index f9baadd0d02..e6c7ecc138c 100644
--- a/src/mongo/s/config_server_catalog_cache_loader.cpp
+++ b/src/mongo/s/config_server_catalog_cache_loader.cpp
@@ -42,7 +42,6 @@
 namespace mongo {
 
 using CollectionAndChangedChunks = CatalogCacheLoader::CollectionAndChangedChunks;
-MONGO_FP_DECLARE(callShardServerCallbackFn);
 
 namespace {
 
@@ -198,25 +197,24 @@ std::shared_ptr<Notification<void>> ConfigServerCatalogCacheLoader::getChunksSin
 void ConfigServerCatalogCacheLoader::getDatabase(
     StringData dbName,
     stdx::function<void(OperationContext*, StatusWith<DatabaseType>)> callbackFn) {
+    uassertStatusOK(_threadPool.schedule([ name = dbName.toString(), callbackFn ]() noexcept {
+        auto opCtx = Client::getCurrent()->makeOperationContext();
 
-    if (MONGO_FAIL_POINT(callShardServerCallbackFn)) {
-        uassertStatusOK(_threadPool.schedule([ name = dbName.toString(), callbackFn ]() noexcept {
-            auto opCtx = Client::getCurrent()->makeOperationContext();
-
-            auto swDbt = [&]() -> StatusWith<DatabaseType> {
-                try {
-
-                    const auto dbVersion = databaseVersion::makeNew();
-                    DatabaseType dbt(std::move(name), ShardId("PrimaryShard"), false, dbVersion);
-                    return dbt;
-                } catch (const DBException& ex) {
-                    return ex.toStatus();
-                }
-            }();
+        auto swDbt = [&]() -> StatusWith<DatabaseType> {
+            try {
+                return uassertStatusOK(
+                           Grid::get(opCtx.get())
+                               ->catalogClient()
+                               ->getDatabase(
+                                   opCtx.get(), name, repl::ReadConcernLevel::kMajorityReadConcern))
+                    .value;
+            } catch (const DBException& ex) {
+                return ex.toStatus();
+            }
+        }();
 
-            callbackFn(opCtx.get(), swDbt);
-        }));
-    }
+        callbackFn(opCtx.get(), std::move(swDbt));
+    }));
 }
 
 }  // namespace mongo