SERVER-43713 SERVER-43715 Create DistCache abstraction, use it in ReadWriteConcernDefaults

22 files changed, 858 insertions, 47 deletions

diff --git a/src/mongo/SConscript b/src/mongo/SConscript
index fa3f27e5b14..8a1a2690dc6 100644
--- a/src/mongo/SConscript
+++ b/src/mongo/SConscript
@@ -354,6 +354,7 @@ mongod = env.Program(
     target="mongod",
     source=[
         "db/db.cpp",
+        "db/read_write_concern_defaults_cache_lookup_mongod.cpp",
     ] + env.WindowsResourceFile("db/db.rc"),
     LIBDEPS=[
         '$BUILD_DIR/third_party/shim_snappy',
@@ -404,6 +405,7 @@ mongod = env.Program(
         'db/pipeline/process_interface_factory_mongod',
         'db/query_exec',
         'db/read_concern_d_impl',
+        'db/read_write_concern_defaults',
         'db/repair_database_and_check_version',
         'db/repl/bgsync',
         'db/repl/oplog_application',
@@ -522,6 +524,7 @@ if not hygienic:
 mongos = env.Program(
     target='mongos',
     source=[
+        "db/read_write_concern_defaults_cache_lookup_mongos.cpp",
         's/cluster_cursor_stats.cpp',
         's/mongos_options.cpp',
         's/mongos_options_init.cpp',
@@ -545,6 +548,7 @@ mongos = env.Program(
         'db/logical_session_cache',
         'db/logical_session_cache_impl',
         'db/logical_time_metadata_hook',
+        'db/read_write_concern_defaults',
         'db/server_options',
         'db/server_options_base',
         'db/service_liaison_mongos',
diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript
index 6d5be85cbcc..04bdfba8274 100644
--- a/src/mongo/db/SConscript
+++ b/src/mongo/db/SConscript
@@ -59,7 +59,6 @@ env.Library(
         '$BUILD_DIR/mongo/base',
         '$BUILD_DIR/mongo/db/index_names',
         '$BUILD_DIR/mongo/db/write_concern_options',
-        '$BUILD_DIR/mongo/db/read_write_concern_defaults',
     ]
 )
 
@@ -386,6 +385,16 @@ env.Library(
 )
 
 env.Library(
+    target="read_write_concern_defaults_mock",
+    source=[
+        "read_write_concern_defaults_cache_lookup_mock.cpp",
+    ],
+    LIBDEPS=[
+        'read_write_concern_defaults',
+    ],
+)
+
+env.Library(
     target='service_context',
     source=[
         'baton.cpp',
@@ -874,6 +883,7 @@ env.Library(
         "concurrency/lock_manager",
         "concurrency/write_conflict_exception",
         "curop",
+        'read_write_concern_defaults',
         "repl/read_concern_args",
         "repl/repl_coordinator_interface",
         "repl/speculative_majority_read_info",
@@ -1810,6 +1820,7 @@ envWithAsio.CppUnitTest(
         'op_observer_impl',
         'query_exec',
         'range_arithmetic',
+        'read_write_concern_defaults_mock',
         'repl/mock_repl_coord_server_fixture',
         'repl/oplog_interface_local',
         'repl/repl_coordinator_interface',
diff --git a/src/mongo/db/commands/SConscript b/src/mongo/db/commands/SConscript
index 38c77097e9f..3bdfb73e08d 100644
--- a/src/mongo/db/commands/SConscript
+++ b/src/mongo/db/commands/SConscript
@@ -148,6 +148,9 @@ env.Library(
         env.Idlc('drop_connections.idl')[0],
         env.Idlc('rwc_defaults_commands.idl')[0],
     ],
+    LIBDEPS=[
+        '$BUILD_DIR/mongo/db/read_write_concern_defaults',
+    ],
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/client/clientdriver_minimal',
         '$BUILD_DIR/mongo/db/commands',
diff --git a/src/mongo/db/commands/rwc_defaults_commands.cpp b/src/mongo/db/commands/rwc_defaults_commands.cpp
index 3389cbe79b2..e251d4dfdc5 100644
--- a/src/mongo/db/commands/rwc_defaults_commands.cpp
+++ b/src/mongo/db/commands/rwc_defaults_commands.cpp
@@ -113,7 +113,7 @@ public:
 
         auto typedRun(OperationContext* opCtx) {
             auto& rwcDefaults = ReadWriteConcernDefaults::get(opCtx->getServiceContext());
-            return rwcDefaults.getDefault();
+            return rwcDefaults.getDefault(opCtx);
         }
 
     private:
diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index bc669196f8c..fb02bce6ed4 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -102,6 +102,7 @@
 #include "mongo/db/periodic_runner_job_abort_expired_transactions.h"
 #include "mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.h"
 #include "mongo/db/query/internal_plans.h"
+#include "mongo/db/read_write_concern_defaults_cache_lookup_mongod.h"
 #include "mongo/db/repair_database_and_check_version.h"
 #include "mongo/db/repl/drop_pending_collection_reaper.h"
 #include "mongo/db/repl/oplog.h"
@@ -1092,6 +1093,8 @@ int mongoDbMain(int argc, char* argv[], char** envp) {
     // initializeServerGlobalState) and before the creation of any other threads
     startSignalProcessingThread();
 
+    ReadWriteConcernDefaults::create(service, readWriteConcernDefaultsCacheLookupMongoD);
+
 #if defined(_WIN32)
     if (ntservice::shouldStartService()) {
         ntservice::startService();
diff --git a/src/mongo/db/dist_cache.h b/src/mongo/db/dist_cache.h
new file mode 100644
index 00000000000..b2b7beb8302
--- /dev/null
+++ b/src/mongo/db/dist_cache.h
@@ -0,0 +1,408 @@
+/**
+ *    Copyright (C) 2019-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include <boost/optional.hpp>
+
+#include "mongo/bson/oid.h"
+#include "mongo/platform/mutex.h"
+#include "mongo/stdx/condition_variable.h"
+#include "mongo/util/invalidating_lru_cache.h"
+
+namespace mongo {
+
+class OperationContext;
+
+/**
+ * A server's cache of cluster-wide information.
+ */
+template <typename Key, typename UnderlyingValue>
+class DistCache {
+public:
+    /**
+     * Wrapper around Value that adds an atomic bool to indicate validity.
+     *
+     * Every Value object is owned by a DistCache.  The DistCache is the only one that should
+     * construct, modify, or delete a Value object.  All other consumers of Value only get const
+     * access to the underlying value.  The DistCache is responsible for maintaining the reference
+     * count on all Value objects it gives out and must not mutate any Value objects with a non-zero
+     * reference count (except to call invalidate()).  Any consumer of a Value object should check
+     * isValid() before using it, and if it has been invalidated, it should return the object to the
+     * DistCache and fetch a new Value object instance for this Key from the DistCache.
+     */
+    class Value {
+        Value(const Value&) = delete;
+        Value& operator=(const Value&) = delete;
+
+    public:
+        explicit Value(UnderlyingValue&& underlyingValue)
+            : _underlyingValue(std::move(underlyingValue)) {}
+
+        UnderlyingValue& operator*() {
+            return _underlyingValue;
+        }
+
+        const UnderlyingValue& operator*() const {
+            return _underlyingValue;
+        }
+
+        /**
+         * Returns true if this copy of information about this value is still valid. If this returns
+         * false, this object should no longer be used and should be returned to the
+         * DistCache and a new Value object for this Key should be requested.
+         */
+        bool isValid() const {
+            return _isValid.loadRelaxed();
+        }
+
+    private:
+        friend class DistCache;
+
+        /**
+         * Marks this instance of the Value object as invalid, most likely because the underlying
+         * value has been updated and needs to be reloaded by the DistCache.
+         *
+         * This method should *only* be called by the DistCache.
+         */
+        void _invalidate() {
+            _isValid.store(false);
+        }
+
+        // The underlying value.
+        UnderlyingValue _underlyingValue;
+
+        // Indicates whether the value has been marked as invalid by the DistCache.
+        AtomicWord<bool> _isValid{true};
+    };
+
+    using ValueHandle = std::shared_ptr<Value>;
+
+    DistCache(const DistCache&) = delete;
+    DistCache& operator=(const DistCache&) = delete;
+
+    virtual ~DistCache() = default;
+
+    /**
+     * Returns the cache generation identifier.
+     */
+    OID getCacheGeneration() {
+        stdx::lock_guard<Latch> lk(_cacheWriteMutex);
+        return _fetchGeneration;
+    }
+
+    /**
+     * Returns a ValueHandle for the given Key. If there is no entry for this key, then returns
+     * boost::none. If the cache already has a value for this key, it returns a handle from the
+     * cache, otherwise it obtains the value from the Lookup - this may block for a long time.
+     *
+     * The returned value may be invalid by the time the caller gets access to it.
+     */
+    boost::optional<ValueHandle> acquire(OperationContext* opCtx, const Key& key) {
+        boost::optional<ValueHandle> cachedValue = _cache.get(key);
+
+        // The _cache is thread-safe, so if we can get a Value out of the cache we don't need to
+        // take any locks!
+        if (cachedValue) {
+            invariant(*cachedValue);
+            return *cachedValue;
+        }
+
+        // Otherwise make sure we have the locks we need and check whether and wait on another
+        // thread is fetching into the cache
+        CacheGuard guard(this);
+
+        while (!(cachedValue = _cache.get(key)) && guard.otherUpdateInFetchPhase()) {
+            guard.wait();
+        }
+
+        if (cachedValue) {
+            invariant(*cachedValue);
+            return *cachedValue;
+        }
+
+        // If there's still no value in the cache, then we need to go and get it. Take the slow
+        // path.
+
+        guard.beginFetchPhase();
+
+        auto underlyingValue = lookup(opCtx, key);
+        if (!underlyingValue)
+            return boost::none;
+        auto value = std::make_unique<Value>(std::move(*underlyingValue));
+
+        // All this does is re-acquire the _cacheWriteMutex if we don't hold it already - a caller
+        // may also call endFetchPhase() after this returns.
+        guard.endFetchPhase();
+
+        ValueHandle ret;
+        if (guard.isSameCacheGeneration()) {
+            ret = _cache.insertOrAssignAndGet(key, std::move(value));
+        } else {
+            // If the cache generation changed while this thread was in fetch mode, the data
+            // associated with the value may now be invalid, so we must mark it as such.  The caller
+            // may still opt to use the information for a short while, but not indefinitely.
+            value->_invalidate();
+            ret = ValueHandle(std::move(value));
+        }
+
+        return ret;
+    }
+
+    /**
+     * Marks the given value as invalid and removes it from the cache.
+     */
+    void invalidate(const Key& key) {
+        CacheGuard guard(this);
+        _updateCacheGeneration(guard);
+        _cache.invalidate(key);
+    }
+
+    template <typename Pred>
+    void invalidateIf(const Pred& predicate) {
+        CacheGuard guard(this);
+        _updateCacheGeneration(guard);
+        _cache.invalidateIf(
+            [&](const Key& key, const Value* value) { return predicate(key, &(*(*value))); });
+    }
+
+    /**
+     * Invalidates the given value and immediately replaces it with a new value.
+     */
+    ValueHandle revalidate(const Key& key, UnderlyingValue&& newUnderlyingValue) {
+        CacheGuard guard(this);
+
+        // Invalidate the old value.
+        _cache.invalidate(key);
+
+        // Put the new value into the cache.
+        auto newValue = std::make_unique<Value>(std::move(newUnderlyingValue));
+        ValueHandle ret = _cache.insertOrAssignAndGet(key, std::move(newValue));
+
+        // Update the cache generation.
+        _updateCacheGeneration(guard);
+
+        return ret;
+    }
+
+    /**
+     * Invalidates all Value objects in the cache and removes them from the cache.
+     */
+    void invalidateCache() {
+        CacheGuard guard(this);
+        _updateCacheGeneration(guard);
+        _cache.invalidateIf([](const Key& a, const Value*) { return true; });
+    }
+
+protected:
+    /**
+     * DistCache constructor, to be called by sub-classes.  Accepts the initial size of the cache,
+     * and a reference to a Mutex.  The Mutex is for the exclusive use of the DistCache, the
+     * sub-class should never actually use it (apart from passing it to this constructor).  Having
+     * the Mutex stored by the sub-class allows latch diagnostics to be correctly associated with
+     * the sub-class (not the generic DistCache class).
+     */
+    DistCache(int cacheSize, Mutex& mutex)
+        : _cache(cacheSize, Invalidator()), _cacheWriteMutex(mutex), _fetchGeneration(OID::gen()) {}
+
+private:
+    /**
+     * Type used to guard accesses and updates to the cache.
+     *
+     * Guard object for synchronizing accesses to data cached in DistCache instances.
+     * This guard allows one thread to access the cache at a time, and provides an exception-safe
+     * mechanism for a thread to release the cache mutex while performing network or disk operations
+     * while allowing other readers to proceed.
+     *
+     * There are two ways to use this guard.  One may simply instantiate the guard like a
+     * std::lock_guard, and perform reads or writes of the cache.
+     *
+     * Alternatively, one may instantiate the guard, examine the cache, and then enter into an
+     * update mode by first wait()ing until otherUpdateInFetchPhase() is false, and then
+     * calling beginFetchPhase().  At this point, other threads may acquire the guard in the simple
+     * manner and do reads, but other threads may not enter into a fetch phase.  During the fetch
+     * phase, the thread should perform required network or disk activity to determine what update
+     * it will make to the cache.  Then, it should call endFetchPhase(), to reacquire the cache
+     * mutex.  At that point, the thread can make its modifications to the cache and let the guard
+     * go out of scope.
+     *
+     * All updates by guards using a fetch-phase are totally ordered with respect to one another,
+     * and all guards using no fetch phase are totally ordered with respect to one another, but
+     * there is not a total ordering among all guard objects.
+     *
+     * The cached data has an associated counter, called the cache generation.  If the cache
+     * generation changes while a guard is in fetch phase, the fetched data should not be stored
+     * into the cache, because some invalidation event occurred during the fetch phase.
+     */
+    class CacheGuard {
+        CacheGuard(const CacheGuard&) = delete;
+        CacheGuard& operator=(const CacheGuard&) = delete;
+
+    public:
+        /**
+         * Constructs a cache guard, locking the mutex that synchronizes DistCache accesses.
+         */
+        explicit CacheGuard(DistCache* distCache)
+            : _isThisGuardInFetchPhase(false),
+              _distCache(distCache),
+              _cacheLock(distCache->_cacheWriteMutex) {}
+
+        /**
+         * Releases the mutex that synchronizes cache access, if held, and notifies
+         * any threads waiting for their own opportunity to update the cache.
+         */
+        ~CacheGuard() {
+            if (!_cacheLock.owns_lock()) {
+                _cacheLock.lock();
+            }
+            if (_isThisGuardInFetchPhase) {
+                invariant(otherUpdateInFetchPhase());
+                _distCache->_isFetchPhaseBusy = false;
+                _distCache->_fetchPhaseIsReady.notify_all();
+            }
+        }
+
+        /**
+         * Returns true if the distCache reports that it is in fetch phase.
+         */
+        bool otherUpdateInFetchPhase() const {
+            return _distCache->_isFetchPhaseBusy;
+        }
+
+        /**
+         * Waits on the _distCache->_fetchPhaseIsReady condition.
+         */
+        void wait() {
+            invariant(!_isThisGuardInFetchPhase);
+            _distCache->_fetchPhaseIsReady.wait(_cacheLock,
+                                                [&] { return !otherUpdateInFetchPhase(); });
+        }
+
+        /**
+         * Enters fetch phase, releasing the _distCache->_cacheMutex after recording the current
+         * cache generation.
+         */
+        void beginFetchPhase() {
+            invariant(!otherUpdateInFetchPhase());
+            _isThisGuardInFetchPhase = true;
+            _distCache->_isFetchPhaseBusy = true;
+            _distCacheFetchGenerationAtFetchBegin = _distCache->_fetchGeneration;
+            _cacheLock.unlock();
+        }
+
+        /**
+         * Exits the fetch phase, reacquiring the _distCache->_cacheMutex.
+         */
+        void endFetchPhase() {
+            _cacheLock.lock();
+            // We do not clear _distCache->_isFetchPhaseBusy or notify waiters until
+            // ~CacheGuard(), for two reasons.  First, there's no value to notifying the waiters
+            // before you're ready to release the mutex, because they'll just go to sleep on the
+            // mutex.  Second, in order to meaningfully check the preconditions of
+            // isSameCacheGeneration(), we need a state that means "fetch phase was entered and now
+            // has been exited."  That state is _isThisGuardInFetchPhase == true and
+            // _lock.owns_lock() == true.
+        }
+
+        /**
+         * Returns true if _distCache->_fetchGeneration remained the same while this guard was
+         * in fetch phase.  Behavior is undefined if this guard never entered fetch phase.
+         *
+         * If this returns true, do not update the cached data with this
+         */
+        bool isSameCacheGeneration() const {
+            invariant(_isThisGuardInFetchPhase);
+            invariant(_cacheLock.owns_lock());
+            return _distCacheFetchGenerationAtFetchBegin == _distCache->_fetchGeneration;
+        }
+
+    private:
+        OID _distCacheFetchGenerationAtFetchBegin;
+        bool _isThisGuardInFetchPhase;
+        DistCache* _distCache;
+
+        stdx::unique_lock<Latch> _cacheLock;
+    };
+
+    friend class DistCache::CacheGuard;
+
+    class Invalidator {
+    public:
+        void operator()(Value* value) {
+            value->_invalidate();
+        }
+    };
+
+    /**
+     * Provide the value for a key when there is a cache miss.  Sub-classes must implement this
+     * function appropriately.  Throw a uassertion to indicate an error while looking up the value,
+     * or return value for this key, o rboost::none if this key has no value.
+     */
+    virtual boost::optional<UnderlyingValue> lookup(OperationContext* opCtx, const Key& key) = 0;
+
+    /**
+     * Updates _fetchGeneration to a new OID
+     */
+    void _updateCacheGeneration(const CacheGuard&) {
+        _fetchGeneration = OID::gen();
+    }
+
+    /**
+     * The cache itself is "self-synchronising" (ie. thread-safe), and so does not need to be
+     * protected by _cacheWriteMutex.
+     */
+    InvalidatingLRUCache<Key, Value, Invalidator> _cache;
+
+    /**
+     * Protects _fetchGeneration and _isFetchPhaseBusy.  Manipulated via CacheGuard.
+     */
+    Mutex& _cacheWriteMutex;
+
+    /**
+     * Current generation of cached data.  Updated every time part of the cache gets
+     * invalidated.  Protected by CacheGuard.
+     */
+    OID _fetchGeneration;
+
+    /**
+     * True if there is an update to the _cache in progress, and that update is currently in
+     * the "fetch phase", during which it does not hold the _cacheMutex.
+     *
+     * Manipulated via CacheGuard.
+     */
+    bool _isFetchPhaseBusy = false;
+
+    /**
+     * Condition used to signal that it is OK for another CacheGuard to enter a fetch phase.
+     * Manipulated via CacheGuard.
+     */
+    stdx::condition_variable _fetchPhaseIsReady;
+};
+
+}  // namespace mongo
diff --git a/src/mongo/db/read_write_concern_defaults.cpp b/src/mongo/db/read_write_concern_defaults.cpp
index 267ee413045..1e2c2197c83 100644
--- a/src/mongo/db/read_write_concern_defaults.cpp
+++ b/src/mongo/db/read_write_concern_defaults.cpp
@@ -76,9 +76,8 @@ void ReadWriteConcernDefaults::checkSuitabilityAsDefault(const WriteConcern& wc)
             !(wc.wMode.empty() && wc.wNumNodes < 1));
 }
 
-void ReadWriteConcernDefaults::_setDefault(WithLock, RWConcernDefault&& rwc) {
-    _defaults.erase(kReadWriteConcernEntry);
-    _defaults.emplace(kReadWriteConcernEntry, rwc);
+void ReadWriteConcernDefaults::_setDefault(RWConcernDefault&& rwc) {
+    _defaults.revalidate(Type::kReadWriteConcernEntry, std::move(rwc));
 }
 
 RWConcernDefault ReadWriteConcernDefaults::setConcerns(OperationContext* opCtx,
@@ -103,68 +102,80 @@ RWConcernDefault ReadWriteConcernDefaults::setConcerns(OperationContext* opCtx,
     rwc.setSetTime(now);
     rwc.setLocalSetTime(now);
 
-    stdx::lock_guard<Latch> lk(_mutex);
-
-    auto current = _getDefault(lk);
+    auto current = _getDefault(opCtx);
     if (!rc && current) {
         rwc.setDefaultReadConcern(current->getDefaultReadConcern());
     }
     if (!wc && current) {
         rwc.setDefaultWriteConcern(current->getDefaultWriteConcern());
     }
-    _setDefault(lk, std::move(rwc));
-    return *_getDefault(lk);
+    _setDefault(std::move(rwc));
+    return *_getDefault(opCtx);
 }
 
 void ReadWriteConcernDefaults::invalidate() {
-    stdx::lock_guard<Latch> lk(_mutex);
-    _defaults.erase(kReadWriteConcernEntry);
+    _defaults.invalidate(Type::kReadWriteConcernEntry);
 }
 
-boost::optional<RWConcernDefault> ReadWriteConcernDefaults::_getDefault(WithLock) const {
-    if (_defaults.find(kReadWriteConcernEntry) == _defaults.end()) {
-        return boost::none;
+boost::optional<RWConcernDefault> ReadWriteConcernDefaults::_getDefault(OperationContext* opCtx) {
+    auto defaultsHandle = _defaults.acquire(opCtx, Type::kReadWriteConcernEntry);
+    if (defaultsHandle) {
+        auto& defaultsValue = **defaultsHandle;
+        // Since CWRWC is ok with continuing to use a value well after it has been invalidated
+        // (since RWC defaults apply for the lifetime of the op/cursor), we don't need to check
+        // defaultsValue.isValid() here, and we don't need to return the Handle, since callers don't
+        // need to check defaultsValue.isValid() later, either.  Just dereference it to get the
+        // underlying contents.
+        return *defaultsValue;
     }
-    return _defaults.at(kReadWriteConcernEntry);
+    return boost::none;
 }
 
-RWConcernDefault ReadWriteConcernDefaults::getDefault() const {
-    auto current = ([&]() {
-        stdx::lock_guard<Latch> lk(_mutex);
-        return _getDefault(lk);
-    })();
-    if (!current) {
-        return RWConcernDefault{};
-    }
-    return *current;
+RWConcernDefault ReadWriteConcernDefaults::getDefault(OperationContext* opCtx) {
+    return _getDefault(opCtx).value_or(RWConcernDefault());
 }
 
 boost::optional<ReadWriteConcernDefaults::ReadConcern>
-ReadWriteConcernDefaults::getDefaultReadConcern() const {
-    auto current = getDefault();
+ReadWriteConcernDefaults::getDefaultReadConcern(OperationContext* opCtx) {
+    auto current = getDefault(opCtx);
     return current.getDefaultReadConcern();
 }
 
 boost::optional<ReadWriteConcernDefaults::WriteConcern>
-ReadWriteConcernDefaults::getDefaultWriteConcern() const {
-    auto current = getDefault();
+ReadWriteConcernDefaults::getDefaultWriteConcern(OperationContext* opCtx) {
+    auto current = getDefault(opCtx);
     return current.getDefaultWriteConcern();
 }
 
-
 namespace {
 
 const auto getReadWriteConcernDefaults =
-    ServiceContext::declareDecoration<ReadWriteConcernDefaults>();
+    ServiceContext::declareDecoration<std::unique_ptr<ReadWriteConcernDefaults>>();
 
 }  // namespace
 
 ReadWriteConcernDefaults& ReadWriteConcernDefaults::get(ServiceContext* service) {
-    return getReadWriteConcernDefaults(service);
+    return *getReadWriteConcernDefaults(service);
 }
 
 ReadWriteConcernDefaults& ReadWriteConcernDefaults::get(ServiceContext& service) {
-    return getReadWriteConcernDefaults(service);
+    return *getReadWriteConcernDefaults(service);
+}
+
+void ReadWriteConcernDefaults::create(ServiceContext* service, LookupFn lookupFn) {
+    getReadWriteConcernDefaults(service) = std::make_unique<ReadWriteConcernDefaults>(lookupFn);
+}
+
+ReadWriteConcernDefaults::ReadWriteConcernDefaults(LookupFn lookupFn) : _defaults(lookupFn) {}
+
+ReadWriteConcernDefaults::Cache::Cache(LookupFn lookupFn)
+    : DistCache(1, _mutex), _lookupFn(lookupFn) {}
+
+boost::optional<RWConcernDefault> ReadWriteConcernDefaults::Cache::lookup(
+    OperationContext* opCtx, const ReadWriteConcernDefaults::Type& key) {
+    invariant(key == Type::kReadWriteConcernEntry);
+    // TODO: failpoint to uassert a custom Status
+    return _lookupFn(opCtx);
 }
 
 }  // namespace mongo
diff --git a/src/mongo/db/read_write_concern_defaults.h b/src/mongo/db/read_write_concern_defaults.h
index cb8b6386ae8..de21b27836d 100644
--- a/src/mongo/db/read_write_concern_defaults.h
+++ b/src/mongo/db/read_write_concern_defaults.h
@@ -31,6 +31,7 @@
 
 #include <map>
 
+#include "mongo/db/dist_cache.h"
 #include "mongo/db/operation_context.h"
 #include "mongo/db/repl/read_concern_args.h"
 #include "mongo/db/rw_concern_default_gen.h"
@@ -52,13 +53,17 @@ public:
     using ReadConcern = repl::ReadConcernArgs;
     using WriteConcern = WriteConcernOptions;
 
+    using LookupFn = std::function<boost::optional<RWConcernDefault>(OperationContext*)>;
+
     static constexpr StringData readConcernFieldName = ReadConcern::kReadConcernFieldName;
     static constexpr StringData writeConcernFieldName = WriteConcern::kWriteConcernField;
 
     static ReadWriteConcernDefaults& get(ServiceContext* service);
     static ReadWriteConcernDefaults& get(ServiceContext& service);
+    static void create(ServiceContext* service, LookupFn lookupFn);
 
-    ReadWriteConcernDefaults() = default;
+    ReadWriteConcernDefaults() = delete;
+    ReadWriteConcernDefaults(LookupFn lookupFn);
     ~ReadWriteConcernDefaults() = default;
 
     /**
@@ -92,20 +97,34 @@ public:
      */
     void invalidate();
 
-    RWConcernDefault getDefault() const;
-    boost::optional<ReadConcern> getDefaultReadConcern() const;
-    boost::optional<WriteConcern> getDefaultWriteConcern() const;
+    RWConcernDefault getDefault(OperationContext* opCtx);
+    boost::optional<ReadConcern> getDefaultReadConcern(OperationContext* opCtx);
+    boost::optional<WriteConcern> getDefaultWriteConcern(OperationContext* opCtx);
 
 private:
-    enum Type { kReadWriteConcernEntry };
+    enum class Type { kReadWriteConcernEntry };
+
+    void _setDefault(RWConcernDefault&& rwc);
+    boost::optional<RWConcernDefault> _getDefault(OperationContext* opCtx);
+
+    class Cache : public DistCache<Type, RWConcernDefault> {
+        Cache(const Cache&) = delete;
+        Cache& operator=(const Cache&) = delete;
+
+    public:
+        Cache(LookupFn lookupFn);
+        virtual ~Cache() = default;
+
+    private:
+        boost::optional<RWConcernDefault> lookup(OperationContext* opCtx, const Type& key) override;
 
-    void _setDefault(WithLock, RWConcernDefault&& rwc);
-    boost::optional<RWConcernDefault> _getDefault(WithLock) const;
+        // For exclusive use by DistCache only.
+        Mutex _mutex = MONGO_MAKE_LATCH("ReadWriteConcernDefaults::Cache");
 
-    // Protects access to the private members below.
-    mutable Mutex _mutex = MONGO_MAKE_LATCH("ReadWriteConcernDefaults::_mutex");
+        LookupFn _lookupFn;
+    };
 
-    std::map<Type, RWConcernDefault> _defaults;
+    Cache _defaults;
 };
 
 }  // namespace mongo
diff --git a/src/mongo/db/read_write_concern_defaults_cache_lookup_mock.cpp b/src/mongo/db/read_write_concern_defaults_cache_lookup_mock.cpp
new file mode 100644
index 00000000000..2a6b14cd18b
--- /dev/null
+++ b/src/mongo/db/read_write_concern_defaults_cache_lookup_mock.cpp
@@ -0,0 +1,47 @@
+/**
+ *    Copyright (C) 2019-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/db/read_write_concern_defaults_cache_lookup_mock.h"
+
+namespace mongo {
+
+ReadWriteConcernDefaults::LookupFn ReadWriteConcernDefaultsLookupMock::getLookupFn() {
+    return std::bind(&ReadWriteConcernDefaultsLookupMock::lookup, this, std::placeholders::_1);
+}
+
+boost::optional<RWConcernDefault> ReadWriteConcernDefaultsLookupMock::lookup(
+    OperationContext* opCtx) {
+    return _value;
+}
+
+void ReadWriteConcernDefaultsLookupMock::setLookupCallReturnValue(RWConcernDefault&& rwc) {
+    _value.emplace(rwc);
+}
+
+}  // namespace mongo
diff --git a/src/mongo/db/read_write_concern_defaults_cache_lookup_mock.h b/src/mongo/db/read_write_concern_defaults_cache_lookup_mock.h
new file mode 100644
index 00000000000..7913f576ef4
--- /dev/null
+++ b/src/mongo/db/read_write_concern_defaults_cache_lookup_mock.h
@@ -0,0 +1,54 @@
+/**
+ *    Copyright (C) 2019-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/db/read_write_concern_defaults.h"
+
+namespace mongo {
+
+/**
+ * A class which handles looking up RWConcernDefault values from an in-memory location.
+ */
+class ReadWriteConcernDefaultsLookupMock {
+public:
+    ReadWriteConcernDefaults::LookupFn getLookupFn();
+
+    boost::optional<RWConcernDefault> lookup(OperationContext* opCtx);
+
+    /**
+     * Behind-the-scenes way to update the stored in-memory value that lookup() returns.
+     */
+    void setLookupCallReturnValue(RWConcernDefault&& value);
+
+private:
+    boost::optional<RWConcernDefault> _value;
+};
+
+}  // namespace mongo
diff --git a/src/mongo/db/read_write_concern_defaults_cache_lookup_mongod.cpp b/src/mongo/db/read_write_concern_defaults_cache_lookup_mongod.cpp
new file mode 100644
index 00000000000..f807f67c204
--- /dev/null
+++ b/src/mongo/db/read_write_concern_defaults_cache_lookup_mongod.cpp
@@ -0,0 +1,39 @@
+/**
+ *    Copyright (C) 2019-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/db/read_write_concern_defaults_cache_lookup_mongod.h"
+
+namespace mongo {
+
+boost::optional<RWConcernDefault> readWriteConcernDefaultsCacheLookupMongoD(
+    OperationContext* opCtx) {
+    return boost::none;
+}
+
+}  // namespace mongo
diff --git a/src/mongo/db/read_write_concern_defaults_cache_lookup_mongod.h b/src/mongo/db/read_write_concern_defaults_cache_lookup_mongod.h
new file mode 100644
index 00000000000..44306249a1c
--- /dev/null
+++ b/src/mongo/db/read_write_concern_defaults_cache_lookup_mongod.h
@@ -0,0 +1,43 @@
+/**
+ *    Copyright (C) 2019-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/db/read_write_concern_defaults.h"
+
+namespace mongo {
+
+/**
+ * A function which handles looking up RWConcernDefault values from where they are persisted in
+ * config.settings.
+ */
+boost::optional<RWConcernDefault> readWriteConcernDefaultsCacheLookupMongoD(
+    OperationContext* opCtx);
+
+}  // namespace mongo
diff --git a/src/mongo/db/read_write_concern_defaults_cache_lookup_mongos.cpp b/src/mongo/db/read_write_concern_defaults_cache_lookup_mongos.cpp
new file mode 100644
index 00000000000..907d804cce8
--- /dev/null
+++ b/src/mongo/db/read_write_concern_defaults_cache_lookup_mongos.cpp
@@ -0,0 +1,39 @@
+/**
+ *    Copyright (C) 2019-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/db/read_write_concern_defaults_cache_lookup_mongos.h"
+
+namespace mongo {
+
+boost::optional<RWConcernDefault> readWriteConcernDefaultsCacheLookupMongoS(
+    OperationContext* opCtx) {
+    return boost::none;
+}
+
+}  // namespace mongo
diff --git a/src/mongo/db/read_write_concern_defaults_cache_lookup_mongos.h b/src/mongo/db/read_write_concern_defaults_cache_lookup_mongos.h
new file mode 100644
index 00000000000..d99b076e6f7
--- /dev/null
+++ b/src/mongo/db/read_write_concern_defaults_cache_lookup_mongos.h
@@ -0,0 +1,42 @@
+/**
+ *    Copyright (C) 2019-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/db/read_write_concern_defaults.h"
+
+namespace mongo {
+
+/**
+ * A function which handles looking up RWConcernDefault values from config servers.
+ */
+boost::optional<RWConcernDefault> readWriteConcernDefaultsCacheLookupMongoS(
+    OperationContext* opCtx);
+
+}  // namespace mongo
diff --git a/src/mongo/db/service_entry_point_common.cpp b/src/mongo/db/service_entry_point_common.cpp
index 58db096f7c0..7bbbe76d453 100644
--- a/src/mongo/db/service_entry_point_common.cpp
+++ b/src/mongo/db/service_entry_point_common.cpp
@@ -260,7 +260,7 @@ StatusWith<repl::ReadConcernArgs> _extractReadConcern(OperationContext* opCtx,
             // since this covers both isSpecified() && !isSpecified()
             if (readConcernArgs.isEmpty()) {
                 const auto rcDefault = ReadWriteConcernDefaults::get(opCtx->getServiceContext())
-                                           .getDefaultReadConcern();
+                                           .getDefaultReadConcern(opCtx);
                 if (rcDefault) {
                     readConcernArgs = std::move(*rcDefault);
                     LOG(2) << "Applying default readConcern on "
diff --git a/src/mongo/db/write_concern.cpp b/src/mongo/db/write_concern.cpp
index 0dcc0ddd57f..2b532fed21e 100644
--- a/src/mongo/db/write_concern.cpp
+++ b/src/mongo/db/write_concern.cpp
@@ -91,7 +91,7 @@ StatusWith<WriteConcernOptions> extractWriteConcern(OperationContext* opCtx,
                 serverGlobalParams.clusterRole != ClusterRole::ConfigServer &&
                 !opCtx->getClient()->isInDirectClient()) {
                 auto wcDefault = ReadWriteConcernDefaults::get(opCtx->getServiceContext())
-                                     .getDefaultWriteConcern();
+                                     .getDefaultWriteConcern(opCtx);
                 if (wcDefault) {
                     LOG(2) << "Applying default writeConcern on " << cmdObj.firstElementFieldName()
                            << " of " << wcDefault->toBSON();
diff --git a/src/mongo/embedded/SConscript b/src/mongo/embedded/SConscript
index 09da019b4a8..49baf7939e9 100644
--- a/src/mongo/embedded/SConscript
+++ b/src/mongo/embedded/SConscript
@@ -75,6 +75,7 @@ env.Library(
         'periodic_runner_embedded.cpp',
         'process_interface_factory_embedded.cpp',
         'read_concern_embedded.cpp',
+        'read_write_concern_defaults_cache_lookup_embedded.cpp',
         'replication_coordinator_embedded.cpp',
         'service_entry_point_embedded.cpp',
         'transaction_coordinator_factory_embedded.cpp',
diff --git a/src/mongo/embedded/embedded.cpp b/src/mongo/embedded/embedded.cpp
index 14df124c6c6..69911b0fa21 100644
--- a/src/mongo/embedded/embedded.cpp
+++ b/src/mongo/embedded/embedded.cpp
@@ -62,6 +62,7 @@
 #include "mongo/db/ttl.h"
 #include "mongo/embedded/index_builds_coordinator_embedded.h"
 #include "mongo/embedded/periodic_runner_embedded.h"
+#include "mongo/embedded/read_write_concern_defaults_cache_lookup_embedded.h"
 #include "mongo/embedded/replication_coordinator_embedded.h"
 #include "mongo/embedded/service_entry_point_embedded.h"
 #include "mongo/logger/log_component.h"
@@ -270,6 +271,8 @@ ServiceContext* initialize(const char* yaml_config) {
         boost::filesystem::remove_all(storageGlobalParams.dbpath + "/_tmp/");
     }
 
+    ReadWriteConcernDefaults::create(serviceContext, readWriteConcernDefaultsCacheLookupEmbedded);
+
     auto startupOpCtx = serviceContext->makeOperationContext(&cc());
 
     bool canCallFCVSetIfCleanStartup =
diff --git a/src/mongo/embedded/read_write_concern_defaults_cache_lookup_embedded.cpp b/src/mongo/embedded/read_write_concern_defaults_cache_lookup_embedded.cpp
new file mode 100644
index 00000000000..316056e684b
--- /dev/null
+++ b/src/mongo/embedded/read_write_concern_defaults_cache_lookup_embedded.cpp
@@ -0,0 +1,39 @@
+/**
+ *    Copyright (C) 2019-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/embedded/read_write_concern_defaults_cache_lookup_embedded.h"
+
+namespace mongo {
+
+boost::optional<RWConcernDefault> readWriteConcernDefaultsCacheLookupEmbedded(
+    OperationContext* opCtx) {
+    return boost::none;
+}
+
+}  // namespace mongo
diff --git a/src/mongo/embedded/read_write_concern_defaults_cache_lookup_embedded.h b/src/mongo/embedded/read_write_concern_defaults_cache_lookup_embedded.h
new file mode 100644
index 00000000000..d9fb9e2dbca
--- /dev/null
+++ b/src/mongo/embedded/read_write_concern_defaults_cache_lookup_embedded.h
@@ -0,0 +1,42 @@
+/**
+ *    Copyright (C) 2019-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/db/read_write_concern_defaults.h"
+
+namespace mongo {
+
+/**
+ * A function which handles looking up RWConcernDefault values on embedded.
+ */
+boost::optional<RWConcernDefault> readWriteConcernDefaultsCacheLookupEmbedded(
+    OperationContext* opCtx);
+
+}  // namespace mongo
diff --git a/src/mongo/s/commands/strategy.cpp b/src/mongo/s/commands/strategy.cpp
index 5625c83f37c..db3a8810f22 100644
--- a/src/mongo/s/commands/strategy.cpp
+++ b/src/mongo/s/commands/strategy.cpp
@@ -441,7 +441,7 @@ void runCommand(OperationContext* opCtx,
             // This command supports WC, but wasn't given one - so apply the default, if there is
             // one.
             if (const auto wcDefault = ReadWriteConcernDefaults::get(opCtx->getServiceContext())
-                                           .getDefaultWriteConcern()) {
+                                           .getDefaultWriteConcern(opCtx)) {
                 wc = *wcDefault;
                 LOG(2) << "Applying default writeConcern on " << request.getCommandName() << " of "
                        << wcDefault->toBSON();
@@ -461,7 +461,7 @@ void runCommand(OperationContext* opCtx,
             (startTransaction || !TransactionRouter::get(opCtx))) {
             if (readConcernArgs.isEmpty()) {
                 const auto rcDefault = ReadWriteConcernDefaults::get(opCtx->getServiceContext())
-                                           .getDefaultReadConcern();
+                                           .getDefaultReadConcern(opCtx);
                 if (rcDefault) {
                     {
                         // We must obtain the client lock to set ReadConcernArgs, because it's an
diff --git a/src/mongo/s/server.cpp b/src/mongo/s/server.cpp
index ddc851749fe..1870814d80e 100644
--- a/src/mongo/s/server.cpp
+++ b/src/mongo/s/server.cpp
@@ -60,6 +60,7 @@
 #include "mongo/db/logical_time_metadata_hook.h"
 #include "mongo/db/logical_time_validator.h"
 #include "mongo/db/operation_context.h"
+#include "mongo/db/read_write_concern_defaults_cache_lookup_mongos.h"
 #include "mongo/db/server_options.h"
 #include "mongo/db/service_context.h"
 #include "mongo/db/service_liaison_mongos.h"
@@ -550,6 +551,8 @@ ExitCode runMongosServer(ServiceContext* serviceContext) {
 
     LogicalClock::set(serviceContext, std::make_unique<LogicalClock>(serviceContext));
 
+    ReadWriteConcernDefaults::create(serviceContext, readWriteConcernDefaultsCacheLookupMongoS);
+
     auto opCtxHolder = tc->makeOperationContext();
     auto const opCtx = opCtxHolder.get();