SERVER-43720 Make RWCDefaults commands persist the defaults

11 files changed, 176 insertions, 83 deletions

diff --git a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
index 9ea3a40995c..366414c6b05 100644
--- a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
@@ -200,6 +200,8 @@ selector:
   - jstests/sharding/mongos_no_detect_sharding.js
   # Aggregation with $merge will fail until SERVER-41949 is finished.
   - jstests/sharding/merge_with_move_primary.js
+  # Runs commands directly on the config server primary and is not robust to the primary changing.
+  - jstests/sharding/read_write_concern_defaults_commands_api.js
 
 executor:
   config:
diff --git a/jstests/noPassthrough/failcommand_failpoint_not_parallel.js b/jstests/noPassthrough/failcommand_failpoint_not_parallel.js
index a9d776d0079..4cf36aed31b 100644
--- a/jstests/noPassthrough/failcommand_failpoint_not_parallel.js
+++ b/jstests/noPassthrough/failcommand_failpoint_not_parallel.js
@@ -6,17 +6,19 @@ assert.neq(null, conn);
 const db = conn.getDB("test_failcommand_noparallel");
 
 // Test times when closing connection.
+// Use distinct because it is rarely used by internal operations, making it less likely unrelated
+// activity triggers the failpoint.
 assert.commandWorked(db.adminCommand({
     configureFailPoint: "failCommand",
     mode: {times: 2},
     data: {
         closeConnection: true,
-        failCommands: ["find"],
+        failCommands: ["distinct"],
     }
 }));
-assert.throws(() => db.runCommand({find: "c"}));
-assert.throws(() => db.runCommand({find: "c"}));
-assert.commandWorked(db.runCommand({find: "c"}));
+assert.throws(() => db.runCommand({distinct: "c", key: "_id"}));
+assert.throws(() => db.runCommand({distinct: "c", key: "_id"}));
+assert.commandWorked(db.runCommand({distinct: "c", key: "_id"}));
 assert.commandWorked(db.adminCommand({configureFailPoint: "failCommand", mode: "off"}));
 
 MongoRunner.stopMongod(conn);
diff --git a/jstests/sharding/read_write_concern_defaults_commands_api.js b/jstests/sharding/read_write_concern_defaults_commands_api.js
index 984443b5842..e3e668484cb 100644
--- a/jstests/sharding/read_write_concern_defaults_commands_api.js
+++ b/jstests/sharding/read_write_concern_defaults_commands_api.js
@@ -1,16 +1,16 @@
-// Tests the basic API of the getDefaultRWConcern and setDefaultRWConcern commands against different
-// topologies.
+// Tests the basic API of the getDefaultRWConcern and setDefaultRWConcern commands and their
+// associated persisted state against different topologies.
 //
 // @tags: [requires_fcv_44]
 (function() {
 "use strict";
 
-// Asserts a set/get default RWC command response contains the expected fields. Assumes a default
-// read or write concern has been set previously.
-function verifyResponseFields(res, {expectRC, expectWC}) {
+// Asserts a set/get default RWC command response or persisted document contains the expected
+// fields. Assumes a default read or write concern has been set previously.
+function verifyFields(res, {expectRC, expectWC, isPersistedDocument}) {
     // These fields are always set once a read or write concern has been set at least once.
-    const expectedFields = ["epoch", "setTime", "localSetTime"];
-    const unexpectedFields = [];
+    let expectedFields = ["epoch", "setTime", "localSetTime"];
+    let unexpectedFields = [];
 
     if (expectRC) {
         expectedFields.push("defaultReadConcern");
@@ -24,6 +24,12 @@ function verifyResponseFields(res, {expectRC, expectWC}) {
         unexpectedFields.push("defaultWriteConcern");
     }
 
+    // localSetTime is generated by the in-memory cache and is not stored in the persisted document.
+    if (isPersistedDocument) {
+        expectedFields = expectedFields.filter(field => field !== "localSetTime");
+        unexpectedFields.push("localSetTime");
+    }
+
     assert.hasFields(res, expectedFields);
     unexpectedFields.forEach(field => {
         assert(!res.hasOwnProperty(field),
@@ -109,20 +115,33 @@ function unsetDefaultRWConcern(conn) {
         {setDefaultRWConcern: 1, defaultReadConcern: {}, defaultWriteConcern: {}}));
 }
 
-// Verifies no fields are returned if neither a default read nor write concern has been set.
-function verifyDefaultResponses(conn) {
+// Verifies the default responses for the default RWC commands and the default persisted state.
+function verifyDefaultState(conn) {
     const res = assert.commandWorked(conn.adminCommand({getDefaultRWConcern: 1}));
     const inMemoryRes =
         assert.commandWorked(conn.adminCommand({getDefaultRWConcern: 1, inMemory: true}));
 
-    const unexpectedFields =
-        ["defaultReadConcern", "defaultWriteConcern", "epoch", "setTime", "localSetTime"];
+    // localSetTime is set when a node refreshes its defaults, even if none are found.
+    const expectedFields = ["localSetTime"];
+    expectedFields.forEach(field => {
+        assert(res.hasOwnProperty(field),
+               `response did not have field '${field}', res: ${tojson(res)}`);
+        assert(inMemoryRes.hasOwnProperty(field),
+               `inMemory=true response did not have field '${field}', res: ${tojson(inMemoryRes)}`);
+    });
+
+    // No other fields should be returned if neither a default read nor write concern has been set.
+    const unexpectedFields = ["defaultReadConcern", "defaultWriteConcern", "epoch", "setTime"];
     unexpectedFields.forEach(field => {
         assert(!res.hasOwnProperty(field),
                `response unexpectedly had field '${field}', res: ${tojson(res)}`);
         assert(!inMemoryRes.hasOwnProperty(field),
-               `inMemory=true response unexpectedly had field '${field}', res: ${tojson(res)}`);
+               `inMemory=true response unexpectedly had field '${field}', res: ${
+                   tojson(inMemoryRes)}`);
     });
+
+    // There should be no default RWC document.
+    assert.eq(null, getPersistedRWCDocument(conn));
 }
 
 function verifyDefaultRWCommandsValidInput(conn) {
@@ -168,36 +187,29 @@ function verifyDefaultRWCommandsValidInput(conn) {
     assert.commandWorked(conn.adminCommand({setDefaultRWConcern: 1, defaultReadConcern: {}}));
 }
 
-function verifyDefaultRWCommandsSuccessfulResponses(conn) {
+// Verifies the responses from successful rwc commands and the persisted state after they complete
+// have the expected format.
+function verifyDefaultRWCommandsOnSuccess(conn) {
     //
     // Test responses for getDefaultRWConcern.
     //
 
     // When neither read nor write concern is set.
     unsetDefaultRWConcern(conn);
-    verifyResponseFields(assert.commandWorked(conn.adminCommand({getDefaultRWConcern: 1})),
-                         {expectRC: false, expectWC: false});
-    verifyResponseFields(
-        assert.commandWorked(conn.adminCommand({getDefaultRWConcern: 1, inMemory: true})),
-        {expectRC: false, expectWC: false});
+    verifyFields(assert.commandWorked(conn.adminCommand({getDefaultRWConcern: 1})),
+                 {expectRC: false, expectWC: false});
 
     // When only read concern is set.
     assert.commandWorked(conn.adminCommand(
         {setDefaultRWConcern: 1, defaultReadConcern: {level: "local"}, defaultWriteConcern: {}}));
-    verifyResponseFields(assert.commandWorked(conn.adminCommand({getDefaultRWConcern: 1})),
-                         {expectRC: true, expectWC: false});
-    verifyResponseFields(
-        assert.commandWorked(conn.adminCommand({getDefaultRWConcern: 1, inMemory: true})),
-        {expectRC: true, expectWC: false});
+    verifyFields(assert.commandWorked(conn.adminCommand({getDefaultRWConcern: 1})),
+                 {expectRC: true, expectWC: false});
 
     // When only write concern is set.
     assert.commandWorked(conn.adminCommand(
         {setDefaultRWConcern: 1, defaultReadConcern: {}, defaultWriteConcern: {w: 1}}));
-    verifyResponseFields(assert.commandWorked(conn.adminCommand({getDefaultRWConcern: 1})),
-                         {expectRC: false, expectWC: true});
-    verifyResponseFields(
-        assert.commandWorked(conn.adminCommand({getDefaultRWConcern: 1, inMemory: true})),
-        {expectRC: false, expectWC: true});
+    verifyFields(assert.commandWorked(conn.adminCommand({getDefaultRWConcern: 1})),
+                 {expectRC: false, expectWC: true});
 
     // When both read and write concern are set.
     assert.commandWorked(conn.adminCommand({
@@ -205,55 +217,67 @@ function verifyDefaultRWCommandsSuccessfulResponses(conn) {
         defaultReadConcern: {level: "local"},
         defaultWriteConcern: {w: 1}
     }));
-    verifyResponseFields(assert.commandWorked(conn.adminCommand({getDefaultRWConcern: 1})),
-                         {expectRC: true, expectWC: true});
-    verifyResponseFields(
-        assert.commandWorked(conn.adminCommand({getDefaultRWConcern: 1, inMemory: true})),
-        {expectRC: true, expectWC: true});
+    verifyFields(assert.commandWorked(conn.adminCommand({getDefaultRWConcern: 1})),
+                 {expectRC: true, expectWC: true});
 
     //
-    // Test responses for setDefaultRWConcern.
+    // Test responses for setDefaultRWConcern and persisted state after.
     //
 
     // When unsetting both read and write concern.
     setDefaultRWConcern(conn);
-    verifyResponseFields(
-        assert.commandWorked(conn.adminCommand(
-            {setDefaultRWConcern: 1, defaultReadConcern: {}, defaultWriteConcern: {}})),
-        {expectRC: false, expectWC: false});
+    verifyFields(assert.commandWorked(conn.adminCommand(
+                     {setDefaultRWConcern: 1, defaultReadConcern: {}, defaultWriteConcern: {}})),
+                 {expectRC: false, expectWC: false});
+    verifyFields(getPersistedRWCDocument(conn),
+                 {expectRC: false, expectWC: false, isPersistedDocument: true});
 
     // When unsetting only read concern.
     setDefaultRWConcern(conn);
-    verifyResponseFields(
+    verifyFields(
         assert.commandWorked(conn.adminCommand({setDefaultRWConcern: 1, defaultReadConcern: {}})),
         {expectRC: false, expectWC: true});
+    verifyFields(getPersistedRWCDocument(conn),
+                 {expectRC: false, expectWC: true, isPersistedDocument: true});
 
     // When unsetting only write concern.
     setDefaultRWConcern(conn);
-    verifyResponseFields(
+    verifyFields(
         assert.commandWorked(conn.adminCommand({setDefaultRWConcern: 1, defaultWriteConcern: {}})),
         {expectRC: true, expectWC: false});
+    verifyFields(getPersistedRWCDocument(conn),
+                 {expectRC: true, expectWC: false, isPersistedDocument: true});
 
     // When setting only write concern.
     unsetDefaultRWConcern(conn);
-    verifyResponseFields(assert.commandWorked(conn.adminCommand(
-                             {setDefaultRWConcern: 1, defaultWriteConcern: {w: 1}})),
-                         {expectRC: false, expectWC: true});
+    verifyFields(assert.commandWorked(
+                     conn.adminCommand({setDefaultRWConcern: 1, defaultWriteConcern: {w: 1}})),
+                 {expectRC: false, expectWC: true});
+    verifyFields(getPersistedRWCDocument(conn),
+                 {expectRC: false, expectWC: true, isPersistedDocument: true});
 
     // When setting only read concern.
     unsetDefaultRWConcern(conn);
-    verifyResponseFields(assert.commandWorked(conn.adminCommand(
-                             {setDefaultRWConcern: 1, defaultReadConcern: {level: "local"}})),
-                         {expectRC: true, expectWC: false});
+    verifyFields(assert.commandWorked(conn.adminCommand(
+                     {setDefaultRWConcern: 1, defaultReadConcern: {level: "local"}})),
+                 {expectRC: true, expectWC: false});
+    verifyFields(getPersistedRWCDocument(conn),
+                 {expectRC: true, expectWC: false, isPersistedDocument: true});
 
     // When setting both read and write concern.
     unsetDefaultRWConcern(conn);
-    verifyResponseFields(assert.commandWorked(conn.adminCommand({
+    verifyFields(assert.commandWorked(conn.adminCommand({
         setDefaultRWConcern: 1,
         defaultReadConcern: {level: "local"},
         defaultWriteConcern: {w: 1}
     })),
-                         {expectRC: true, expectWC: true});
+                 {expectRC: true, expectWC: true});
+    verifyFields(getPersistedRWCDocument(conn),
+                 {expectRC: true, expectWC: true, isPersistedDocument: true});
+}
+
+function getPersistedRWCDocument(conn) {
+    return conn.getDB("config").settings.findOne({_id: "ReadWriteConcernDefaults"});
 }
 
 // Verifies the error code returned by connections to nodes that do not support the get/set default
@@ -282,10 +306,10 @@ jsTestLog("Testing standalone replica set...");
     rst.initiate();
 
     // Primary succeeds.
-    verifyDefaultResponses(rst.getPrimary());
+    verifyDefaultState(rst.getPrimary());
     verifyDefaultRWCommandsValidInput(rst.getPrimary());
     verifyDefaultRWCommandsInvalidInput(rst.getPrimary());
-    verifyDefaultRWCommandsSuccessfulResponses(rst.getPrimary());
+    verifyDefaultRWCommandsOnSuccess(rst.getPrimary());
 
     // Secondary succeeds.
     assert.commandWorked(rst.getSecondary().adminCommand({getDefaultRWConcern: 1}));
@@ -301,10 +325,10 @@ jsTestLog("Testing sharded cluster...");
     const st = new ShardingTest({shards: 1, rs: {nodes: 2}});
 
     // Mongos succeeds.
-    verifyDefaultResponses(st.s);
+    verifyDefaultState(st.s);
     verifyDefaultRWCommandsValidInput(st.s);
     verifyDefaultRWCommandsInvalidInput(st.s);
-    verifyDefaultRWCommandsSuccessfulResponses(st.s);
+    verifyDefaultRWCommandsOnSuccess(st.s);
 
     // Shard node fails.
     verifyDefaultRWCommandsFailWithCode(st.rs0.getPrimary(), {failureCode: 51301});
@@ -313,7 +337,7 @@ jsTestLog("Testing sharded cluster...");
     // Config server primary succeeds.
     verifyDefaultRWCommandsValidInput(st.configRS.getPrimary());
     verifyDefaultRWCommandsInvalidInput(st.configRS.getPrimary());
-    verifyDefaultRWCommandsSuccessfulResponses(st.configRS.getPrimary());
+    verifyDefaultRWCommandsOnSuccess(st.configRS.getPrimary());
 
     // Config server secondary succeeds.
     assert.commandWorked(st.configRS.getSecondary().adminCommand({getDefaultRWConcern: 1}));
diff --git a/src/mongo/db/commands/rwc_defaults_commands.cpp b/src/mongo/db/commands/rwc_defaults_commands.cpp
index b6fb559c9a8..df4a0217d9c 100644
--- a/src/mongo/db/commands/rwc_defaults_commands.cpp
+++ b/src/mongo/db/commands/rwc_defaults_commands.cpp
@@ -33,14 +33,42 @@
 
 #include "mongo/db/commands.h"
 #include "mongo/db/commands/rwc_defaults_commands_gen.h"
+#include "mongo/db/dbdirectclient.h"
+#include "mongo/db/namespace_string.h"
+#include "mongo/db/ops/write_ops.h"
 #include "mongo/db/read_write_concern_defaults.h"
 #include "mongo/db/repl/read_concern_args.h"
 #include "mongo/db/repl/replication_coordinator.h"
+#include "mongo/db/rw_concern_default_gen.h"
+#include "mongo/rpc/get_status_from_command_result.h"
 #include "mongo/util/log.h"
 
 namespace mongo {
 namespace {
 
+/**
+ * Replaces the persisted default read/write concern document with a new one representing the given
+ * defaults. Waits for the write concern on the given operation context to be satisfied before
+ * returning.
+ */
+void updatePersistedDefaultRWConcernDocument(OperationContext* opCtx, const RWConcernDefault& rw) {
+    DBDirectClient client(opCtx);
+    const auto commandResponse = client.runCommand([&] {
+        write_ops::Update updateOp(NamespaceString::kConfigSettingsNamespace);
+        updateOp.setUpdates({[&] {
+            write_ops::UpdateOpEntry entry;
+            entry.setQ(BSON("_id" << ReadWriteConcernDefaults::kPersistedDocumentId));
+            // Note the _id is propagated from the query into the upserted document.
+            entry.setU(rw.toBSON());
+            entry.setUpsert(true);
+            return entry;
+        }()});
+        return updateOp.serialize(
+            BSON(WriteConcernOptions::kWriteConcernField << opCtx->getWriteConcern().toBSON()));
+    }());
+    uassertStatusOK(getStatusFromWriteCommandReply(commandResponse->getCommandReply()));
+}
+
 void assertNotStandaloneOrShardServer(OperationContext* opCtx, StringData cmdName) {
     const auto replCoord = repl::ReplicationCoordinator::get(opCtx);
     uassert(51300,
@@ -78,10 +106,20 @@ public:
             assertNotStandaloneOrShardServer(opCtx, SetDefaultRWConcern::kCommandName);
 
             auto& rwcDefaults = ReadWriteConcernDefaults::get(opCtx->getServiceContext());
-            auto newDefaults = rwcDefaults.setConcerns(
+            auto newDefaults = rwcDefaults.generateNewConcerns(
                 opCtx, request().getDefaultReadConcern(), request().getDefaultWriteConcern());
-            log() << "successfully set RWC defaults to " << newDefaults.toBSON();
-            return newDefaults;
+
+            // TODO SERVER-44890 Remove this check once this command can only run on a primary node.
+            if (repl::ReplicationCoordinator::get(opCtx)->getMemberState() ==
+                repl::MemberState::RS_PRIMARY) {
+                // TODO SERVER-44890: Make this update invalidate the RWC cache through an
+                // OpObserver so setting the new values below is safe to be best effort.
+                updatePersistedDefaultRWConcernDocument(opCtx, newDefaults);
+            }
+
+            // Force a refresh to find the newly set defaults, then return them.
+            rwcDefaults.refreshIfNecessary(opCtx);
+            return rwcDefaults.getDefault(opCtx);
         }
 
     private:
@@ -121,9 +159,13 @@ public:
         auto typedRun(OperationContext* opCtx) {
             assertNotStandaloneOrShardServer(opCtx, GetDefaultRWConcern::kCommandName);
 
-            // TODO SERVER-43720 Implement inMemory option.
-
             auto& rwcDefaults = ReadWriteConcernDefaults::get(opCtx->getServiceContext());
+            if (request().getInMemory()) {
+                return rwcDefaults.getDefault(opCtx);
+            }
+
+            // Force a refresh to find the most recent defaults, then return them.
+            rwcDefaults.refreshIfNecessary(opCtx);
             return rwcDefaults.getDefault(opCtx);
         }
 
diff --git a/src/mongo/db/namespace_string.cpp b/src/mongo/db/namespace_string.cpp
index 2397e3c1d0d..dd8f965afb5 100644
--- a/src/mongo/db/namespace_string.cpp
+++ b/src/mongo/db/namespace_string.cpp
@@ -82,6 +82,8 @@ const NamespaceString NamespaceString::kIndexBuildEntryNamespace(NamespaceString
                                                                  "system.indexBuilds");
 const NamespaceString NamespaceString::kRangeDeletionNamespace(NamespaceString::kConfigDb,
                                                                "rangeDeletions");
+const NamespaceString NamespaceString::kConfigSettingsNamespace(NamespaceString::kConfigDb,
+                                                                "settings");
 
 bool NamespaceString::isListCollectionsCursorNS() const {
     return coll() == listCollectionsCursorCol;
diff --git a/src/mongo/db/namespace_string.h b/src/mongo/db/namespace_string.h
index add77a0dfaf..52a0febef7c 100644
--- a/src/mongo/db/namespace_string.h
+++ b/src/mongo/db/namespace_string.h
@@ -107,6 +107,9 @@ public:
     // Namespace for pending range deletions.
     static const NamespaceString kRangeDeletionNamespace;
 
+    // Namespace for balancer settings and default read and write concerns.
+    static const NamespaceString kConfigSettingsNamespace;
+
     /**
      * Constructs an empty NamespaceString.
      */
diff --git a/src/mongo/db/read_write_concern_defaults.cpp b/src/mongo/db/read_write_concern_defaults.cpp
index 9ba17d28101..d4ea514d8f9 100644
--- a/src/mongo/db/read_write_concern_defaults.cpp
+++ b/src/mongo/db/read_write_concern_defaults.cpp
@@ -79,13 +79,10 @@ void ReadWriteConcernDefaults::checkSuitabilityAsDefault(const WriteConcern& wc)
             !(wc.wMode.empty() && wc.wNumNodes < 1));
 }
 
-void ReadWriteConcernDefaults::_setDefault(RWConcernDefault&& rwc) {
-    _defaults.revalidate(Type::kReadWriteConcernEntry, std::move(rwc));
-}
-
-RWConcernDefault ReadWriteConcernDefaults::setConcerns(OperationContext* opCtx,
-                                                       const boost::optional<ReadConcern>& rc,
-                                                       const boost::optional<WriteConcern>& wc) {
+RWConcernDefault ReadWriteConcernDefaults::generateNewConcerns(
+    OperationContext* opCtx,
+    const boost::optional<ReadConcern>& rc,
+    const boost::optional<WriteConcern>& wc) {
     uassert(ErrorCodes::BadValue,
             str::stream() << "At least one of the \""
                           << RWConcernDefault::kDefaultReadConcernFieldName << "\" or \""
@@ -106,7 +103,6 @@ RWConcernDefault ReadWriteConcernDefaults::setConcerns(OperationContext* opCtx,
     rwc.setEpoch(epoch);
     auto now = opCtx->getServiceContext()->getFastClockSource()->now();
     rwc.setSetTime(now);
-    rwc.setLocalSetTime(now);
 
     auto current = _getDefault(opCtx);
     if (!rc && current) {
@@ -115,8 +111,8 @@ RWConcernDefault ReadWriteConcernDefaults::setConcerns(OperationContext* opCtx,
     if (!wc && current) {
         rwc.setDefaultWriteConcern(current->getDefaultWriteConcern());
     }
-    _setDefault(std::move(rwc));
-    return *_getDefault(opCtx);
+
+    return rwc;
 }
 
 void ReadWriteConcernDefaults::invalidate() {
diff --git a/src/mongo/db/read_write_concern_defaults.h b/src/mongo/db/read_write_concern_defaults.h
index 001f9b5da77..2f296a5a0df 100644
--- a/src/mongo/db/read_write_concern_defaults.h
+++ b/src/mongo/db/read_write_concern_defaults.h
@@ -58,6 +58,9 @@ public:
     static constexpr StringData readConcernFieldName = ReadConcern::kReadConcernFieldName;
     static constexpr StringData writeConcernFieldName = WriteConcern::kWriteConcernField;
 
+    // The _id of the persisted default read/write concern document.
+    static constexpr StringData kPersistedDocumentId = "ReadWriteConcernDefaults"_sd;
+
     static ReadWriteConcernDefaults& get(ServiceContext* service);
     static ReadWriteConcernDefaults& get(ServiceContext& service);
     static void create(ServiceContext* service, LookupFn lookupFn);
@@ -79,14 +82,15 @@ public:
     static void checkSuitabilityAsDefault(const WriteConcern& wc);
 
     /**
-     * Interface when an admin has run the command to change the defaults.
+     * Generates a new read and write concern default to be persisted on disk, without updating the
+     * cached value.
      * At least one of the `rc` or `wc` params must be set.
      * Will generate and use a new epoch and setTime for the updated defaults, which are returned.
      * Validates the supplied read and write concerns can serve as defaults.
      */
-    RWConcernDefault setConcerns(OperationContext* opCtx,
-                                 const boost::optional<ReadConcern>& rc,
-                                 const boost::optional<WriteConcern>& wc);
+    RWConcernDefault generateNewConcerns(OperationContext* opCtx,
+                                         const boost::optional<ReadConcern>& rc,
+                                         const boost::optional<WriteConcern>& wc);
 
     /**
      * Invalidates the cached RWC defaults, causing them to be refreshed.
@@ -111,7 +115,6 @@ public:
 private:
     enum class Type { kReadWriteConcernEntry };
 
-    void _setDefault(RWConcernDefault&& rwc);
     boost::optional<RWConcernDefault> _getDefault(OperationContext* opCtx);
 
     class Cache : public DistCache<Type, RWConcernDefault> {
diff --git a/src/mongo/db/read_write_concern_defaults_cache_lookup_mongod.cpp b/src/mongo/db/read_write_concern_defaults_cache_lookup_mongod.cpp
index f807f67c204..33a0cc3f22e 100644
--- a/src/mongo/db/read_write_concern_defaults_cache_lookup_mongod.cpp
+++ b/src/mongo/db/read_write_concern_defaults_cache_lookup_mongod.cpp
@@ -29,11 +29,29 @@
 
 #include "mongo/db/read_write_concern_defaults_cache_lookup_mongod.h"
 
+#include "mongo/db/dbdirectclient.h"
+#include "mongo/db/namespace_string.h"
+#include "mongo/db/read_write_concern_defaults.h"
+
 namespace mongo {
+namespace {
+
+BSONObj getPersistedDefaultRWConcernDocument(OperationContext* opCtx) {
+    DBDirectClient client(opCtx);
+    return client.findOne(NamespaceString::kConfigSettingsNamespace.toString(),
+                          QUERY("_id" << ReadWriteConcernDefaults::kPersistedDocumentId));
+}
+
+}  // namespace
 
 boost::optional<RWConcernDefault> readWriteConcernDefaultsCacheLookupMongoD(
     OperationContext* opCtx) {
-    return boost::none;
+    // Note that a default constructed RWConcern is returned if no document is found instead of
+    // boost::none. This is to avoid excessive lookups when there is no defaults document, because
+    // otherwise every attempt to get the defaults from the RWC cache would trigger a lookup.
+    return RWConcernDefault::parse(
+        IDLParserErrorContext("ReadWriteConcernDefaultsCacheLookupMongoD"),
+        getPersistedDefaultRWConcernDocument(opCtx));
 }
 
 }  // namespace mongo
diff --git a/src/mongo/s/balancer_configuration.cpp b/src/mongo/s/balancer_configuration.cpp
index c54b1112786..8197c03e19c 100644
--- a/src/mongo/s/balancer_configuration.cpp
+++ b/src/mongo/s/balancer_configuration.cpp
@@ -42,6 +42,7 @@
 #include "mongo/base/status_with.h"
 #include "mongo/bson/bsonobj.h"
 #include "mongo/bson/util/bson_extract.h"
+#include "mongo/db/namespace_string.h"
 #include "mongo/s/catalog/sharding_catalog_client.h"
 #include "mongo/s/grid.h"
 #include "mongo/util/log.h"
@@ -77,8 +78,6 @@ const char kActiveWindow[] = "activeWindow";
 const char kWaitForDelete[] = "_waitForDelete";
 const char kAttemptToBalanceJumboChunks[] = "attemptToBalanceJumboChunks";
 
-const NamespaceString kSettingsNamespace("config", "settings");
-
 }  // namespace
 
 const char BalancerSettingsType::kKey[] = "balancer";
@@ -105,7 +104,7 @@ Status BalancerConfiguration::setBalancerMode(OperationContext* opCtx,
                                               BalancerSettingsType::BalancerMode mode) {
     auto updateStatus = Grid::get(opCtx)->catalogClient()->updateConfigDocument(
         opCtx,
-        kSettingsNamespace,
+        NamespaceString::kConfigSettingsNamespace,
         BSON("_id" << BalancerSettingsType::kKey),
         BSON("$set" << BSON(kStopped << (mode == BalancerSettingsType::kOff) << kMode
                                      << BalancerSettingsType::kBalancerModes[mode])),
@@ -129,7 +128,7 @@ Status BalancerConfiguration::setBalancerMode(OperationContext* opCtx,
 Status BalancerConfiguration::enableAutoSplit(OperationContext* opCtx, bool enable) {
     auto updateStatus = Grid::get(opCtx)->catalogClient()->updateConfigDocument(
         opCtx,
-        kSettingsNamespace,
+        NamespaceString::kConfigSettingsNamespace,
         BSON("_id" << AutoSplitSettingsType::kKey),
         BSON("$set" << BSON(kEnabled << enable)),
         true,
diff --git a/src/mongo/s/commands/cluster_rwc_defaults_commands.cpp b/src/mongo/s/commands/cluster_rwc_defaults_commands.cpp
index 8286a8d42c4..e7976c147e4 100644
--- a/src/mongo/s/commands/cluster_rwc_defaults_commands.cpp
+++ b/src/mongo/s/commands/cluster_rwc_defaults_commands.cpp
@@ -110,7 +110,9 @@ public:
         using InvocationBase::InvocationBase;
 
         Response typedRun(OperationContext* opCtx) {
-            // TODO SERVER-43720 Implement inMemory option.
+            if (request().getInMemory()) {
+                return ReadWriteConcernDefaults::get(opCtx->getServiceContext()).getDefault(opCtx);
+            }
 
             GetDefaultRWConcern configsvrRequest;
             configsvrRequest.setDbName(request().getDbName());