SERVER-46218 Fix removal/shutdown race in arbiter

If an arbiter is shut down soon after it is removed from the replica set
by a reconfig, the arbiter crashes due to a race between shutdown and
reconfig in KeysCollectionManager::enableKeyGenerator.

(cherry picked from commit 32f47846d78a4fdae9564b7ebb442d53e737d845)

2 files changed, 57 insertions, 1 deletions

diff --git a/jstests/replsets/reconfig_add_remove_arbiter.js b/jstests/replsets/reconfig_add_remove_arbiter.js
new file mode 100644
index 00000000000..e3d2f7e51d2
--- /dev/null
+++ b/jstests/replsets/reconfig_add_remove_arbiter.js
@@ -0,0 +1,53 @@
+/*
+ * Test that replSetReconfig can add and remove arbiters.
+ */
+
+// isMaster fails on the arbiter once it's removed, which blocks all checks.
+TestData.skipCheckDBHashes = true;
+
+(function() {
+'use strict';
+
+load('jstests/replsets/rslib.js');
+
+const replTest = new ReplSetTest({nodes: 2});
+replTest.startSet();
+replTest.initiate();
+
+jsTestLog('Start arbiter');
+const arbiterConn = replTest.add();
+const admin = replTest.getPrimary().getDB('admin');
+const conf = replTest.getReplSetConfigFromNode();
+conf.members.push({_id: 2, host: arbiterConn.host, arbiterOnly: true});
+conf.version++;
+
+jsTestLog('Add arbiter');
+assert.commandWorked(admin.runCommand({replSetReconfig: conf}));
+
+replTest.waitForState(arbiterConn, ReplSetTest.State.ARBITER);
+jsTestLog(`Wait for ${arbiterConn} to enter state ARBITER in primary's replSetGetStatus`);
+assert.soon(() => {
+    let status = assert.commandWorked(admin.runCommand({replSetGetStatus: 1}));
+    return ReplSetTest.State.ARBITER === status.members[2].state;
+});
+
+conf.members.pop();
+conf.version++;
+
+jsTestLog('Remove arbiter');
+assert.commandWorked(admin.runCommand({replSetReconfig: conf}));
+
+assert.soon(
+    () => {
+        // The arbiter dropped connections when it was removed.
+        reconnect(arbiterConn);
+        let status = arbiterConn.getDB('admin').runCommand({replSetGetStatus: 1});
+        print(`replSetGetStatus: ${tojson(status)}`);
+        return status.code === ErrorCodes.InvalidReplicaSetConfig;
+    },
+    "waiting for arbiter's replSetGetStatus to show that the arbiter was removed",
+    undefined /* timeout */,
+    1000 /* intervalMS */);
+
+replTest.stopSet();
+})();
diff --git a/src/mongo/db/keys_collection_manager.cpp b/src/mongo/db/keys_collection_manager.cpp
index 77037f80c84..db7e03da015 100644
--- a/src/mongo/db/keys_collection_manager.cpp
+++ b/src/mongo/db/keys_collection_manager.cpp
@@ -157,7 +157,7 @@ void KeysCollectionManager::stopMonitoring() {
     _refresher.stop();
 }
 
-void KeysCollectionManager::enableKeyGenerator(OperationContext* opCtx, bool doEnable) {
+void KeysCollectionManager::enableKeyGenerator(OperationContext* opCtx, bool doEnable) try {
     if (doEnable) {
         _refresher.switchFunc(opCtx, [this](OperationContext* opCtx) {
             KeyGenerator keyGenerator(_purpose, _client.get(), _keyValidForInterval);
@@ -180,6 +180,9 @@ void KeysCollectionManager::enableKeyGenerator(OperationContext* opCtx, bool doE
         _refresher.switchFunc(
             opCtx, [this](OperationContext* opCtx) { return _keysCache.refresh(opCtx); });
     }
+} catch (const ExceptionForCat<ErrorCategory::ShutdownError>& ex) {
+    LOG(0) << ex.what();
+    return;
 }
 
 bool KeysCollectionManager::hasSeenKeys() {