diff options
author | A. Jesse Jiryu Davis <jesse@mongodb.com> | 2020-02-17 21:42:26 -0500 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-02-20 11:33:38 +0000 |
commit | 32f47846d78a4fdae9564b7ebb442d53e737d845 (patch) | |
tree | ab90f16c47ea085fa0972f5420f8ee09b50ce90b | |
parent | 7eb6f03dd34615f2774a2cb70dadeffadce4512e (diff) | |
download | mongo-32f47846d78a4fdae9564b7ebb442d53e737d845.tar.gz |
SERVER-46218 Fix removal/shutdown race in arbiter
If an arbiter is shut down soon after it is removed from the replica set
by a reconfig, the arbiter crashes due to a race between shutdown and
reconfig in KeysCollectionManager::enableKeyGenerator.
-rw-r--r-- | jstests/replsets/reconfig_add_remove_arbiter.js | 53 | ||||
-rw-r--r-- | src/mongo/db/keys_collection_manager.cpp | 5 |
2 files changed, 57 insertions, 1 deletions
diff --git a/jstests/replsets/reconfig_add_remove_arbiter.js b/jstests/replsets/reconfig_add_remove_arbiter.js new file mode 100644 index 00000000000..e3d2f7e51d2 --- /dev/null +++ b/jstests/replsets/reconfig_add_remove_arbiter.js @@ -0,0 +1,53 @@ +/* + * Test that replSetReconfig can add and remove arbiters. + */ + +// isMaster fails on the arbiter once it's removed, which blocks all checks. +TestData.skipCheckDBHashes = true; + +(function() { +'use strict'; + +load('jstests/replsets/rslib.js'); + +const replTest = new ReplSetTest({nodes: 2}); +replTest.startSet(); +replTest.initiate(); + +jsTestLog('Start arbiter'); +const arbiterConn = replTest.add(); +const admin = replTest.getPrimary().getDB('admin'); +const conf = replTest.getReplSetConfigFromNode(); +conf.members.push({_id: 2, host: arbiterConn.host, arbiterOnly: true}); +conf.version++; + +jsTestLog('Add arbiter'); +assert.commandWorked(admin.runCommand({replSetReconfig: conf})); + +replTest.waitForState(arbiterConn, ReplSetTest.State.ARBITER); +jsTestLog(`Wait for ${arbiterConn} to enter state ARBITER in primary's replSetGetStatus`); +assert.soon(() => { + let status = assert.commandWorked(admin.runCommand({replSetGetStatus: 1})); + return ReplSetTest.State.ARBITER === status.members[2].state; +}); + +conf.members.pop(); +conf.version++; + +jsTestLog('Remove arbiter'); +assert.commandWorked(admin.runCommand({replSetReconfig: conf})); + +assert.soon( + () => { + // The arbiter dropped connections when it was removed. + reconnect(arbiterConn); + let status = arbiterConn.getDB('admin').runCommand({replSetGetStatus: 1}); + print(`replSetGetStatus: ${tojson(status)}`); + return status.code === ErrorCodes.InvalidReplicaSetConfig; + }, + "waiting for arbiter's replSetGetStatus to show that the arbiter was removed", + undefined /* timeout */, + 1000 /* intervalMS */); + +replTest.stopSet(); +})(); diff --git a/src/mongo/db/keys_collection_manager.cpp b/src/mongo/db/keys_collection_manager.cpp index a921df95c59..4a9238c5f2d 100644 --- a/src/mongo/db/keys_collection_manager.cpp +++ b/src/mongo/db/keys_collection_manager.cpp @@ -159,7 +159,7 @@ void KeysCollectionManager::stopMonitoring() { _refresher.stop(); } -void KeysCollectionManager::enableKeyGenerator(OperationContext* opCtx, bool doEnable) { +void KeysCollectionManager::enableKeyGenerator(OperationContext* opCtx, bool doEnable) try { if (doEnable) { _refresher.switchFunc(opCtx, [this](OperationContext* opCtx) { KeyGenerator keyGenerator(_purpose, _client.get(), _keyValidForInterval); @@ -182,6 +182,9 @@ void KeysCollectionManager::enableKeyGenerator(OperationContext* opCtx, bool doE _refresher.switchFunc( opCtx, [this](OperationContext* opCtx) { return _keysCache.refresh(opCtx); }); } +} catch (const ExceptionForCat<ErrorCategory::ShutdownError>& ex) { + LOGV2(518091, "{ex}, doEnable = {doEnable}", "ex"_attr = ex, "doEnable"_attr = doEnable); + return; } bool KeysCollectionManager::hasSeenKeys() { |