summaryrefslogtreecommitdiff
path: root/src/mongo/db/s
diff options
context:
space:
mode:
authorKaloian Manassiev <kaloian.manassiev@mongodb.com>2020-12-04 08:13:42 -0500
committerEvergreen Agent <no-reply@evergreen.mongodb.com>2021-01-03 11:56:04 +0000
commit5431150d116b70fc9401c46deaacc9ae049f064e (patch)
treed1784b3cb2355cee9401e539f4574f0172995d8d /src/mongo/db/s
parent98a7731d21a8746e584f7092aadbee60a5fad6ef (diff)
downloadmongo-5431150d116b70fc9401c46deaacc9ae049f064e.tar.gz
SERVER-53227 Move the DistLockManager to only be available on MongoD
Diffstat (limited to 'src/mongo/db/s')
-rw-r--r--src/mongo/db/s/README.md4
-rw-r--r--src/mongo/db/s/SConscript16
-rw-r--r--src/mongo/db/s/add_shard_cmd.cpp2
-rw-r--r--src/mongo/db/s/balancer/migration_manager.cpp81
-rw-r--r--src/mongo/db/s/balancer/migration_manager.h1
-rw-r--r--src/mongo/db/s/balancer/migration_manager_test.cpp18
-rw-r--r--src/mongo/db/s/balancer/migration_test_fixture.h1
-rw-r--r--src/mongo/db/s/balancer/scoped_migration_request_test.cpp1
-rw-r--r--src/mongo/db/s/clone_catalog_data_command.cpp4
-rw-r--r--src/mongo/db/s/config/config_server_test_fixture.cpp20
-rw-r--r--src/mongo/db/s/config/config_server_test_fixture.h8
-rw-r--r--src/mongo/db/s/config/configsvr_clear_jumbo_flag_command.cpp6
-rw-r--r--src/mongo/db/s/config/configsvr_create_database_command.cpp6
-rw-r--r--src/mongo/db/s/config/configsvr_drop_collection_command.cpp8
-rw-r--r--src/mongo/db/s/config/configsvr_drop_database_command.cpp10
-rw-r--r--src/mongo/db/s/config/configsvr_enable_sharding_command.cpp11
-rw-r--r--src/mongo/db/s/config/configsvr_move_primary_command.cpp4
-rw-r--r--src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp18
-rw-r--r--src/mongo/db/s/config/configsvr_shard_collection_command.cpp5
-rw-r--r--src/mongo/db/s/config/initial_split_policy.h53
-rw-r--r--src/mongo/db/s/config/initial_split_policy_test.cpp12
-rw-r--r--src/mongo/db/s/config/sharding_catalog_manager.h8
-rw-r--r--src/mongo/db/s/config/sharding_catalog_manager_collection_operations.cpp34
-rw-r--r--src/mongo/db/s/config/sharding_catalog_manager_create_database_test.cpp2
-rw-r--r--src/mongo/db/s/config/sharding_catalog_manager_enable_sharding_test.cpp1
-rw-r--r--src/mongo/db/s/dist_lock_catalog.cpp52
-rw-r--r--src/mongo/db/s/dist_lock_catalog.h187
-rw-r--r--src/mongo/db/s/dist_lock_catalog_mock.cpp357
-rw-r--r--src/mongo/db/s/dist_lock_catalog_mock.h223
-rw-r--r--src/mongo/db/s/dist_lock_catalog_replset.cpp545
-rw-r--r--src/mongo/db/s/dist_lock_catalog_replset.h103
-rw-r--r--src/mongo/db/s/dist_lock_catalog_replset_test.cpp1807
-rw-r--r--src/mongo/db/s/dist_lock_manager.cpp107
-rw-r--r--src/mongo/db/s/dist_lock_manager.h189
-rw-r--r--src/mongo/db/s/dist_lock_manager_mock.cpp139
-rw-r--r--src/mongo/db/s/dist_lock_manager_mock.h86
-rw-r--r--src/mongo/db/s/dist_lock_manager_replset.cpp622
-rw-r--r--src/mongo/db/s/dist_lock_manager_replset.h173
-rw-r--r--src/mongo/db/s/dist_lock_manager_replset_test.cpp2135
-rw-r--r--src/mongo/db/s/merge_chunks_command.cpp3
-rw-r--r--src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp5
-rw-r--r--src/mongo/db/s/migration_util_test.cpp6
-rw-r--r--src/mongo/db/s/resharding/resharding_donor_oplog_iterator_test.cpp8
-rw-r--r--src/mongo/db/s/resharding/resharding_donor_service_test.cpp12
-rw-r--r--src/mongo/db/s/resharding/resharding_oplog_applier_test.cpp1
-rw-r--r--src/mongo/db/s/resharding/resharding_oplog_fetcher_test.cpp6
-rw-r--r--src/mongo/db/s/resharding/resharding_txn_cloner_test.cpp6
-rw-r--r--src/mongo/db/s/resharding_destined_recipient_test.cpp6
-rw-r--r--src/mongo/db/s/resharding_util.cpp1
-rw-r--r--src/mongo/db/s/session_catalog_migration_destination_test.cpp5
-rw-r--r--src/mongo/db/s/shard_collection_legacy.cpp7
-rw-r--r--src/mongo/db/s/shard_server_test_fixture.cpp18
-rw-r--r--src/mongo/db/s/shard_server_test_fixture.h17
-rw-r--r--src/mongo/db/s/sharding_initialization_mongod.cpp131
-rw-r--r--src/mongo/db/s/sharding_initialization_mongod.h12
-rw-r--r--src/mongo/db/s/sharding_initialization_mongod_test.cpp18
-rw-r--r--src/mongo/db/s/sharding_initialization_op_observer_test.cpp1
-rw-r--r--src/mongo/db/s/sharding_mongod_test_fixture.cpp34
-rw-r--r--src/mongo/db/s/sharding_mongod_test_fixture.h21
-rw-r--r--src/mongo/db/s/shardsvr_drop_collection_command.cpp2
-rw-r--r--src/mongo/db/s/shardsvr_drop_database_command.cpp1
-rw-r--r--src/mongo/db/s/shardsvr_refine_collection_shard_key_command.cpp1
-rw-r--r--src/mongo/db/s/split_chunk.cpp3
-rw-r--r--src/mongo/db/s/split_chunk_test.cpp2
-rw-r--r--src/mongo/db/s/transaction_coordinator_futures_util_test.cpp5
-rw-r--r--src/mongo/db/s/transaction_coordinator_test_fixture.cpp7
-rw-r--r--src/mongo/db/s/transaction_coordinator_test_fixture.h3
-rw-r--r--src/mongo/db/s/vector_clock_config_server_test.cpp8
68 files changed, 6989 insertions, 420 deletions
diff --git a/src/mongo/db/s/README.md b/src/mongo/db/s/README.md
index 0f906fee8d7..1c8712233c8 100644
--- a/src/mongo/db/s/README.md
+++ b/src/mongo/db/s/README.md
@@ -521,8 +521,8 @@ collections, such as config.shards, config.chunks, and config.tags. For example,
mergeChunks, and moveChunk all take the chunk ResourceMutex.
#### Code references
-* [**DistLockManager class**](https://github.com/mongodb/mongo/blob/r4.3.4/src/mongo/s/catalog/dist_lock_manager.h)
-* [**DistLockCatalog class**](https://github.com/mongodb/mongo/blob/r4.3.4/src/mongo/s/catalog/dist_lock_catalog.h)
+* [**DistLockManager class**](https://github.com/mongodb/mongo/blob/master/src/mongo/db/s/dist_lock_manager.h)
+* [**DistLockCatalog class**](https://github.com/mongodb/mongo/blob/master/src/mongo/db/s/dist_lock_catalog.h)
* [**NamespaceSerializer class**](https://github.com/mongodb/mongo/blob/r4.3.4/src/mongo/db/s/config/namespace_serializer.h)
* The interface for acquiring NamespaceSerializer locks
[**via the ShardingCatalogManager**](https://github.com/mongodb/mongo/blob/r4.3.4/src/mongo/db/s/config/sharding_catalog_manager.h#L276)
diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript
index 83d1e7488ee..2cbb4cc140e 100644
--- a/src/mongo/db/s/SConscript
+++ b/src/mongo/db/s/SConscript
@@ -263,6 +263,10 @@ env.Library(
'config/sharding_catalog_manager_shard_operations.cpp',
'config/sharding_catalog_manager_zone_operations.cpp',
'config/sharding_catalog_manager.cpp',
+ 'dist_lock_catalog_replset.cpp',
+ 'dist_lock_catalog.cpp',
+ 'dist_lock_manager_replset.cpp',
+ 'dist_lock_manager.cpp',
],
LIBDEPS=[
'$BUILD_DIR/mongo/db/catalog_raii',
@@ -289,7 +293,6 @@ env.Library(
'$BUILD_DIR/mongo/db/repl/replica_set_aware_service',
'$BUILD_DIR/mongo/db/snapshot_window_options',
'$BUILD_DIR/mongo/db/vector_clock_mongod',
- '$BUILD_DIR/mongo/s/catalog/dist_lock_manager',
'$BUILD_DIR/mongo/s/catalog/sharding_catalog_client_impl',
'$BUILD_DIR/mongo/util/log_and_backoff',
'$BUILD_DIR/mongo/util/options_parser/options_parser',
@@ -366,7 +369,6 @@ env.Library(
'$BUILD_DIR/mongo/s/sharding_initialization',
'$BUILD_DIR/mongo/s/sharding_router_api',
'resharding_util',
- 'sharding_catalog_manager',
'sharding_runtime_d',
],
)
@@ -400,6 +402,8 @@ env.Library(
env.Library(
target='sharding_mongod_test_fixture',
source=[
+ 'dist_lock_catalog_mock.cpp',
+ 'dist_lock_manager_mock.cpp',
'sharding_mongod_test_fixture.cpp',
],
LIBDEPS=[
@@ -417,8 +421,6 @@ env.Library(
'shard_server_test_fixture.cpp',
],
LIBDEPS=[
- '$BUILD_DIR/mongo/s/catalog/dist_lock_catalog_mock',
- '$BUILD_DIR/mongo/s/catalog/dist_lock_manager_mock',
'sharding_mongod_test_fixture',
],
)
@@ -429,7 +431,6 @@ env.Library(
'config/config_server_test_fixture.cpp',
],
LIBDEPS=[
- 'sharding_catalog_manager',
'sharding_mongod_test_fixture',
],
)
@@ -444,6 +445,8 @@ env.CppUnitTest(
'collection_metadata_filtering_test.cpp',
'collection_metadata_test.cpp',
'collection_sharding_runtime_test.cpp',
+ 'dist_lock_catalog_replset_test.cpp',
+ 'dist_lock_manager_replset_test.cpp',
'metadata_manager_test.cpp',
'migration_chunk_cloner_source_legacy_test.cpp',
'migration_destination_manager_test.cpp',
@@ -501,7 +504,6 @@ env.CppUnitTest(
'$BUILD_DIR/mongo/db/repl/storage_interface_impl',
'$BUILD_DIR/mongo/db/repl/wait_for_majority_service',
'$BUILD_DIR/mongo/executor/thread_pool_task_executor_test_fixture',
- '$BUILD_DIR/mongo/s/catalog/dist_lock_manager_mock',
'$BUILD_DIR/mongo/s/catalog/sharding_catalog_client_mock',
'$BUILD_DIR/mongo/s/sharding_router_test_fixture',
'resharding_util',
@@ -554,10 +556,8 @@ env.CppUnitTest(
'$BUILD_DIR/mongo/db/pipeline/document_source_mock',
'$BUILD_DIR/mongo/db/read_write_concern_defaults_mock',
'$BUILD_DIR/mongo/db/repl/replication_info',
- '$BUILD_DIR/mongo/s/catalog/dist_lock_manager_mock',
'$BUILD_DIR/mongo/util/version_impl',
'config_server_test_fixture',
'resharding_util',
- 'sharding_catalog_manager',
],
)
diff --git a/src/mongo/db/s/add_shard_cmd.cpp b/src/mongo/db/s/add_shard_cmd.cpp
index ab4c58a0272..38b39b7bd47 100644
--- a/src/mongo/db/s/add_shard_cmd.cpp
+++ b/src/mongo/db/s/add_shard_cmd.cpp
@@ -39,13 +39,13 @@
#include "mongo/db/dbdirectclient.h"
#include "mongo/db/s/add_shard_cmd_gen.h"
#include "mongo/db/s/add_shard_util.h"
-#include "mongo/db/s/config/sharding_catalog_manager.h"
#include "mongo/rpc/get_status_from_command_result.h"
#include "mongo/s/balancer_configuration.h"
#include "mongo/s/grid.h"
namespace mongo {
namespace {
+
/**
* Internal sharding command run on mongod to initialize itself as a shard in the cluster.
*/
diff --git a/src/mongo/db/s/balancer/migration_manager.cpp b/src/mongo/db/s/balancer/migration_manager.cpp
index f20edc35e43..49b2dcd90c6 100644
--- a/src/mongo/db/s/balancer/migration_manager.cpp
+++ b/src/mongo/db/s/balancer/migration_manager.cpp
@@ -33,8 +33,6 @@
#include "mongo/db/s/balancer/migration_manager.h"
-#include <memory>
-
#include "mongo/bson/simple_bsonobj_comparator.h"
#include "mongo/bson/util/bson_extract.h"
#include "mongo/client/remote_command_targeter.h"
@@ -42,6 +40,7 @@
#include "mongo/db/repl/repl_set_config.h"
#include "mongo/db/repl/replication_coordinator.h"
#include "mongo/db/s/balancer/scoped_migration_request.h"
+#include "mongo/db/s/dist_lock_manager.h"
#include "mongo/executor/task_executor_pool.h"
#include "mongo/logv2/log.h"
#include "mongo/rpc/get_status_from_command_result.h"
@@ -53,14 +52,10 @@
#include "mongo/util/scopeguard.h"
namespace mongo {
+namespace {
using executor::RemoteCommandRequest;
using executor::RemoteCommandResponse;
-using std::shared_ptr;
-using std::vector;
-using str::stream;
-
-namespace {
const char kChunkTooBig[] = "chunkTooBig"; // TODO: delete in 3.8
@@ -86,7 +81,6 @@ Status extractMigrationStatusFromCommandResponse(const BSONObj& commandResponse)
return commandStatus;
}
-
/**
* Returns whether the specified status is an error caused by stepdown of the primary config node
* currently running the balancer.
@@ -109,7 +103,7 @@ MigrationManager::~MigrationManager() {
MigrationStatuses MigrationManager::executeMigrationsForAutoBalance(
OperationContext* opCtx,
- const vector<MigrateInfo>& migrateInfos,
+ const std::vector<MigrateInfo>& migrateInfos,
uint64_t maxChunkSizeBytes,
const MigrationSecondaryThrottleOptions& secondaryThrottle,
bool waitForDelete) {
@@ -117,7 +111,8 @@ MigrationStatuses MigrationManager::executeMigrationsForAutoBalance(
MigrationStatuses migrationStatuses;
ScopedMigrationRequestsMap scopedMigrationRequests;
- vector<std::pair<shared_ptr<Notification<RemoteCommandResponse>>, MigrateInfo>> responses;
+ std::vector<std::pair<std::shared_ptr<Notification<RemoteCommandResponse>>, MigrateInfo>>
+ responses;
for (const auto& migrateInfo : migrateInfos) {
responses.emplace_back(_schedule(opCtx,
@@ -229,8 +224,6 @@ void MigrationManager::startRecoveryAndAcquireDistLocks(OperationContext* opCtx)
_abandonActiveMigrationsAndEnableManager(opCtx);
});
- auto distLockManager = Grid::get(opCtx)->catalogClient()->getDistLockManager();
-
// Load the active migrations from the config.migrations collection.
auto statusWithMigrationsQueryResponse =
Grid::get(opCtx)->shardRegistry()->getConfigShard()->exhaustiveFindOnConfig(
@@ -273,11 +266,12 @@ void MigrationManager::startRecoveryAndAcquireDistLocks(OperationContext* opCtx)
it = _migrationRecoveryMap.insert(std::make_pair(migrateType.getNss(), list)).first;
// Reacquire the matching distributed lock for this namespace.
- const std::string whyMessage(stream() << "Migrating chunk(s) in collection "
- << migrateType.getNss().ns());
+ const std::string whyMessage(str::stream() << "Migrating chunk(s) in collection "
+ << migrateType.getNss().ns());
- auto statusWithDistLockHandle = distLockManager->tryLockWithLocalWriteConcern(
- opCtx, migrateType.getNss().ns(), whyMessage, _lockSessionID);
+ auto statusWithDistLockHandle =
+ DistLockManager::get(opCtx)->tryLockWithLocalWriteConcern(
+ opCtx, migrateType.getNss().ns(), whyMessage, _lockSessionID);
if (!statusWithDistLockHandle.isOK()) {
LOGV2(21898,
"Failed to acquire distributed lock for collection {namespace} "
@@ -322,8 +316,8 @@ void MigrationManager::finishRecovery(OperationContext* opCtx,
});
// Schedule recovered migrations.
- vector<ScopedMigrationRequest> scopedMigrationRequests;
- vector<shared_ptr<Notification<RemoteCommandResponse>>> responses;
+ std::vector<ScopedMigrationRequest> scopedMigrationRequests;
+ std::vector<std::shared_ptr<Notification<RemoteCommandResponse>>> responses;
for (auto& nssAndMigrateInfos : _migrationRecoveryMap) {
auto& nss = nssAndMigrateInfos.first;
@@ -379,8 +373,7 @@ void MigrationManager::finishRecovery(OperationContext* opCtx,
// If no migrations were scheduled for this namespace, free the dist lock
if (!scheduledMigrations) {
- Grid::get(opCtx)->catalogClient()->getDistLockManager()->unlock(
- opCtx, _lockSessionID, nss.ns());
+ DistLockManager::get(opCtx)->unlock(opCtx, _lockSessionID, nss.ns());
}
}
@@ -432,7 +425,7 @@ void MigrationManager::drainActiveMigrations() {
_state = State::kStopped;
}
-shared_ptr<Notification<RemoteCommandResponse>> MigrationManager::_schedule(
+std::shared_ptr<Notification<RemoteCommandResponse>> MigrationManager::_schedule(
OperationContext* opCtx,
const MigrateInfo& migrateInfo,
uint64_t maxChunkSizeBytes,
@@ -524,25 +517,25 @@ void MigrationManager::_schedule(WithLock lock,
auto it = _activeMigrations.find(nss);
if (it == _activeMigrations.end()) {
- const std::string whyMessage(stream() << "Migrating chunk(s) in collection " << nss.ns());
+ const std::string whyMessage(str::stream()
+ << "Migrating chunk(s) in collection " << nss.ns());
// Acquire the NamespaceSerializer lock for this nss (blocking call)
auto scopedCollLock =
ShardingCatalogManager::get(opCtx)->serializeCreateOrDropCollection(opCtx, nss);
// Acquire the collection distributed lock (blocking call)
- auto statusWithDistLockHandle =
- Grid::get(opCtx)->catalogClient()->getDistLockManager()->lockWithSessionID(
- opCtx,
- nss.ns(),
- whyMessage,
- _lockSessionID,
- DistLockManager::kSingleLockAttemptTimeout);
+ auto statusWithDistLockHandle = DistLockManager::get(opCtx)->lockWithSessionID(
+ opCtx,
+ nss.ns(),
+ whyMessage,
+ _lockSessionID,
+ DistLockManager::kSingleLockAttemptTimeout);
if (!statusWithDistLockHandle.isOK()) {
migration.completionNotification->set(statusWithDistLockHandle.getStatus().withContext(
- stream() << "Could not acquire collection lock for " << nss.ns()
- << " to migrate chunks"));
+ str::stream() << "Could not acquire collection lock for " << nss.ns()
+ << " to migrate chunks"));
return;
}
@@ -618,8 +611,7 @@ void MigrationManager::_complete(WithLock lock,
migrations->erase(itMigration);
if (migrations->empty()) {
- Grid::get(opCtx)->catalogClient()->getDistLockManager()->unlock(
- opCtx, _lockSessionID, nss.ns());
+ DistLockManager::get(opCtx)->unlock(opCtx, _lockSessionID, nss.ns());
_activeMigrations.erase(it);
_checkDrained(lock);
}
@@ -651,15 +643,14 @@ void MigrationManager::_abandonActiveMigrationsAndEnableManager(OperationContext
}
invariant(_state == State::kRecovering);
- auto catalogClient = Grid::get(opCtx)->catalogClient();
-
// Unlock all balancer distlocks we aren't using anymore.
- auto distLockManager = catalogClient->getDistLockManager();
+ auto distLockManager = DistLockManager::get(opCtx);
distLockManager->unlockAll(opCtx, distLockManager->getProcessID());
// Clear the config.migrations collection so that those chunks can be scheduled for migration
// again.
- catalogClient
+ Grid::get(opCtx)
+ ->catalogClient()
->removeConfigDocuments(
opCtx, MigrationType::ConfigNS, BSONObj(), ShardingCatalogClient::kLocalWriteConcern)
.transitional_ignore();
@@ -680,8 +671,8 @@ Status MigrationManager::_processRemoteCommandResponse(
_state != State::kEnabled && _state != State::kRecovering)) {
scopedMigrationRequest->keepDocumentOnDestruct();
return {ErrorCodes::BalancerInterrupted,
- stream() << "Migration interrupted because the balancer is stopping."
- << " Command status: " << remoteCommandResponse.status.toString()};
+ str::stream() << "Migration interrupted because the balancer is stopping."
+ << " Command status: " << remoteCommandResponse.status.toString()};
}
if (!remoteCommandResponse.isOK()) {
@@ -693,8 +684,8 @@ Status MigrationManager::_processRemoteCommandResponse(
if (!Shard::shouldErrorBePropagated(commandStatus.code())) {
commandStatus = {ErrorCodes::OperationFailed,
- stream() << "moveChunk command failed on source shard."
- << causedBy(commandStatus)};
+ str::stream() << "moveChunk command failed on source shard."
+ << causedBy(commandStatus)};
}
// Any failure to remove the migration document should be because the config server is
@@ -705,10 +696,10 @@ Status MigrationManager::_processRemoteCommandResponse(
if (!status.isOK()) {
commandStatus = {
ErrorCodes::BalancerInterrupted,
- stream() << "Migration interrupted because the balancer is stopping"
- << " and failed to remove the config.migrations document."
- << " Command status: "
- << (commandStatus.isOK() ? status.toString() : commandStatus.toString())};
+ str::stream() << "Migration interrupted because the balancer is stopping"
+ << " and failed to remove the config.migrations document."
+ << " Command status: "
+ << (commandStatus.isOK() ? status.toString() : commandStatus.toString())};
}
return commandStatus;
diff --git a/src/mongo/db/s/balancer/migration_manager.h b/src/mongo/db/s/balancer/migration_manager.h
index 1f0f3511198..fd5db687100 100644
--- a/src/mongo/db/s/balancer/migration_manager.h
+++ b/src/mongo/db/s/balancer/migration_manager.h
@@ -40,7 +40,6 @@
#include "mongo/db/s/config/sharding_catalog_manager.h"
#include "mongo/executor/task_executor.h"
#include "mongo/platform/mutex.h"
-#include "mongo/s/catalog/dist_lock_manager.h"
#include "mongo/s/request_types/migration_secondary_throttle_options.h"
#include "mongo/stdx/condition_variable.h"
#include "mongo/stdx/unordered_map.h"
diff --git a/src/mongo/db/s/balancer/migration_manager_test.cpp b/src/mongo/db/s/balancer/migration_manager_test.cpp
index 4948ac24e48..961036e6f92 100644
--- a/src/mongo/db/s/balancer/migration_manager_test.cpp
+++ b/src/mongo/db/s/balancer/migration_manager_test.cpp
@@ -29,14 +29,12 @@
#include "mongo/platform/basic.h"
-#include <memory>
-
#include "mongo/db/commands.h"
#include "mongo/db/read_write_concern_defaults.h"
#include "mongo/db/read_write_concern_defaults_cache_lookup_mock.h"
#include "mongo/db/s/balancer/migration_manager.h"
#include "mongo/db/s/balancer/migration_test_fixture.h"
-#include "mongo/db/s/config/sharding_catalog_manager.h"
+#include "mongo/db/s/dist_lock_manager.h"
#include "mongo/s/request_types/move_chunk_request.h"
namespace mongo {
@@ -521,7 +519,7 @@ TEST_F(MigrationManagerTest, MigrationRecovery) {
setUpMigration(chunk2, kShardId3.toString());
// Mimic all config distlocks being released on config server stepup to primary.
- auto distLockManager = catalogClient()->getDistLockManager();
+ auto distLockManager = DistLockManager::get(operationContext());
distLockManager->unlockAll(operationContext(), distLockManager->getProcessID());
_migrationManager->startRecoveryAndAcquireDistLocks(operationContext());
@@ -588,12 +586,12 @@ TEST_F(MigrationManagerTest, FailMigrationRecovery) {
// Take the distributed lock for the collection, which should be released during recovery when
// it fails. Any dist lock held by the config server will be released via proccessId, so the
// session ID used here doesn't matter.
- ASSERT_OK(catalogClient()->getDistLockManager()->lockWithSessionID(
- operationContext(),
- collName.ns(),
- "MigrationManagerTest",
- OID::gen(),
- DistLockManager::kSingleLockAttemptTimeout));
+ ASSERT_OK(DistLockManager::get(operationContext())
+ ->lockWithSessionID(operationContext(),
+ collName.ns(),
+ "MigrationManagerTest",
+ OID::gen(),
+ DistLockManager::kSingleLockAttemptTimeout));
_migrationManager->startRecoveryAndAcquireDistLocks(operationContext());
_migrationManager->finishRecovery(operationContext(), 0, kDefaultSecondaryThrottle);
diff --git a/src/mongo/db/s/balancer/migration_test_fixture.h b/src/mongo/db/s/balancer/migration_test_fixture.h
index d2fef72b513..2e6dc250e1e 100644
--- a/src/mongo/db/s/balancer/migration_test_fixture.h
+++ b/src/mongo/db/s/balancer/migration_test_fixture.h
@@ -37,7 +37,6 @@
#include "mongo/db/s/balancer/type_migration.h"
#include "mongo/db/s/config/config_server_test_fixture.h"
#include "mongo/db/write_concern_options.h"
-#include "mongo/s/catalog/dist_lock_manager_mock.h"
#include "mongo/s/catalog/type_collection.h"
#include "mongo/s/catalog/type_database.h"
#include "mongo/s/catalog/type_locks.h"
diff --git a/src/mongo/db/s/balancer/scoped_migration_request_test.cpp b/src/mongo/db/s/balancer/scoped_migration_request_test.cpp
index 75442f5c09a..3731321ad2d 100644
--- a/src/mongo/db/s/balancer/scoped_migration_request_test.cpp
+++ b/src/mongo/db/s/balancer/scoped_migration_request_test.cpp
@@ -32,7 +32,6 @@
#include "mongo/db/s/balancer/scoped_migration_request.h"
#include "mongo/db/s/balancer/type_migration.h"
#include "mongo/db/s/config/config_server_test_fixture.h"
-#include "mongo/db/s/config/sharding_catalog_manager.h"
#include "mongo/s/client/shard_registry.h"
#include "mongo/s/request_types/migration_secondary_throttle_options.h"
diff --git a/src/mongo/db/s/clone_catalog_data_command.cpp b/src/mongo/db/s/clone_catalog_data_command.cpp
index a10c599d499..a182aaeaa7f 100644
--- a/src/mongo/db/s/clone_catalog_data_command.cpp
+++ b/src/mongo/db/s/clone_catalog_data_command.cpp
@@ -35,8 +35,8 @@
#include "mongo/db/catalog/document_validation.h"
#include "mongo/db/cloner.h"
#include "mongo/db/commands.h"
+#include "mongo/db/concurrency/d_concurrency.h"
#include "mongo/db/dbdirectclient.h"
-#include "mongo/db/s/config/sharding_catalog_manager.h"
#include "mongo/db/s/sharding_state.h"
#include "mongo/rpc/get_status_from_command_result.h"
#include "mongo/s/grid.h"
@@ -136,7 +136,7 @@ public:
return true;
}
-} CloneCatalogDataCmd;
+} cloneCatalogDataCmd;
} // namespace
} // namespace mongo
diff --git a/src/mongo/db/s/config/config_server_test_fixture.cpp b/src/mongo/db/s/config/config_server_test_fixture.cpp
index ab331cb2f4c..b8a21753d7e 100644
--- a/src/mongo/db/s/config/config_server_test_fixture.cpp
+++ b/src/mongo/db/s/config/config_server_test_fixture.cpp
@@ -51,13 +51,13 @@
#include "mongo/db/repl/repl_settings.h"
#include "mongo/db/repl/replication_coordinator_mock.h"
#include "mongo/db/s/config/sharding_catalog_manager.h"
+#include "mongo/db/s/dist_lock_catalog_replset.h"
+#include "mongo/db/s/dist_lock_manager_replset.h"
#include "mongo/executor/task_executor_pool.h"
#include "mongo/executor/thread_pool_task_executor_test_fixture.h"
#include "mongo/rpc/metadata/repl_set_metadata.h"
#include "mongo/rpc/metadata/tracking_metadata.h"
#include "mongo/s/balancer_configuration.h"
-#include "mongo/s/catalog/dist_lock_catalog_impl.h"
-#include "mongo/s/catalog/replset_dist_lock_manager.h"
#include "mongo/s/catalog/sharding_catalog_client_impl.h"
#include "mongo/s/catalog/type_chunk.h"
#include "mongo/s/catalog/type_collection.h"
@@ -161,25 +161,17 @@ void ConfigServerTestFixture::tearDown() {
ShardingMongodTestFixture::tearDown();
}
-std::unique_ptr<DistLockCatalog> ConfigServerTestFixture::makeDistLockCatalog() {
- return std::make_unique<DistLockCatalogImpl>();
-}
-
-std::unique_ptr<DistLockManager> ConfigServerTestFixture::makeDistLockManager(
- std::unique_ptr<DistLockCatalog> distLockCatalog) {
- invariant(distLockCatalog);
+std::unique_ptr<DistLockManager> ConfigServerTestFixture::makeDistLockManager() {
return std::make_unique<ReplSetDistLockManager>(
getServiceContext(),
"distLockProcessId",
- std::move(distLockCatalog),
+ std::make_unique<DistLockCatalogImpl>(),
ReplSetDistLockManager::kDistLockPingInterval,
ReplSetDistLockManager::kDistLockExpirationTime);
}
-std::unique_ptr<ShardingCatalogClient> ConfigServerTestFixture::makeShardingCatalogClient(
- std::unique_ptr<DistLockManager> distLockManager) {
- invariant(distLockManager);
- return std::make_unique<ShardingCatalogClientImpl>(std::move(distLockManager));
+std::unique_ptr<ShardingCatalogClient> ConfigServerTestFixture::makeShardingCatalogClient() {
+ return std::make_unique<ShardingCatalogClientImpl>();
}
std::unique_ptr<BalancerConfiguration> ConfigServerTestFixture::makeBalancerConfiguration() {
diff --git a/src/mongo/db/s/config/config_server_test_fixture.h b/src/mongo/db/s/config/config_server_test_fixture.h
index b5264d9131c..3e82cd8fcc3 100644
--- a/src/mongo/db/s/config/config_server_test_fixture.h
+++ b/src/mongo/db/s/config/config_server_test_fixture.h
@@ -167,13 +167,9 @@ protected:
*/
void setUpAndInitializeConfigDb();
- std::unique_ptr<DistLockCatalog> makeDistLockCatalog() override;
+ std::unique_ptr<DistLockManager> makeDistLockManager() override;
- std::unique_ptr<DistLockManager> makeDistLockManager(
- std::unique_ptr<DistLockCatalog> distLockCatalog) override;
-
- std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient(
- std::unique_ptr<DistLockManager> distLockManager) override;
+ std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient() override;
std::unique_ptr<ClusterCursorManager> makeClusterCursorManager() override;
diff --git a/src/mongo/db/s/config/configsvr_clear_jumbo_flag_command.cpp b/src/mongo/db/s/config/configsvr_clear_jumbo_flag_command.cpp
index ed955b9f209..c25d4dc9bdf 100644
--- a/src/mongo/db/s/config/configsvr_clear_jumbo_flag_command.cpp
+++ b/src/mongo/db/s/config/configsvr_clear_jumbo_flag_command.cpp
@@ -36,7 +36,7 @@
#include "mongo/db/commands.h"
#include "mongo/db/repl/repl_client_info.h"
#include "mongo/db/s/config/sharding_catalog_manager.h"
-#include "mongo/s/catalog/dist_lock_manager.h"
+#include "mongo/db/s/dist_lock_manager.h"
#include "mongo/s/grid.h"
#include "mongo/s/request_types/clear_jumbo_flag_gen.h"
@@ -70,10 +70,10 @@ public:
// Acquire distlocks on the namespace's database and collection.
DistLockManager::ScopedDistLock dbDistLock(
- uassertStatusOK(catalogClient->getDistLockManager()->lock(
+ uassertStatusOK(DistLockManager::get(opCtx)->lock(
opCtx, nss.db(), "clearJumboFlag", DistLockManager::kDefaultLockTimeout)));
DistLockManager::ScopedDistLock collDistLock(
- uassertStatusOK(catalogClient->getDistLockManager()->lock(
+ uassertStatusOK(DistLockManager::get(opCtx)->lock(
opCtx, nss.ns(), "clearJumboFlag", DistLockManager::kDefaultLockTimeout)));
CollectionType collType;
diff --git a/src/mongo/db/s/config/configsvr_create_database_command.cpp b/src/mongo/db/s/config/configsvr_create_database_command.cpp
index ec6793676b6..17725f4451d 100644
--- a/src/mongo/db/s/config/configsvr_create_database_command.cpp
+++ b/src/mongo/db/s/config/configsvr_create_database_command.cpp
@@ -42,6 +42,7 @@
#include "mongo/db/operation_context.h"
#include "mongo/db/repl/read_concern_args.h"
#include "mongo/db/s/config/sharding_catalog_manager.h"
+#include "mongo/db/s/dist_lock_manager.h"
#include "mongo/s/catalog/type_database.h"
#include "mongo/s/catalog_cache.h"
#include "mongo/s/grid.h"
@@ -90,9 +91,8 @@ public:
auto scopedLock =
ShardingCatalogManager::get(opCtx)->serializeCreateOrDropDatabase(opCtx, dbname);
- auto dbDistLock =
- uassertStatusOK(Grid::get(opCtx)->catalogClient()->getDistLockManager()->lock(
- opCtx, dbname, "createDatabase", DistLockManager::kDefaultLockTimeout));
+ auto dbDistLock = uassertStatusOK(DistLockManager::get(opCtx)->lock(
+ opCtx, dbname, "createDatabase", DistLockManager::kDefaultLockTimeout));
ShardingCatalogManager::get(opCtx)->createDatabase(opCtx, dbname, ShardId());
}
diff --git a/src/mongo/db/s/config/configsvr_drop_collection_command.cpp b/src/mongo/db/s/config/configsvr_drop_collection_command.cpp
index 005746b79f6..aa8b22651e6 100644
--- a/src/mongo/db/s/config/configsvr_drop_collection_command.cpp
+++ b/src/mongo/db/s/config/configsvr_drop_collection_command.cpp
@@ -36,8 +36,8 @@
#include "mongo/db/repl/read_concern_args.h"
#include "mongo/db/repl/repl_client_info.h"
#include "mongo/db/s/config/sharding_catalog_manager.h"
+#include "mongo/db/s/dist_lock_manager.h"
#include "mongo/db/s/operation_sharding_state.h"
-#include "mongo/s/catalog/dist_lock_manager.h"
#include "mongo/s/catalog/type_database.h"
#include "mongo/s/catalog_cache.h"
#include "mongo/s/client/shard_registry.h"
@@ -121,17 +121,15 @@ public:
setDropCollDistLockWait.execute(
[&](const BSONObj& data) { waitFor = Seconds(data["waitForSecs"].numberInt()); });
- auto const catalogClient = Grid::get(opCtx)->catalogClient();
-
auto scopedDbLock =
ShardingCatalogManager::get(opCtx)->serializeCreateOrDropDatabase(opCtx, nss.db());
auto scopedCollLock =
ShardingCatalogManager::get(opCtx)->serializeCreateOrDropCollection(opCtx, nss);
auto dbDistLock = uassertStatusOK(
- catalogClient->getDistLockManager()->lock(opCtx, nss.db(), "dropCollection", waitFor));
+ DistLockManager::get(opCtx)->lock(opCtx, nss.db(), "dropCollection", waitFor));
auto collDistLock = uassertStatusOK(
- catalogClient->getDistLockManager()->lock(opCtx, nss.ns(), "dropCollection", waitFor));
+ DistLockManager::get(opCtx)->lock(opCtx, nss.ns(), "dropCollection", waitFor));
ON_BLOCK_EXIT([opCtx, nss] {
Grid::get(opCtx)->catalogCache()->invalidateCollectionEntry_LINEARIZABLE(nss);
diff --git a/src/mongo/db/s/config/configsvr_drop_database_command.cpp b/src/mongo/db/s/config/configsvr_drop_database_command.cpp
index e2e69a6f042..7ef3658490c 100644
--- a/src/mongo/db/s/config/configsvr_drop_database_command.cpp
+++ b/src/mongo/db/s/config/configsvr_drop_database_command.cpp
@@ -37,15 +37,14 @@
#include "mongo/db/repl/repl_client_info.h"
#include "mongo/db/repl/replication_coordinator.h"
#include "mongo/db/s/config/sharding_catalog_manager.h"
+#include "mongo/db/s/dist_lock_manager.h"
#include "mongo/db/s/sharding_logging.h"
-#include "mongo/s/catalog/dist_lock_manager.h"
#include "mongo/s/catalog/type_database.h"
#include "mongo/s/catalog_cache.h"
#include "mongo/s/grid.h"
#include "mongo/util/scopeguard.h"
namespace mongo {
-
namespace {
/**
@@ -121,10 +120,9 @@ public:
auto const catalogClient = Grid::get(opCtx)->catalogClient();
auto const catalogManager = ShardingCatalogManager::get(opCtx);
- auto scopedLock =
- ShardingCatalogManager::get(opCtx)->serializeCreateOrDropDatabase(opCtx, dbname);
+ auto scopedLock = catalogManager->serializeCreateOrDropDatabase(opCtx, dbname);
- auto dbDistLock = uassertStatusOK(catalogClient->getDistLockManager()->lock(
+ auto dbDistLock = uassertStatusOK(DistLockManager::get(opCtx)->lock(
opCtx, dbname, "dropDatabase", DistLockManager::kDefaultLockTimeout));
// Invalidate the database metadata so the next access kicks off a full reload.
@@ -153,7 +151,7 @@ public:
// Drop the database's collections.
for (const auto& nss : catalogClient->getAllShardedCollectionsForDb(
opCtx, dbname, repl::ReadConcernArgs::get(opCtx).getLevel())) {
- auto collDistLock = uassertStatusOK(catalogClient->getDistLockManager()->lock(
+ auto collDistLock = uassertStatusOK(DistLockManager::get(opCtx)->lock(
opCtx, nss.ns(), "dropCollection", DistLockManager::kDefaultLockTimeout));
catalogManager->dropCollection(opCtx, nss);
}
diff --git a/src/mongo/db/s/config/configsvr_enable_sharding_command.cpp b/src/mongo/db/s/config/configsvr_enable_sharding_command.cpp
index aed8625b1b1..512448bf572 100644
--- a/src/mongo/db/s/config/configsvr_enable_sharding_command.cpp
+++ b/src/mongo/db/s/config/configsvr_enable_sharding_command.cpp
@@ -43,17 +43,13 @@
#include "mongo/db/operation_context.h"
#include "mongo/db/repl/read_concern_args.h"
#include "mongo/db/s/config/sharding_catalog_manager.h"
+#include "mongo/db/s/dist_lock_manager.h"
#include "mongo/s/catalog/type_database.h"
#include "mongo/s/catalog_cache.h"
#include "mongo/s/grid.h"
#include "mongo/util/scopeguard.h"
namespace mongo {
-
-using std::set;
-using std::shared_ptr;
-using std::string;
-
namespace {
/**
@@ -137,9 +133,8 @@ public:
// Make sure to force update of any stale metadata
ON_BLOCK_EXIT([opCtx, dbname] { Grid::get(opCtx)->catalogCache()->purgeDatabase(dbname); });
- auto dbDistLock =
- uassertStatusOK(Grid::get(opCtx)->catalogClient()->getDistLockManager()->lock(
- opCtx, dbname, "enableSharding", DistLockManager::kDefaultLockTimeout));
+ auto dbDistLock = uassertStatusOK(DistLockManager::get(opCtx)->lock(
+ opCtx, dbname, "enableSharding", DistLockManager::kDefaultLockTimeout));
ShardingCatalogManager::get(opCtx)->enableSharding(opCtx, dbname, shardId);
audit::logEnableSharding(Client::getCurrent(), dbname);
diff --git a/src/mongo/db/s/config/configsvr_move_primary_command.cpp b/src/mongo/db/s/config/configsvr_move_primary_command.cpp
index aabe58ef3a4..dc714ee9211 100644
--- a/src/mongo/db/s/config/configsvr_move_primary_command.cpp
+++ b/src/mongo/db/s/config/configsvr_move_primary_command.cpp
@@ -42,7 +42,7 @@
#include "mongo/db/operation_context.h"
#include "mongo/db/repl/read_concern_args.h"
#include "mongo/db/repl/repl_client_info.h"
-#include "mongo/db/s/config/sharding_catalog_manager.h"
+#include "mongo/db/s/dist_lock_manager.h"
#include "mongo/db/server_options.h"
#include "mongo/logv2/log.h"
#include "mongo/s/catalog/type_database.h"
@@ -145,7 +145,7 @@ public:
auto const catalogClient = Grid::get(opCtx)->catalogClient();
auto const shardRegistry = Grid::get(opCtx)->shardRegistry();
- auto dbDistLock = uassertStatusOK(catalogClient->getDistLockManager()->lock(
+ auto dbDistLock = uassertStatusOK(DistLockManager::get(opCtx)->lock(
opCtx, dbname, "movePrimary", DistLockManager::kDefaultLockTimeout));
auto dbType =
diff --git a/src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp b/src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp
index d3fa6bc250e..05b82b1af50 100644
--- a/src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp
+++ b/src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp
@@ -36,9 +36,9 @@
#include "mongo/db/commands.h"
#include "mongo/db/repl/repl_client_info.h"
#include "mongo/db/s/config/sharding_catalog_manager.h"
+#include "mongo/db/s/dist_lock_manager.h"
#include "mongo/db/s/shard_key_util.h"
#include "mongo/logv2/log.h"
-#include "mongo/s/catalog/dist_lock_manager.h"
#include "mongo/s/grid.h"
#include "mongo/s/request_types/refine_collection_shard_key_gen.h"
#include "mongo/s/stale_shard_version_helpers.h"
@@ -74,15 +74,15 @@ public:
// Acquire distlocks on the namespace's database and collection.
DistLockManager::ScopedDistLock dbDistLock(uassertStatusOK(
- catalogClient->getDistLockManager()->lock(opCtx,
- nss.db(),
- "refineCollectionShardKey",
- DistLockManager::kDefaultLockTimeout)));
+ DistLockManager::get(opCtx)->lock(opCtx,
+ nss.db(),
+ "refineCollectionShardKey",
+ DistLockManager::kDefaultLockTimeout)));
DistLockManager::ScopedDistLock collDistLock(uassertStatusOK(
- catalogClient->getDistLockManager()->lock(opCtx,
- nss.ns(),
- "refineCollectionShardKey",
- DistLockManager::kDefaultLockTimeout)));
+ DistLockManager::get(opCtx)->lock(opCtx,
+ nss.ns(),
+ "refineCollectionShardKey",
+ DistLockManager::kDefaultLockTimeout)));
// Validate the given namespace is (i) sharded, (ii) doesn't already have the proposed
// key, and (iii) has the same epoch as the router that received
diff --git a/src/mongo/db/s/config/configsvr_shard_collection_command.cpp b/src/mongo/db/s/config/configsvr_shard_collection_command.cpp
index 7cfc6445161..816369f690a 100644
--- a/src/mongo/db/s/config/configsvr_shard_collection_command.cpp
+++ b/src/mongo/db/s/config/configsvr_shard_collection_command.cpp
@@ -41,6 +41,7 @@
#include "mongo/db/query/collation/collator_factory_interface.h"
#include "mongo/db/repl/read_concern_args.h"
#include "mongo/db/s/config/sharding_catalog_manager.h"
+#include "mongo/db/s/dist_lock_manager.h"
#include "mongo/db/s/shard_key_util.h"
#include "mongo/s/balancer_configuration.h"
#include "mongo/s/catalog/type_database.h"
@@ -248,10 +249,10 @@ public:
// Make the distlocks boost::optional so that they can be released by being reset below.
boost::optional<DistLockManager::ScopedDistLock> dbDistLock(
- uassertStatusOK(catalogClient->getDistLockManager()->lock(
+ uassertStatusOK(DistLockManager::get(opCtx)->lock(
opCtx, nss.db(), "shardCollection", DistLockManager::kDefaultLockTimeout)));
boost::optional<DistLockManager::ScopedDistLock> collDistLock(
- uassertStatusOK(catalogClient->getDistLockManager()->lock(
+ uassertStatusOK(DistLockManager::get(opCtx)->lock(
opCtx, nss.ns(), "shardCollection", DistLockManager::kDefaultLockTimeout)));
// Ensure sharding is allowed on the database.
diff --git a/src/mongo/db/s/config/initial_split_policy.h b/src/mongo/db/s/config/initial_split_policy.h
index 75f2380baff..6687c53c933 100644
--- a/src/mongo/db/s/config/initial_split_policy.h
+++ b/src/mongo/db/s/config/initial_split_policy.h
@@ -66,6 +66,31 @@ public:
size_t numShards,
bool collectionIsEmpty);
+ virtual ~InitialSplitPolicy() {}
+
+ /**
+ * Generates a list of initial chunks to be created during a shardCollection operation.
+ */
+ struct ShardCollectionConfig {
+ std::vector<ChunkType> chunks;
+ Timestamp creationTime;
+
+ const auto& collVersion() const {
+ return chunks.back().getVersion();
+ }
+ };
+ virtual ShardCollectionConfig createFirstChunks(OperationContext* opCtx,
+ const ShardKeyPattern& shardKeyPattern,
+ SplitPolicyParams params) = 0;
+
+ /**
+ * Returns whether the chunk generation strategy being used is optimized or not. Since there is
+ * only a single unoptimized policy, we return true by default here.
+ */
+ virtual bool isOptimized() {
+ return true;
+ }
+
/**
* Returns split points to use for creating chunks in cases where the shard key contains a
* hashed field. For new collections which use hashed shard keys, we can can pre-split the range
@@ -79,15 +104,6 @@ public:
BSONObj prefix,
int numInitialChunks);
- struct ShardCollectionConfig {
- std::vector<ChunkType> chunks;
- Timestamp creationTime;
-
- const auto& collVersion() const {
- return chunks.back().getVersion();
- }
- };
-
/**
* Produces the initial chunks that need to be written for an *empty* collection which is being
* sharded based on a set of 'splitPoints' and 'numContiguousChunksPerShard'.
@@ -111,24 +127,7 @@ public:
const Timestamp& validAfter,
const std::vector<BSONObj>& splitPoints,
const std::vector<ShardId>& allShardIds,
- const int numContiguousChunksPerShard = 1);
-
- /**
- * Generates a list of initial chunks to be created during a shardCollection operation.
- */
- virtual ShardCollectionConfig createFirstChunks(OperationContext* opCtx,
- const ShardKeyPattern& shardKeyPattern,
- SplitPolicyParams params) = 0;
-
- /**
- * Returns whether the chunk generation strategy being used is optimized or not. Since there is
- * only a single unoptimized policy, we return true by default here.
- */
- virtual bool isOptimized() {
- return true;
- }
-
- virtual ~InitialSplitPolicy() {}
+ int numContiguousChunksPerShard);
};
/**
diff --git a/src/mongo/db/s/config/initial_split_policy_test.cpp b/src/mongo/db/s/config/initial_split_policy_test.cpp
index b583c6b87c4..b653e8f59c4 100644
--- a/src/mongo/db/s/config/initial_split_policy_test.cpp
+++ b/src/mongo/db/s/config/initial_split_policy_test.cpp
@@ -278,8 +278,13 @@ private:
TEST_F(GenerateInitialHashedSplitChunksTest, NoSplitPoints) {
const std::vector<BSONObj> splitPoints;
const std::vector<ShardId> shardIds = makeShardIds(2);
- const auto shardCollectionConfig = InitialSplitPolicy::generateShardCollectionInitialChunks(
- {nss(), boost::none, shardIds[0]}, shardKeyPattern(), timeStamp(), splitPoints, shardIds);
+ const auto shardCollectionConfig =
+ InitialSplitPolicy::generateShardCollectionInitialChunks({nss(), boost::none, shardIds[0]},
+ shardKeyPattern(),
+ timeStamp(),
+ splitPoints,
+ shardIds,
+ 1);
// there should only be one chunk
const auto expectedChunks =
@@ -296,7 +301,8 @@ TEST_F(GenerateInitialHashedSplitChunksTest, SplitPointsMoreThanAvailableShards)
shardKeyPattern(),
timeStamp(),
hashedSplitPoints(),
- shardIds);
+ shardIds,
+ 1);
// // chunks should be distributed in a round-robin manner
const std::vector<ChunkType> expectedChunks = makeChunks(
diff --git a/src/mongo/db/s/config/sharding_catalog_manager.h b/src/mongo/db/s/config/sharding_catalog_manager.h
index d68bd3a4dad..e9f0a69269a 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager.h
+++ b/src/mongo/db/s/config/sharding_catalog_manager.h
@@ -47,14 +47,7 @@
namespace mongo {
-struct CollectionOptions;
-class OperationContext;
-class RemoteCommandTargeter;
-class ServiceContext;
-class UUID;
-
struct RemoveShardProgress {
-
/**
* Used to indicate to the caller of the removeShard method whether draining of chunks for
* a particular shard has started, is ongoing, or has been completed.
@@ -89,7 +82,6 @@ struct RemoveShardProgress {
class ShardingCatalogManager {
ShardingCatalogManager(const ShardingCatalogManager&) = delete;
ShardingCatalogManager& operator=(const ShardingCatalogManager&) = delete;
- friend class ConfigSvrShardCollectionCommand;
public:
ShardingCatalogManager(ServiceContext* serviceContext,
diff --git a/src/mongo/db/s/config/sharding_catalog_manager_collection_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_collection_operations.cpp
index 83af868f7f7..89246451c01 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager_collection_operations.cpp
+++ b/src/mongo/db/s/config/sharding_catalog_manager_collection_operations.cpp
@@ -218,40 +218,6 @@ void triggerFireAndForgetShardRefreshes(OperationContext* opCtx, const Namespace
} // namespace
-void checkForExistingChunks(OperationContext* opCtx, const NamespaceString& nss) {
- BSONObjBuilder countBuilder;
- countBuilder.append("count", ChunkType::ConfigNS.coll());
- countBuilder.append("query", BSON(ChunkType::ns(nss.ns())));
-
- // OK to use limit=1, since if any chunks exist, we will fail.
- countBuilder.append("limit", 1);
-
- // Use readConcern local to guarantee we see any chunks that have been written and may
- // become committed; readConcern majority will not see the chunks if they have not made it
- // to the majority snapshot.
- repl::ReadConcernArgs readConcern(repl::ReadConcernLevel::kLocalReadConcern);
- readConcern.appendInfo(&countBuilder);
-
- auto cmdResponse = uassertStatusOK(
- Grid::get(opCtx)->shardRegistry()->getConfigShard()->runCommandWithFixedRetryAttempts(
- opCtx,
- kConfigReadSelector,
- ChunkType::ConfigNS.db().toString(),
- countBuilder.done(),
- Shard::kDefaultConfigCommandTimeout,
- Shard::RetryPolicy::kIdempotent));
- uassertStatusOK(cmdResponse.commandStatus);
-
- long long numChunks;
- uassertStatusOK(bsonExtractIntegerField(cmdResponse.response, "n", &numChunks));
- uassert(ErrorCodes::ManualInterventionRequired,
- str::stream() << "A previous attempt to shard collection " << nss.ns()
- << " failed after writing some initial chunks to config.chunks. Please "
- "manually delete the partially written chunks for collection "
- << nss.ns() << " from config.chunks",
- numChunks == 0);
-}
-
void sendDropCollectionToAllShards(OperationContext* opCtx, const NamespaceString& nss) {
const auto catalogClient = Grid::get(opCtx)->catalogClient();
diff --git a/src/mongo/db/s/config/sharding_catalog_manager_create_database_test.cpp b/src/mongo/db/s/config/sharding_catalog_manager_create_database_test.cpp
index 8e0cd0745f3..bc6d12ae396 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager_create_database_test.cpp
+++ b/src/mongo/db/s/config/sharding_catalog_manager_create_database_test.cpp
@@ -38,9 +38,9 @@
#include "mongo/db/repl/read_concern_args.h"
#include "mongo/db/s/config/config_server_test_fixture.h"
#include "mongo/db/s/config/sharding_catalog_manager.h"
+#include "mongo/db/s/dist_lock_catalog_replset.h"
#include "mongo/rpc/get_status_from_command_result.h"
#include "mongo/rpc/metadata/tracking_metadata.h"
-#include "mongo/s/catalog/dist_lock_catalog_impl.h"
#include "mongo/s/catalog/type_database.h"
#include "mongo/s/catalog/type_shard.h"
#include "mongo/util/time_support.h"
diff --git a/src/mongo/db/s/config/sharding_catalog_manager_enable_sharding_test.cpp b/src/mongo/db/s/config/sharding_catalog_manager_enable_sharding_test.cpp
index 393e8533e50..6a09a6f1358 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager_enable_sharding_test.cpp
+++ b/src/mongo/db/s/config/sharding_catalog_manager_enable_sharding_test.cpp
@@ -40,7 +40,6 @@
#include "mongo/db/s/config/sharding_catalog_manager.h"
#include "mongo/rpc/get_status_from_command_result.h"
#include "mongo/rpc/metadata/tracking_metadata.h"
-#include "mongo/s/catalog/dist_lock_catalog_impl.h"
#include "mongo/s/catalog/type_database.h"
#include "mongo/s/catalog/type_shard.h"
#include "mongo/s/client/shard_registry.h"
diff --git a/src/mongo/db/s/dist_lock_catalog.cpp b/src/mongo/db/s/dist_lock_catalog.cpp
new file mode 100644
index 00000000000..cb9ec13c1a9
--- /dev/null
+++ b/src/mongo/db/s/dist_lock_catalog.cpp
@@ -0,0 +1,52 @@
+/**
+ * Copyright (C) 2018-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/s/dist_lock_catalog.h"
+
+namespace mongo {
+
+const WriteConcernOptions DistLockCatalog::kLocalWriteConcern(1,
+ WriteConcernOptions::SyncMode::UNSET,
+ Milliseconds(0));
+
+const WriteConcernOptions DistLockCatalog::kMajorityWriteConcern(
+ WriteConcernOptions::kMajority,
+ // Note: Even though we're setting UNSET here, kMajority implies JOURNAL if journaling is
+ // supported by this mongod.
+ WriteConcernOptions::SyncMode::UNSET,
+ WriteConcernOptions::kWriteConcernTimeoutSystem);
+
+DistLockCatalog::DistLockCatalog() = default;
+
+DistLockCatalog::ServerInfo::ServerInfo(Date_t time, OID _electionId)
+ : serverTime(std::move(time)), electionId(std::move(_electionId)) {}
+
+} // namespace mongo
diff --git a/src/mongo/db/s/dist_lock_catalog.h b/src/mongo/db/s/dist_lock_catalog.h
new file mode 100644
index 00000000000..7f774915b66
--- /dev/null
+++ b/src/mongo/db/s/dist_lock_catalog.h
@@ -0,0 +1,187 @@
+/**
+ * Copyright (C) 2018-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/base/string_data.h"
+#include "mongo/bson/oid.h"
+#include "mongo/db/write_concern_options.h"
+#include "mongo/util/time_support.h"
+
+namespace mongo {
+
+class LockpingsType;
+class LocksType;
+class OperationContext;
+class Status;
+template <typename T>
+class StatusWith;
+
+/**
+ * Interface for the distributed lock operations.
+ */
+class DistLockCatalog {
+ DistLockCatalog(const DistLockCatalog&) = delete;
+ DistLockCatalog& operator=(const DistLockCatalog&) = delete;
+
+public:
+ static const WriteConcernOptions kLocalWriteConcern;
+ static const WriteConcernOptions kMajorityWriteConcern;
+
+ /**
+ * Simple data structure for storing server local time and election id.
+ */
+ struct ServerInfo {
+ public:
+ ServerInfo(Date_t time, OID electionId);
+
+ // The local time of the server at the time this was created.
+ Date_t serverTime;
+
+ // The election id of the replica set member at the time this was created.
+ OID electionId;
+ };
+
+ virtual ~DistLockCatalog() = default;
+
+ /**
+ * Returns the ping document of the specified processID.
+ * Common status errors include socket errors.
+ */
+ virtual StatusWith<LockpingsType> getPing(OperationContext* opCtx, StringData processID) = 0;
+
+ /**
+ * Updates the ping document. Creates a new entry if it does not exists.
+ * Common status errors include socket errors.
+ */
+ virtual Status ping(OperationContext* opCtx, StringData processID, Date_t ping) = 0;
+
+ /**
+ * Attempts to update the owner of a lock identified by lockID to lockSessionID.
+ * Will only be successful if lock is not held.
+ *
+ * The other parameters are for diagnostic purposes:
+ * - who: unique string for the caller trying to grab the lock.
+ * - processId: unique string for the process trying to grab the lock.
+ * - time: the time when this is attempted.
+ * - why: reason for taking the lock.
+ *
+ * Returns the result of the operation.
+ * Returns LockStateChangeFailed if the lock acquisition cannot be done because lock
+ * is already held elsewhere.
+ *
+ * Common status errors include socket and duplicate key errors.
+ */
+ virtual StatusWith<LocksType> grabLock(
+ OperationContext* opCtx,
+ StringData lockID,
+ const OID& lockSessionID,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why,
+ const WriteConcernOptions& writeConcern = kMajorityWriteConcern) = 0;
+
+ /**
+ * Attempts to forcefully transfer the ownership of a lock from currentHolderTS
+ * to lockSessionID.
+ *
+ * The other parameters are for diagnostic purposes:
+ * - who: unique string for the caller trying to grab the lock.
+ * - processId: unique string for the process trying to grab the lock.
+ * - time: the time when this is attempted.
+ * - why: reason for taking the lock.
+ *
+ * Returns the result of the operation.
+ * Returns LockStateChangeFailed if the lock acquisition fails.
+ *
+ * Common status errors include socket errors.
+ */
+ virtual StatusWith<LocksType> overtakeLock(OperationContext* opCtx,
+ StringData lockID,
+ const OID& lockSessionID,
+ const OID& currentHolderTS,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) = 0;
+
+ /**
+ * Attempts to set the state of the lock document with lockSessionID to unlocked. Returns OK,
+ * if at the end of this call it is determined that the lock is definitely not owned by the
+ * specified session (i.e., it is not owned at all or if it is owned by a different session).
+ * Otherwise, it returns an error status. Common errors include socket errors.
+ */
+ virtual Status unlock(OperationContext* opCtx, const OID& lockSessionID) = 0;
+
+ /**
+ * Same as unlock() above except that it unlocks the lock document that matches "lockSessionID"
+ * AND "name", rather than just "lockSessionID". This is necessary if multiple documents have
+ * been locked with the same lockSessionID.
+ */
+ virtual Status unlock(OperationContext* opCtx, const OID& lockSessionID, StringData name) = 0;
+
+ /**
+ * Unlocks all distributed locks with the given owning process ID. Does not provide any
+ * indication as to how many locks were actually unlocked. So long as the update command runs
+ * successfully, returns OK, otherwise returns an error status.
+ */
+ virtual Status unlockAll(OperationContext* opCtx, const std::string& processID) = 0;
+
+ /**
+ * Get some information from the config server primary.
+ * Common status errors include socket errors.
+ */
+ virtual StatusWith<ServerInfo> getServerInfo(OperationContext* opCtx) = 0;
+
+ /**
+ * Returns the lock document.
+ * Returns LockNotFound if lock document doesn't exist.
+ * Common status errors include socket errors.
+ */
+ virtual StatusWith<LocksType> getLockByTS(OperationContext* opCtx,
+ const OID& lockSessionID) = 0;
+
+ /**
+ * Returns the lock document.
+ * Common status errors include socket errors.
+ */
+ virtual StatusWith<LocksType> getLockByName(OperationContext* opCtx, StringData name) = 0;
+
+ /**
+ * Attempts to delete the ping document corresponding to the given processId.
+ * Common status errors include socket errors.
+ */
+ virtual Status stopPing(OperationContext* opCtx, StringData processId) = 0;
+
+protected:
+ DistLockCatalog();
+};
+
+} // namespace mongo
diff --git a/src/mongo/db/s/dist_lock_catalog_mock.cpp b/src/mongo/db/s/dist_lock_catalog_mock.cpp
new file mode 100644
index 00000000000..fba94f75502
--- /dev/null
+++ b/src/mongo/db/s/dist_lock_catalog_mock.cpp
@@ -0,0 +1,357 @@
+/**
+ * Copyright (C) 2018-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/s/dist_lock_catalog_mock.h"
+
+#include "mongo/unittest/unittest.h"
+#include "mongo/util/str.h"
+
+namespace mongo {
+namespace {
+
+Status kBadRetValue(ErrorCodes::InternalError, "no return value");
+StatusWith<LocksType> kLocksTypeBadRetValue(kBadRetValue);
+StatusWith<LockpingsType> kLockpingsTypeBadRetValue(kBadRetValue);
+StatusWith<DistLockCatalog::ServerInfo> kServerInfoBadRetValue(kBadRetValue);
+
+void noGrabLockFuncSet(StringData lockID,
+ const OID& lockSessionID,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ FAIL(str::stream() << "grabLock not expected to be called. "
+ << "lockID: " << lockID << ", who: " << who << ", processId: " << processId
+ << ", why: " << why);
+}
+
+void noOvertakeLockFuncSet(StringData lockID,
+ const OID& lockSessionID,
+ const OID& currentHolderTS,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ FAIL(str::stream() << "overtakeLock not expected to be called. "
+ << "lockID: " << lockID << ", currentHolderTS: " << currentHolderTS
+ << ", who: " << who << ", processId: " << processId << ", why: " << why);
+}
+
+void noUnLockFuncSet(const OID& lockSessionID) {
+ FAIL(str::stream() << "unlock not expected to be called. "
+ << "lockSessionID: " << lockSessionID);
+}
+
+void noPingFuncSet(StringData processID, Date_t ping) {
+ // Ping is expected to be called all the time, so default behavior is do nothing.
+}
+
+void noStopPingFuncSet(StringData processID) {
+ FAIL(str::stream() << "stopPing not expected to be called. "
+ << "processID: " << processID);
+}
+
+void noGetLockByTSSet(const OID& lockSessionID) {
+ FAIL(str::stream() << "getLockByTS not expected to be called. "
+ << "lockSessionID: " << lockSessionID);
+}
+
+void noGetLockByNameSet(StringData name) {
+ FAIL(str::stream() << "getLockByName not expected to be called. "
+ << "lockName: " << name);
+}
+
+void noGetPingSet(StringData processId) {
+ FAIL(str::stream() << "getPing not expected to be called. "
+ << "lockName: " << processId);
+}
+
+void noGetServerInfoSet() {
+ FAIL("getServerInfo not expected to be called");
+}
+
+} // namespace
+
+DistLockCatalogMock::DistLockCatalogMock()
+ : _grabLockChecker(noGrabLockFuncSet),
+ _grabLockReturnValue(kLocksTypeBadRetValue),
+ _unlockChecker(noUnLockFuncSet),
+ _unlockReturnValue(kBadRetValue),
+ _pingChecker(noPingFuncSet),
+ _pingReturnValue(kBadRetValue),
+ _stopPingChecker(noStopPingFuncSet),
+ _stopPingReturnValue(kBadRetValue),
+ _getLockByTSChecker(noGetLockByTSSet),
+ _getLockByTSReturnValue(kLocksTypeBadRetValue),
+ _getLockByNameChecker(noGetLockByNameSet),
+ _getLockByNameReturnValue(kLocksTypeBadRetValue),
+ _overtakeLockChecker(noOvertakeLockFuncSet),
+ _overtakeLockReturnValue(kLocksTypeBadRetValue),
+ _getPingChecker(noGetPingSet),
+ _getPingReturnValue(kLockpingsTypeBadRetValue),
+ _getServerInfoChecker(noGetServerInfoSet),
+ _getServerInfoReturnValue(kServerInfoBadRetValue) {}
+
+DistLockCatalogMock::~DistLockCatalogMock() {}
+
+StatusWith<LockpingsType> DistLockCatalogMock::getPing(OperationContext* opCtx,
+ StringData processID) {
+ auto ret = kLockpingsTypeBadRetValue;
+ GetPingFunc checkerFunc = noGetPingSet;
+
+ {
+ stdx::lock_guard<Latch> lk(_mutex);
+ ret = _getPingReturnValue;
+ checkerFunc = _getPingChecker;
+ }
+
+ checkerFunc(processID);
+ return ret;
+}
+
+Status DistLockCatalogMock::ping(OperationContext* opCtx, StringData processID, Date_t ping) {
+ auto ret = kBadRetValue;
+ PingFunc checkerFunc = noPingFuncSet;
+
+ {
+ stdx::lock_guard<Latch> lk(_mutex);
+ ret = _pingReturnValue;
+ checkerFunc = _pingChecker;
+ }
+
+ checkerFunc(processID, ping);
+ return ret;
+}
+
+StatusWith<LocksType> DistLockCatalogMock::grabLock(OperationContext* opCtx,
+ StringData lockID,
+ const OID& lockSessionID,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why,
+ const WriteConcernOptions& writeConcern) {
+ auto ret = kLocksTypeBadRetValue;
+ GrabLockFunc checkerFunc = noGrabLockFuncSet;
+
+ {
+ stdx::lock_guard<Latch> lk(_mutex);
+ ret = _grabLockReturnValue;
+ checkerFunc = _grabLockChecker;
+ }
+
+ checkerFunc(lockID, lockSessionID, who, processId, time, why);
+ return ret;
+}
+
+StatusWith<LocksType> DistLockCatalogMock::overtakeLock(OperationContext* opCtx,
+ StringData lockID,
+ const OID& lockSessionID,
+ const OID& currentHolderTS,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ auto ret = kLocksTypeBadRetValue;
+ OvertakeLockFunc checkerFunc = noOvertakeLockFuncSet;
+
+ {
+ stdx::lock_guard<Latch> lk(_mutex);
+ ret = _overtakeLockReturnValue;
+ checkerFunc = _overtakeLockChecker;
+ }
+
+ checkerFunc(lockID, lockSessionID, currentHolderTS, who, processId, time, why);
+ return ret;
+}
+
+Status DistLockCatalogMock::unlock(OperationContext* opCtx, const OID& lockSessionID) {
+ auto ret = kBadRetValue;
+ UnlockFunc checkerFunc = noUnLockFuncSet;
+
+ {
+ stdx::lock_guard<Latch> lk(_mutex);
+ ret = _unlockReturnValue;
+ checkerFunc = _unlockChecker;
+ }
+
+ checkerFunc(lockSessionID);
+ return ret;
+}
+
+Status DistLockCatalogMock::unlock(OperationContext* opCtx,
+ const OID& lockSessionID,
+ StringData name) {
+ auto ret = kBadRetValue;
+ UnlockFunc checkerFunc = noUnLockFuncSet;
+
+ {
+ stdx::lock_guard<Latch> lk(_mutex);
+ ret = _unlockReturnValue;
+ checkerFunc = _unlockChecker;
+ }
+
+ checkerFunc(lockSessionID);
+
+ return ret;
+}
+
+StatusWith<DistLockCatalog::ServerInfo> DistLockCatalogMock::getServerInfo(
+ OperationContext* opCtx) {
+ auto ret = kServerInfoBadRetValue;
+ GetServerInfoFunc checkerFunc = noGetServerInfoSet;
+
+ {
+ stdx::lock_guard<Latch> lk(_mutex);
+ ret = _getServerInfoReturnValue;
+ checkerFunc = _getServerInfoChecker;
+ }
+
+ checkerFunc();
+ return ret;
+}
+
+StatusWith<LocksType> DistLockCatalogMock::getLockByTS(OperationContext* opCtx,
+ const OID& lockSessionID) {
+ auto ret = kLocksTypeBadRetValue;
+ GetLockByTSFunc checkerFunc = noGetLockByTSSet;
+
+ {
+ stdx::lock_guard<Latch> lk(_mutex);
+ ret = _getLockByTSReturnValue;
+ checkerFunc = _getLockByTSChecker;
+ }
+
+ checkerFunc(lockSessionID);
+ return ret;
+}
+
+StatusWith<LocksType> DistLockCatalogMock::getLockByName(OperationContext* opCtx, StringData name) {
+ auto ret = kLocksTypeBadRetValue;
+ GetLockByNameFunc checkerFunc = noGetLockByNameSet;
+
+ {
+ stdx::lock_guard<Latch> lk(_mutex);
+ ret = _getLockByNameReturnValue;
+ checkerFunc = _getLockByNameChecker;
+ }
+
+ checkerFunc(name);
+ return ret;
+}
+
+Status DistLockCatalogMock::stopPing(OperationContext* opCtx, StringData processId) {
+ auto ret = kBadRetValue;
+ StopPingFunc checkerFunc = noStopPingFuncSet;
+
+ {
+ stdx::lock_guard<Latch> lk(_mutex);
+ ret = _stopPingReturnValue;
+ checkerFunc = _stopPingChecker;
+ }
+
+ checkerFunc(processId);
+ return ret;
+}
+
+void DistLockCatalogMock::expectGrabLock(DistLockCatalogMock::GrabLockFunc checkerFunc,
+ StatusWith<LocksType> returnThis) {
+ stdx::lock_guard<Latch> lk(_mutex);
+ _grabLockChecker = checkerFunc;
+ _grabLockReturnValue = returnThis;
+}
+
+void DistLockCatalogMock::expectNoGrabLock() {
+ stdx::lock_guard<Latch> lk(_mutex);
+ _grabLockChecker = noGrabLockFuncSet;
+ _grabLockReturnValue = kLocksTypeBadRetValue;
+}
+
+void DistLockCatalogMock::expectUnLock(DistLockCatalogMock::UnlockFunc checkerFunc,
+ Status returnThis) {
+ stdx::lock_guard<Latch> lk(_mutex);
+ _unlockChecker = checkerFunc;
+ _unlockReturnValue = returnThis;
+}
+
+void DistLockCatalogMock::expectPing(DistLockCatalogMock::PingFunc checkerFunc, Status returnThis) {
+ stdx::lock_guard<Latch> lk(_mutex);
+ _pingChecker = checkerFunc;
+ _pingReturnValue = returnThis;
+}
+
+void DistLockCatalogMock::expectStopPing(StopPingFunc checkerFunc, Status returnThis) {
+ stdx::lock_guard<Latch> lk(_mutex);
+ _stopPingChecker = checkerFunc;
+ _stopPingReturnValue = returnThis;
+}
+
+void DistLockCatalogMock::expectGetLockByTS(GetLockByTSFunc checkerFunc,
+ StatusWith<LocksType> returnThis) {
+ stdx::lock_guard<Latch> lk(_mutex);
+ _getLockByTSChecker = checkerFunc;
+ _getLockByTSReturnValue = returnThis;
+}
+
+void DistLockCatalogMock::expectGetLockByName(GetLockByNameFunc checkerFunc,
+ StatusWith<LocksType> returnThis) {
+ stdx::lock_guard<Latch> lk(_mutex);
+ _getLockByNameChecker = checkerFunc;
+ _getLockByNameReturnValue = returnThis;
+}
+
+void DistLockCatalogMock::expectOvertakeLock(OvertakeLockFunc checkerFunc,
+ StatusWith<LocksType> returnThis) {
+ stdx::lock_guard<Latch> lk(_mutex);
+ _overtakeLockChecker = checkerFunc;
+ _overtakeLockReturnValue = returnThis;
+}
+
+void DistLockCatalogMock::expectGetPing(GetPingFunc checkerFunc,
+ StatusWith<LockpingsType> returnThis) {
+ stdx::lock_guard<Latch> lk(_mutex);
+ _getPingChecker = checkerFunc;
+ _getPingReturnValue = returnThis;
+}
+
+void DistLockCatalogMock::expectGetServerInfo(GetServerInfoFunc checkerFunc,
+ StatusWith<DistLockCatalog::ServerInfo> returnThis) {
+ stdx::lock_guard<Latch> lk(_mutex);
+ _getServerInfoChecker = checkerFunc;
+ _getServerInfoReturnValue = returnThis;
+}
+
+Status DistLockCatalogMock::unlockAll(OperationContext* opCtx, const std::string& processID) {
+ return Status(ErrorCodes::IllegalOperation,
+ str::stream() << "unlockAll not expected to be called; processID: " << processID);
+}
+
+} // namespace mongo
diff --git a/src/mongo/db/s/dist_lock_catalog_mock.h b/src/mongo/db/s/dist_lock_catalog_mock.h
new file mode 100644
index 00000000000..04996f05633
--- /dev/null
+++ b/src/mongo/db/s/dist_lock_catalog_mock.h
@@ -0,0 +1,223 @@
+/**
+ * Copyright (C) 2018-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#pragma once
+
+#include <functional>
+
+#include "mongo/db/s/dist_lock_catalog.h"
+#include "mongo/platform/mutex.h"
+#include "mongo/s/catalog/type_lockpings.h"
+#include "mongo/s/catalog/type_locks.h"
+
+namespace mongo {
+
+/**
+ * Mock implementation of DistLockCatalog for testing.
+ *
+ * Example usage:
+ *
+ * DistLockCatalogMock mock;
+ * LocksType badLock;
+ * mock.expectGrabLock([](StringData lockID,
+ * const OID& lockSessionID,
+ * StringData who,
+ * StringData processId,
+ * Date_t time,
+ * StringData why) {
+ * ASSERT_EQUALS("test", lockID);
+ * }, badLock);
+ *
+ * mock.grabLock("test", OID(), "me", "x", Date_t::now(), "end");
+ *
+ * It is also possible to chain the callbacks. For example, if we want to set the test
+ * such that grabLock can only be called once, you can do this:
+ *
+ * DistLockCatalogMock mock;
+ * mock.expectGrabLock([&mock](...) {
+ * mock.expectNoGrabLock();
+ * }, Status::OK());
+ */
+class DistLockCatalogMock : public DistLockCatalog {
+public:
+ DistLockCatalogMock();
+ virtual ~DistLockCatalogMock();
+
+ using GrabLockFunc = std::function<void(StringData lockID,
+ const OID& lockSessionID,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why)>;
+ using OvertakeLockFunc = std::function<void(StringData lockID,
+ const OID& lockSessionID,
+ const OID& currentHolderTS,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why)>;
+ using UnlockFunc = std::function<void(const OID& lockSessionID)>;
+ using PingFunc = std::function<void(StringData processID, Date_t ping)>;
+ using StopPingFunc = std::function<void(StringData processID)>;
+ using GetPingFunc = StopPingFunc;
+ using GetLockByTSFunc = std::function<void(const OID& ts)>;
+ using GetLockByNameFunc = std::function<void(StringData name)>;
+ using GetServerInfoFunc = std::function<void()>;
+
+ virtual StatusWith<LockpingsType> getPing(OperationContext* opCtx,
+ StringData processID) override;
+
+ virtual Status ping(OperationContext* opCtx, StringData processID, Date_t ping) override;
+
+ virtual StatusWith<LocksType> grabLock(OperationContext* opCtx,
+ StringData lockID,
+ const OID& lockSessionID,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why,
+ const WriteConcernOptions& writeConcern) override;
+
+ virtual StatusWith<LocksType> overtakeLock(OperationContext* opCtx,
+ StringData lockID,
+ const OID& lockSessionID,
+ const OID& currentHolderTS,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) override;
+
+ virtual Status unlock(OperationContext* opCtx, const OID& lockSessionID) override;
+
+ virtual Status unlock(OperationContext* opCtx,
+ const OID& lockSessionID,
+ StringData name) override;
+
+ virtual Status unlockAll(OperationContext* opCtx, const std::string& processID) override;
+
+ virtual StatusWith<ServerInfo> getServerInfo(OperationContext* opCtx) override;
+
+ virtual StatusWith<LocksType> getLockByTS(OperationContext* opCtx,
+ const OID& lockSessionID) override;
+
+ virtual StatusWith<LocksType> getLockByName(OperationContext* opCtx, StringData name) override;
+
+ virtual Status stopPing(OperationContext* opCtx, StringData processId) override;
+
+ /**
+ * Sets the checker method to use and the return value for grabLock to return every
+ * time it is called.
+ */
+ void expectGrabLock(GrabLockFunc checkerFunc, StatusWith<LocksType> returnThis);
+
+ /**
+ * Expect grabLock to never be called after this is called.
+ */
+ void expectNoGrabLock();
+
+ /**
+ * Sets the checker method to use and the return value for unlock to return every
+ * time it is called.
+ */
+ void expectUnLock(UnlockFunc checkerFunc, Status returnThis);
+
+ /**
+ * Sets the checker method to use and its return value the every time ping is called.
+ */
+ void expectPing(PingFunc checkerFunc, Status returnThis);
+
+ /**
+ * Sets the checker method to use and its return value the every time stopPing is called.
+ */
+ void expectStopPing(StopPingFunc checkerFunc, Status returnThis);
+
+ /**
+ * Sets the checker method to use and its return value the every time
+ * getLockByTS is called.
+ */
+ void expectGetLockByTS(GetLockByTSFunc checkerFunc, StatusWith<LocksType> returnThis);
+
+ /**
+ * Sets the checker method to use and its return value the every time
+ * getLockByName is called.
+ */
+ void expectGetLockByName(GetLockByNameFunc checkerFunc, StatusWith<LocksType> returnThis);
+
+ /**
+ * Sets the checker method to use and its return value the every time
+ * overtakeLock is called.
+ */
+ void expectOvertakeLock(OvertakeLockFunc checkerFunc, StatusWith<LocksType> returnThis);
+
+ /**
+ * Sets the checker method to use and its return value the every time
+ * getPing is called.
+ */
+ void expectGetPing(GetPingFunc checkerFunc, StatusWith<LockpingsType> returnThis);
+
+ /**
+ * Sets the checker method to use and its return value the every time
+ * getServerInfo is called.
+ */
+ void expectGetServerInfo(GetServerInfoFunc checkerFunc,
+ StatusWith<DistLockCatalog::ServerInfo> returnThis);
+
+private:
+ // Protects all the member variables.
+ Mutex _mutex = MONGO_MAKE_LATCH("DistLockCatalogMock::_mutex");
+
+ GrabLockFunc _grabLockChecker;
+ StatusWith<LocksType> _grabLockReturnValue;
+
+ UnlockFunc _unlockChecker;
+ Status _unlockReturnValue;
+
+ PingFunc _pingChecker;
+ Status _pingReturnValue;
+
+ StopPingFunc _stopPingChecker;
+ Status _stopPingReturnValue;
+
+ GetLockByTSFunc _getLockByTSChecker;
+ StatusWith<LocksType> _getLockByTSReturnValue;
+
+ GetLockByNameFunc _getLockByNameChecker;
+ StatusWith<LocksType> _getLockByNameReturnValue;
+
+ OvertakeLockFunc _overtakeLockChecker;
+ StatusWith<LocksType> _overtakeLockReturnValue;
+
+ GetPingFunc _getPingChecker;
+ StatusWith<LockpingsType> _getPingReturnValue;
+
+ GetServerInfoFunc _getServerInfoChecker;
+ StatusWith<DistLockCatalog::ServerInfo> _getServerInfoReturnValue;
+};
+
+} // namespace mongo
diff --git a/src/mongo/db/s/dist_lock_catalog_replset.cpp b/src/mongo/db/s/dist_lock_catalog_replset.cpp
new file mode 100644
index 00000000000..a9997a6fd60
--- /dev/null
+++ b/src/mongo/db/s/dist_lock_catalog_replset.cpp
@@ -0,0 +1,545 @@
+/**
+ * Copyright (C) 2018-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/s/dist_lock_catalog_replset.h"
+
+#include <string>
+
+#include "mongo/base/status.h"
+#include "mongo/base/status_with.h"
+#include "mongo/bson/util/bson_extract.h"
+#include "mongo/client/read_preference.h"
+#include "mongo/db/lasterror.h"
+#include "mongo/db/repl/read_concern_args.h"
+#include "mongo/rpc/get_status_from_command_result.h"
+#include "mongo/rpc/metadata.h"
+#include "mongo/rpc/metadata/repl_set_metadata.h"
+#include "mongo/rpc/metadata/sharding_metadata.h"
+#include "mongo/s/catalog/type_lockpings.h"
+#include "mongo/s/catalog/type_locks.h"
+#include "mongo/s/client/shard_registry.h"
+#include "mongo/s/grid.h"
+#include "mongo/s/write_ops/batched_command_request.h"
+#include "mongo/s/write_ops/batched_command_response.h"
+#include "mongo/util/time_support.h"
+
+namespace mongo {
+
+using std::string;
+using std::vector;
+
+namespace {
+
+const char kFindAndModifyResponseResultDocField[] = "value";
+const char kLocalTimeField[] = "localTime";
+
+const ReadPreferenceSetting kReadPref(ReadPreference::PrimaryOnly, TagSet());
+
+/**
+ * Returns the resulting new object from the findAndModify response object.
+ * Returns LockStateChangeFailed if value field was null, which indicates that
+ * the findAndModify command did not modify any document.
+ * This also checks for errors in the response object.
+ */
+StatusWith<BSONObj> extractFindAndModifyNewObj(StatusWith<Shard::CommandResponse> response) {
+ if (!response.isOK()) {
+ return response.getStatus();
+ }
+ if (!response.getValue().commandStatus.isOK()) {
+ return response.getValue().commandStatus;
+ }
+ if (!response.getValue().writeConcernStatus.isOK()) {
+ return response.getValue().writeConcernStatus;
+ }
+
+ auto responseObj = std::move(response.getValue().response);
+
+ if (const auto& newDocElem = responseObj[kFindAndModifyResponseResultDocField]) {
+ if (newDocElem.isNull()) {
+ return {ErrorCodes::LockStateChangeFailed,
+ "findAndModify query predicate didn't match any lock document"};
+ }
+
+ if (!newDocElem.isABSONObj()) {
+ return {ErrorCodes::UnsupportedFormat,
+ str::stream() << "expected an object from the findAndModify response '"
+ << kFindAndModifyResponseResultDocField
+ << "'field, got: " << newDocElem};
+ }
+
+ return newDocElem.Obj().getOwned();
+ }
+
+ return {ErrorCodes::UnsupportedFormat,
+ str::stream() << "no '" << kFindAndModifyResponseResultDocField
+ << "' in findAndModify response"};
+}
+
+/**
+ * Extract the electionId from a serverStatus command response.
+ */
+StatusWith<OID> extractElectionId(const BSONObj& responseObj) {
+ BSONElement replElem;
+ auto replElemStatus = bsonExtractTypedField(responseObj, "repl", Object, &replElem);
+
+ if (!replElemStatus.isOK()) {
+ return {ErrorCodes::UnsupportedFormat, replElemStatus.reason()};
+ }
+
+ const auto replSubObj = replElem.Obj();
+ OID electionId;
+ auto electionIdStatus = bsonExtractOIDField(replSubObj, "electionId", &electionId);
+
+ if (!electionIdStatus.isOK()) {
+ // Secondaries don't have electionId.
+ if (electionIdStatus.code() == ErrorCodes::NoSuchKey) {
+ // Verify that the from replSubObj that this is indeed not a primary.
+ bool isPrimary = false;
+ auto isPrimaryStatus = bsonExtractBooleanField(replSubObj, "ismaster", &isPrimary);
+
+ if (!isPrimaryStatus.isOK()) {
+ return {ErrorCodes::UnsupportedFormat, isPrimaryStatus.reason()};
+ }
+
+ if (isPrimary) {
+ string hostContacted;
+ auto hostContactedStatus = bsonExtractStringField(replSubObj, "me", &hostContacted);
+
+ if (!hostContactedStatus.isOK()) {
+ return {
+ ErrorCodes::UnsupportedFormat,
+ str::stream()
+ << "failed to extract 'me' field from repl subsection of serverStatus: "
+ << hostContactedStatus.reason()};
+ }
+
+ return {ErrorCodes::UnsupportedFormat,
+ str::stream() << "expected primary to have electionId but not present on "
+ << hostContacted};
+ }
+
+ return {ErrorCodes::NotWritablePrimary, "only primary can have electionId"};
+ }
+
+ return {ErrorCodes::UnsupportedFormat, electionIdStatus.reason()};
+ }
+
+ return electionId;
+}
+
+write_ops::FindAndModifyCommand makeFindAndModifyRequest(
+ NamespaceString fullNs, BSONObj query, boost::optional<write_ops::UpdateModification> update) {
+ auto request = write_ops::FindAndModifyCommand(fullNs);
+ request.setQuery(query);
+ if (update) {
+ request.setUpdate(std::move(update));
+ } else {
+ request.setRemove(true);
+ }
+ return request;
+}
+
+} // unnamed namespace
+
+DistLockCatalogImpl::DistLockCatalogImpl()
+ : _lockPingNS(LockpingsType::ConfigNS), _locksNS(LocksType::ConfigNS) {}
+
+DistLockCatalogImpl::~DistLockCatalogImpl() = default;
+
+StatusWith<LockpingsType> DistLockCatalogImpl::getPing(OperationContext* opCtx,
+ StringData processID) {
+ auto findResult = _findOnConfig(
+ opCtx, kReadPref, _lockPingNS, BSON(LockpingsType::process() << processID), {}, 1);
+
+ if (!findResult.isOK()) {
+ return findResult.getStatus();
+ }
+
+ const auto& findResultSet = findResult.getValue();
+
+ if (findResultSet.empty()) {
+ return {ErrorCodes::NoMatchingDocument,
+ str::stream() << "ping entry for " << processID << " not found"};
+ }
+
+ BSONObj doc = findResultSet.front();
+ auto pingDocResult = LockpingsType::fromBSON(doc);
+ if (!pingDocResult.isOK()) {
+ return {ErrorCodes::FailedToParse,
+ str::stream() << "failed to parse document: " << doc << " : "
+ << pingDocResult.getStatus().toString()};
+ }
+
+ return pingDocResult.getValue();
+}
+
+Status DistLockCatalogImpl::ping(OperationContext* opCtx, StringData processID, Date_t ping) {
+ auto request = write_ops::FindAndModifyCommand(_lockPingNS);
+ request.setQuery(BSON(LockpingsType::process() << processID));
+ request.setUpdate(write_ops::UpdateModification::parseFromClassicUpdate(
+ BSON("$set" << BSON(LockpingsType::ping(ping)))));
+ request.setUpsert(true);
+ request.setWriteConcern(kMajorityWriteConcern.toBSON());
+
+ auto const shardRegistry = Grid::get(opCtx)->shardRegistry();
+ auto resultStatus = shardRegistry->getConfigShard()->runCommandWithFixedRetryAttempts(
+ opCtx,
+ ReadPreferenceSetting{ReadPreference::PrimaryOnly},
+ _locksNS.db().toString(),
+ request.toBSON({}),
+ Shard::kDefaultConfigCommandTimeout,
+ Shard::RetryPolicy::kNotIdempotent);
+
+ auto findAndModifyStatus = extractFindAndModifyNewObj(std::move(resultStatus));
+ return findAndModifyStatus.getStatus();
+}
+
+StatusWith<LocksType> DistLockCatalogImpl::grabLock(OperationContext* opCtx,
+ StringData lockID,
+ const OID& lockSessionID,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why,
+ const WriteConcernOptions& writeConcern) {
+ BSONObj newLockDetails(BSON(LocksType::lockID(lockSessionID)
+ << LocksType::state(LocksType::LOCKED) << LocksType::who() << who
+ << LocksType::process() << processId << LocksType::when(time)
+ << LocksType::why() << why));
+
+ auto request = makeFindAndModifyRequest(
+ _locksNS,
+ BSON(LocksType::name() << lockID << LocksType::state(LocksType::UNLOCKED)),
+ write_ops::UpdateModification::parseFromClassicUpdate(BSON("$set" << newLockDetails)));
+ request.setUpsert(true);
+ request.setNew(true);
+ request.setWriteConcern(writeConcern.toBSON());
+
+ auto const shardRegistry = Grid::get(opCtx)->shardRegistry();
+ auto resultStatus = shardRegistry->getConfigShard()->runCommandWithFixedRetryAttempts(
+ opCtx,
+ ReadPreferenceSetting{ReadPreference::PrimaryOnly},
+ _locksNS.db().toString(),
+ request.toBSON({}),
+ Shard::kDefaultConfigCommandTimeout,
+ Shard::RetryPolicy::kNoRetry); // Dist lock manager is handling own retries
+
+ auto findAndModifyStatus = extractFindAndModifyNewObj(std::move(resultStatus));
+ if (!findAndModifyStatus.isOK()) {
+ if (findAndModifyStatus == ErrorCodes::DuplicateKey) {
+ // Another thread won the upsert race. Also see SERVER-14322.
+ return {ErrorCodes::LockStateChangeFailed,
+ str::stream() << "duplicateKey error during upsert of lock: " << lockID};
+ }
+
+ return findAndModifyStatus.getStatus();
+ }
+
+ BSONObj doc = findAndModifyStatus.getValue();
+ auto locksTypeResult = LocksType::fromBSON(doc);
+ if (!locksTypeResult.isOK()) {
+ return {ErrorCodes::FailedToParse,
+ str::stream() << "failed to parse: " << doc << " : "
+ << locksTypeResult.getStatus().toString()};
+ }
+
+ return locksTypeResult.getValue();
+}
+
+StatusWith<LocksType> DistLockCatalogImpl::overtakeLock(OperationContext* opCtx,
+ StringData lockID,
+ const OID& lockSessionID,
+ const OID& currentHolderTS,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ BSONArrayBuilder orQueryBuilder;
+ orQueryBuilder.append(
+ BSON(LocksType::name() << lockID << LocksType::state(LocksType::UNLOCKED)));
+ orQueryBuilder.append(BSON(LocksType::name() << lockID << LocksType::lockID(currentHolderTS)));
+
+ BSONObj newLockDetails(BSON(LocksType::lockID(lockSessionID)
+ << LocksType::state(LocksType::LOCKED) << LocksType::who() << who
+ << LocksType::process() << processId << LocksType::when(time)
+ << LocksType::why() << why));
+
+ auto request = makeFindAndModifyRequest(
+ _locksNS,
+ BSON("$or" << orQueryBuilder.arr()),
+ write_ops::UpdateModification::parseFromClassicUpdate(BSON("$set" << newLockDetails)));
+ request.setNew(true);
+ request.setWriteConcern(kMajorityWriteConcern.toBSON());
+
+ auto const shardRegistry = Grid::get(opCtx)->shardRegistry();
+ auto resultStatus = shardRegistry->getConfigShard()->runCommandWithFixedRetryAttempts(
+ opCtx,
+ ReadPreferenceSetting{ReadPreference::PrimaryOnly},
+ _locksNS.db().toString(),
+ request.toBSON({}),
+ Shard::kDefaultConfigCommandTimeout,
+ Shard::RetryPolicy::kNotIdempotent);
+
+ auto findAndModifyStatus = extractFindAndModifyNewObj(std::move(resultStatus));
+ if (!findAndModifyStatus.isOK()) {
+ return findAndModifyStatus.getStatus();
+ }
+
+ BSONObj doc = findAndModifyStatus.getValue();
+ auto locksTypeResult = LocksType::fromBSON(doc);
+ if (!locksTypeResult.isOK()) {
+ return {ErrorCodes::FailedToParse,
+ str::stream() << "failed to parse: " << doc << " : "
+ << locksTypeResult.getStatus().toString()};
+ }
+
+ return locksTypeResult.getValue();
+}
+
+Status DistLockCatalogImpl::unlock(OperationContext* opCtx, const OID& lockSessionID) {
+ auto request =
+ makeFindAndModifyRequest(_locksNS,
+ BSON(LocksType::lockID(lockSessionID)),
+ write_ops::UpdateModification::parseFromClassicUpdate(
+ BSON("$set" << BSON(LocksType::state(LocksType::UNLOCKED)))));
+ request.setWriteConcern(kMajorityWriteConcern.toBSON());
+ return _unlock(opCtx, request);
+}
+
+Status DistLockCatalogImpl::unlock(OperationContext* opCtx,
+ const OID& lockSessionID,
+ StringData name) {
+ auto request = makeFindAndModifyRequest(
+ _locksNS,
+ BSON(LocksType::lockID(lockSessionID) << LocksType::name(name.toString())),
+ write_ops::UpdateModification::parseFromClassicUpdate(
+ BSON("$set" << BSON(LocksType::state(LocksType::UNLOCKED)))));
+ request.setWriteConcern(kMajorityWriteConcern.toBSON());
+ return _unlock(opCtx, request);
+}
+
+Status DistLockCatalogImpl::_unlock(OperationContext* opCtx,
+ const write_ops::FindAndModifyCommand& request) {
+ auto const shardRegistry = Grid::get(opCtx)->shardRegistry();
+ auto resultStatus = shardRegistry->getConfigShard()->runCommandWithFixedRetryAttempts(
+ opCtx,
+ ReadPreferenceSetting{ReadPreference::PrimaryOnly},
+ _locksNS.db().toString(),
+ request.toBSON({}),
+ Shard::kDefaultConfigCommandTimeout,
+ Shard::RetryPolicy::kIdempotent);
+
+ auto findAndModifyStatus = extractFindAndModifyNewObj(std::move(resultStatus));
+ if (findAndModifyStatus == ErrorCodes::LockStateChangeFailed) {
+ // Did not modify any document, which implies that the lock already has a
+ // a different owner. This is ok since it means that the objective of
+ // releasing ownership of the lock has already been accomplished.
+ return Status::OK();
+ }
+
+ return findAndModifyStatus.getStatus();
+}
+
+Status DistLockCatalogImpl::unlockAll(OperationContext* opCtx, const std::string& processID) {
+ BatchedCommandRequest request([&] {
+ write_ops::Update updateOp(_locksNS);
+ updateOp.setUpdates({[&] {
+ write_ops::UpdateOpEntry entry;
+ entry.setQ(BSON(LocksType::process(processID)));
+ entry.setU(write_ops::UpdateModification::parseFromClassicUpdate(
+ BSON("$set" << BSON(LocksType::state(LocksType::UNLOCKED)))));
+ entry.setUpsert(false);
+ entry.setMulti(true);
+ return entry;
+ }()});
+ return updateOp;
+ }());
+ request.setWriteConcern(kLocalWriteConcern.toBSON());
+
+ BSONObj cmdObj = request.toBSON();
+
+ auto const shardRegistry = Grid::get(opCtx)->shardRegistry();
+ auto response = shardRegistry->getConfigShard()->runCommandWithFixedRetryAttempts(
+ opCtx,
+ ReadPreferenceSetting{ReadPreference::PrimaryOnly},
+ _locksNS.db().toString(),
+ cmdObj,
+ Shard::kDefaultConfigCommandTimeout,
+ Shard::RetryPolicy::kIdempotent);
+
+ if (!response.isOK()) {
+ return response.getStatus();
+ }
+ if (!response.getValue().commandStatus.isOK()) {
+ return response.getValue().commandStatus;
+ }
+ if (!response.getValue().writeConcernStatus.isOK()) {
+ return response.getValue().writeConcernStatus;
+ }
+
+ BatchedCommandResponse batchResponse;
+ std::string errmsg;
+ if (!batchResponse.parseBSON(response.getValue().response, &errmsg)) {
+ return Status(ErrorCodes::FailedToParse,
+ str::stream()
+ << "Failed to parse config server response to batch request for "
+ "unlocking existing distributed locks"
+ << causedBy(errmsg));
+ }
+ return batchResponse.toStatus();
+}
+
+StatusWith<DistLockCatalog::ServerInfo> DistLockCatalogImpl::getServerInfo(
+ OperationContext* opCtx) {
+ auto const shardRegistry = Grid::get(opCtx)->shardRegistry();
+ auto resultStatus = shardRegistry->getConfigShard()->runCommandWithFixedRetryAttempts(
+ opCtx,
+ kReadPref,
+ "admin",
+ BSON("serverStatus" << 1),
+ Shard::kDefaultConfigCommandTimeout,
+ Shard::RetryPolicy::kIdempotent);
+
+ if (!resultStatus.isOK()) {
+ return resultStatus.getStatus();
+ }
+ if (!resultStatus.getValue().commandStatus.isOK()) {
+ return resultStatus.getValue().commandStatus;
+ }
+
+ BSONObj responseObj(std::move(resultStatus.getValue().response));
+
+ BSONElement localTimeElem;
+ auto localTimeStatus =
+ bsonExtractTypedField(responseObj, kLocalTimeField, Date, &localTimeElem);
+
+ if (!localTimeStatus.isOK()) {
+ return {ErrorCodes::UnsupportedFormat, localTimeStatus.reason()};
+ }
+
+ auto electionIdStatus = extractElectionId(responseObj);
+
+ if (!electionIdStatus.isOK()) {
+ return electionIdStatus.getStatus();
+ }
+
+ return DistLockCatalog::ServerInfo(localTimeElem.date(), electionIdStatus.getValue());
+}
+
+StatusWith<LocksType> DistLockCatalogImpl::getLockByTS(OperationContext* opCtx,
+ const OID& lockSessionID) {
+ auto findResult =
+ _findOnConfig(opCtx, kReadPref, _locksNS, BSON(LocksType::lockID(lockSessionID)), {}, 1);
+
+ if (!findResult.isOK()) {
+ return findResult.getStatus();
+ }
+
+ const auto& findResultSet = findResult.getValue();
+
+ if (findResultSet.empty()) {
+ return {ErrorCodes::LockNotFound,
+ str::stream() << "lock with ts " << lockSessionID << " not found"};
+ }
+
+ BSONObj doc = findResultSet.front();
+ auto locksTypeResult = LocksType::fromBSON(doc);
+ if (!locksTypeResult.isOK()) {
+ return {ErrorCodes::FailedToParse,
+ str::stream() << "failed to parse: " << doc << " : "
+ << locksTypeResult.getStatus().toString()};
+ }
+
+ return locksTypeResult.getValue();
+}
+
+StatusWith<LocksType> DistLockCatalogImpl::getLockByName(OperationContext* opCtx, StringData name) {
+ auto findResult =
+ _findOnConfig(opCtx, kReadPref, _locksNS, BSON(LocksType::name() << name), {}, 1);
+
+ if (!findResult.isOK()) {
+ return findResult.getStatus();
+ }
+
+ const auto& findResultSet = findResult.getValue();
+
+ if (findResultSet.empty()) {
+ return {ErrorCodes::LockNotFound,
+ str::stream() << "lock with name " << name << " not found"};
+ }
+
+ BSONObj doc = findResultSet.front();
+ auto locksTypeResult = LocksType::fromBSON(doc);
+ if (!locksTypeResult.isOK()) {
+ return {ErrorCodes::FailedToParse,
+ str::stream() << "failed to parse: " << doc << " : "
+ << locksTypeResult.getStatus().toString()};
+ }
+
+ return locksTypeResult.getValue();
+}
+
+Status DistLockCatalogImpl::stopPing(OperationContext* opCtx, StringData processId) {
+ auto request =
+ makeFindAndModifyRequest(_lockPingNS, BSON(LockpingsType::process() << processId), {});
+ request.setWriteConcern(kMajorityWriteConcern.toBSON());
+
+ auto const shardRegistry = Grid::get(opCtx)->shardRegistry();
+ auto resultStatus = shardRegistry->getConfigShard()->runCommandWithFixedRetryAttempts(
+ opCtx,
+ ReadPreferenceSetting{ReadPreference::PrimaryOnly},
+ _locksNS.db().toString(),
+ request.toBSON({}),
+ Shard::kDefaultConfigCommandTimeout,
+ Shard::RetryPolicy::kNotIdempotent);
+
+ auto findAndModifyStatus = extractFindAndModifyNewObj(std::move(resultStatus));
+ return findAndModifyStatus.getStatus();
+}
+
+StatusWith<vector<BSONObj>> DistLockCatalogImpl::_findOnConfig(
+ OperationContext* opCtx,
+ const ReadPreferenceSetting& readPref,
+ const NamespaceString& nss,
+ const BSONObj& query,
+ const BSONObj& sort,
+ boost::optional<long long> limit) {
+ auto const shardRegistry = Grid::get(opCtx)->shardRegistry();
+ auto result = shardRegistry->getConfigShard()->exhaustiveFindOnConfig(
+ opCtx, readPref, repl::ReadConcernLevel::kMajorityReadConcern, nss, query, sort, limit);
+ if (!result.isOK()) {
+ return result.getStatus();
+ }
+
+ return result.getValue().docs;
+}
+
+} // namespace mongo
diff --git a/src/mongo/db/s/dist_lock_catalog_replset.h b/src/mongo/db/s/dist_lock_catalog_replset.h
new file mode 100644
index 00000000000..9421d290af3
--- /dev/null
+++ b/src/mongo/db/s/dist_lock_catalog_replset.h
@@ -0,0 +1,103 @@
+/**
+ * Copyright (C) 2018-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#pragma once
+
+#include <boost/optional.hpp>
+#include <vector>
+
+#include "mongo/bson/oid.h"
+#include "mongo/db/jsobj.h"
+#include "mongo/db/namespace_string.h"
+#include "mongo/db/ops/find_and_modify_command_gen.h"
+#include "mongo/db/s/dist_lock_catalog.h"
+#include "mongo/db/write_concern_options.h"
+#include "mongo/util/time_support.h"
+
+namespace mongo {
+
+struct ReadPreferenceSetting;
+
+class DistLockCatalogImpl final : public DistLockCatalog {
+public:
+ DistLockCatalogImpl();
+ ~DistLockCatalogImpl();
+
+ StatusWith<LockpingsType> getPing(OperationContext* opCtx, StringData processID) override;
+
+ Status ping(OperationContext* opCtx, StringData processID, Date_t ping) override;
+
+ StatusWith<LocksType> grabLock(OperationContext* opCtx,
+ StringData lockID,
+ const OID& lockSessionID,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why,
+ const WriteConcernOptions& writeConcern) override;
+
+ StatusWith<LocksType> overtakeLock(OperationContext* opCtx,
+ StringData lockID,
+ const OID& lockSessionID,
+ const OID& currentHolderTS,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) override;
+
+ Status unlock(OperationContext* opCtx, const OID& lockSessionID) override;
+
+ Status unlock(OperationContext* opCtx, const OID& lockSessionID, StringData name) override;
+
+ Status unlockAll(OperationContext* opCtx, const std::string& processID) override;
+
+ StatusWith<ServerInfo> getServerInfo(OperationContext* opCtx) override;
+
+ StatusWith<LocksType> getLockByTS(OperationContext* opCtx, const OID& lockSessionID) override;
+
+ StatusWith<LocksType> getLockByName(OperationContext* opCtx, StringData name) override;
+
+ Status stopPing(OperationContext* opCtx, StringData processId) override;
+
+private:
+ Status _unlock(OperationContext* opCtx, const write_ops::FindAndModifyCommand& request);
+
+ StatusWith<std::vector<BSONObj>> _findOnConfig(OperationContext* opCtx,
+ const ReadPreferenceSetting& readPref,
+ const NamespaceString& nss,
+ const BSONObj& query,
+ const BSONObj& sort,
+ boost::optional<long long> limit);
+
+ // These are not static to avoid initialization order fiasco.
+ const NamespaceString _lockPingNS;
+ const NamespaceString _locksNS;
+};
+
+} // namespace mongo
diff --git a/src/mongo/db/s/dist_lock_catalog_replset_test.cpp b/src/mongo/db/s/dist_lock_catalog_replset_test.cpp
new file mode 100644
index 00000000000..c440d207085
--- /dev/null
+++ b/src/mongo/db/s/dist_lock_catalog_replset_test.cpp
@@ -0,0 +1,1807 @@
+/**
+ * Copyright (C) 2018-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include <memory>
+#include <utility>
+
+#include "mongo/bson/json.h"
+#include "mongo/client/remote_command_targeter_mock.h"
+#include "mongo/db/commands.h"
+#include "mongo/db/repl/read_concern_args.h"
+#include "mongo/db/s/dist_lock_catalog_replset.h"
+#include "mongo/db/s/dist_lock_manager_mock.h"
+#include "mongo/db/s/shard_server_test_fixture.h"
+#include "mongo/db/storage/duplicate_key_error_info.h"
+#include "mongo/executor/network_test_env.h"
+#include "mongo/s/catalog/sharding_catalog_client_mock.h"
+#include "mongo/s/catalog/type_lockpings.h"
+#include "mongo/s/catalog/type_locks.h"
+#include "mongo/s/client/shard_factory.h"
+#include "mongo/s/client/shard_registry.h"
+#include "mongo/s/grid.h"
+#include "mongo/s/write_ops/batched_command_request.h"
+#include "mongo/util/time_support.h"
+
+namespace mongo {
+namespace {
+
+using executor::NetworkInterfaceMock;
+using executor::NetworkTestEnv;
+using executor::RemoteCommandRequest;
+using executor::RemoteCommandResponse;
+using repl::ReadConcernArgs;
+
+const HostAndPort dummyHost("dummy", 123);
+
+/**
+ * Sets up the mocked out objects for testing the replica-set backed catalog manager
+ *
+ * NOTE: Even though the dist lock manager only runs on the config server, this test is using the
+ * ShardServerTestFixture and emulating the network due to legacy reasons.
+ */
+class DistLockCatalogReplSetTest : public ShardServerTestFixture {
+protected:
+ std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient() override {
+ return std::make_unique<ShardingCatalogClientMock>();
+ }
+
+ std::shared_ptr<RemoteCommandTargeterMock> configTargeter() {
+ return RemoteCommandTargeterMock::get(shardRegistry()->getConfigShard()->getTargeter());
+ }
+
+ auto launchOnSeparateThread(std::function<void(OperationContext*)> func) {
+ auto const serviceContext = getServiceContext();
+ return launchAsync([serviceContext, func] {
+ ThreadClient tc("Test", getGlobalServiceContext());
+ auto opCtx = Client::getCurrent()->makeOperationContext();
+ func(opCtx.get());
+ });
+ }
+
+ DistLockCatalogImpl _distLockCatalog;
+};
+
+void checkReadConcern(const BSONObj& findCmd) {
+ ReadConcernArgs readConcernArgs;
+ ASSERT_OK(readConcernArgs.initialize(findCmd[ReadConcernArgs::kReadConcernFieldName]));
+ ASSERT(repl::ReadConcernLevel::kMajorityReadConcern == readConcernArgs.getLevel());
+}
+
+TEST_F(DistLockCatalogReplSetTest, BasicPing) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ Date_t ping(dateFromISOString("2014-03-11T09:17:18.098Z").getValue());
+ auto status = _distLockCatalog.ping(opCtx, "abcd", ping);
+ ASSERT_OK(status);
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ ASSERT_EQUALS(dummyHost, request.target);
+ ASSERT_EQUALS("config", request.dbname);
+
+ BSONObj expectedCmd(fromjson(R"({
+ findAndModify: "lockpings",
+ query: { _id: "abcd" },
+ update: {
+ $set: {
+ ping: { $date: "2014-03-11T09:17:18.098Z" }
+ }
+ },
+ upsert: true,
+ writeConcern: { w: "majority", wtimeout: 15000 },
+ maxTimeMS: 30000
+ })"));
+
+ ASSERT_BSONOBJ_EQ(expectedCmd, request.cmdObj);
+
+ return fromjson(R"({
+ ok: 1,
+ value: {
+ _id: "abcd",
+ ping: { $date: "2014-03-11T09:17:18.098Z" }
+ }
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, PingTargetError) {
+ configTargeter()->setFindHostReturnValue({ErrorCodes::InternalError, "can't target"});
+ auto status = _distLockCatalog.ping(operationContext(), "abcd", Date_t::now());
+ ASSERT_NOT_OK(status);
+}
+
+TEST_F(DistLockCatalogReplSetTest, PingRunCmdError) {
+ shutdownExecutorPool();
+
+ auto status = _distLockCatalog.ping(operationContext(), "abcd", Date_t::now());
+ ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, status.code());
+ ASSERT_FALSE(status.reason().empty());
+}
+
+TEST_F(DistLockCatalogReplSetTest, PingCommandError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.ping(opCtx, "abcd", Date_t::now());
+ ASSERT_EQUALS(ErrorCodes::FailedToParse, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ ok: 0,
+ errmsg: "bad",
+ code: 9
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, PingWriteError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.ping(opCtx, "abcd", Date_t::now());
+ ASSERT_EQUALS(ErrorCodes::Unauthorized, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ ok: 0,
+ code: 13,
+ errmsg: "not authorized"
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, PingWriteConcernError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.ping(opCtx, "abcd", Date_t::now());
+ ASSERT_EQUALS(ErrorCodes::WriteConcernFailed, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ ok: 1,
+ value: null,
+ writeConcernError: {
+ code: 64,
+ errmsg: "waiting for replication timed out"
+ }
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, PingUnsupportedWriteConcernResponse) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.ping(opCtx, "abcd", Date_t::now());
+ ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ // return non numeric code for writeConcernError.code
+ return fromjson(R"({
+ ok: 1,
+ value: null,
+ writeConcernError: {
+ code: "bad format",
+ errmsg: "waiting for replication timed out"
+ }
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, PingUnsupportedResponseFormat) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.ping(opCtx, "abcd", Date_t::now());
+ ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, status.code());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return BSON("ok" << 1 << "value"
+ << "NaN");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GrabLockNoOp) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ OID myID("555f80be366c194b13fb0372");
+ Date_t now(dateFromISOString("2015-05-22T19:17:18.098Z").getValue());
+ auto resultStatus = _distLockCatalog
+ .grabLock(opCtx,
+ "test",
+ myID,
+ "me",
+ "mongos",
+ now,
+ "because",
+ DistLockCatalog::kMajorityWriteConcern)
+ .getStatus();
+
+ ASSERT_EQUALS(ErrorCodes::LockStateChangeFailed, resultStatus.code());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ ASSERT_EQUALS(dummyHost, request.target);
+ ASSERT_EQUALS("config", request.dbname);
+
+ BSONObj expectedCmd(fromjson(R"({
+ findAndModify: "locks",
+ query: { _id: "test", state: 0 },
+ update: {
+ $set: {
+ ts: ObjectId("555f80be366c194b13fb0372"),
+ state: 2,
+ who: "me",
+ process: "mongos",
+ when: { $date: "2015-05-22T19:17:18.098Z" },
+ why: "because"
+ }
+ },
+ upsert: true,
+ new: true,
+ writeConcern: { w: "majority", wtimeout: 15000 },
+ maxTimeMS: 30000
+ })"));
+
+ ASSERT_BSONOBJ_EQ(expectedCmd, request.cmdObj);
+
+ return fromjson("{ ok: 1, value: null }");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GrabLockWithNewDoc) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ OID myID("555f80be366c194b13fb0372");
+ Date_t now(dateFromISOString("2015-05-22T19:17:18.098Z").getValue());
+ auto resultStatus = _distLockCatalog.grabLock(opCtx,
+ "test",
+ myID,
+ "me",
+ "mongos",
+ now,
+ "because",
+ DistLockCatalog::kMajorityWriteConcern);
+ ASSERT_OK(resultStatus.getStatus());
+
+ const auto& lockDoc = resultStatus.getValue();
+ ASSERT_OK(lockDoc.validate());
+ ASSERT_EQUALS("test", lockDoc.getName());
+ ASSERT_EQUALS(myID, lockDoc.getLockID());
+ ASSERT_EQUALS("me", lockDoc.getWho());
+ ASSERT_EQUALS("mongos", lockDoc.getProcess());
+ ASSERT_EQUALS("because", lockDoc.getWhy());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ ASSERT_EQUALS(dummyHost, request.target);
+ ASSERT_EQUALS("config", request.dbname);
+
+ BSONObj expectedCmd(fromjson(R"({
+ findAndModify: "locks",
+ query: { _id: "test", state: 0 },
+ update: {
+ $set: {
+ ts: ObjectId("555f80be366c194b13fb0372"),
+ state: 2,
+ who: "me",
+ process: "mongos",
+ when: { $date: "2015-05-22T19:17:18.098Z" },
+ why: "because"
+ }
+ },
+ upsert: true,
+ new: true,
+ writeConcern: { w: "majority", wtimeout: 15000 },
+ maxTimeMS: 30000
+ })"));
+
+ ASSERT_BSONOBJ_EQ(expectedCmd, request.cmdObj);
+
+ return fromjson(R"({
+ lastErrorObject: {
+ updatedExisting: false,
+ n: 1,
+ upserted: 1
+ },
+ value: {
+ _id: "test",
+ ts: ObjectId("555f80be366c194b13fb0372"),
+ state: 2,
+ who: "me",
+ process: "mongos",
+ when: { $date: "2015-05-22T19:17:18.098Z" },
+ why: "because"
+ },
+ ok: 1
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GrabLockWithBadLockDoc) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ Date_t now(dateFromISOString("2015-05-22T19:17:18.098Z").getValue());
+ auto resultStatus =
+ _distLockCatalog
+ .grabLock(
+ opCtx, "test", OID(), "", "", now, "", DistLockCatalog::kMajorityWriteConcern)
+ .getStatus();
+ ASSERT_EQUALS(ErrorCodes::FailedToParse, resultStatus.code());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ // Return an invalid lock document on value. This is theoretically impossible because
+ // the vital parts of the resulting doc are derived from the update request.
+ return fromjson(R"({
+ lastErrorObject: {
+ updatedExisting: false,
+ n: 1,
+ upserted: 1
+ },
+ value: {
+ _id: "test",
+ ts: ObjectId("555f80be366c194b13fb0372"),
+ state: "x",
+ who: "me",
+ process: "mongos",
+ when: { $date: "2015-05-22T19:17:18.098Z" },
+ why: "because"
+ },
+ ok: 1
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GrabLockTargetError) {
+ configTargeter()->setFindHostReturnValue({ErrorCodes::InternalError, "can't target"});
+
+ auto status = _distLockCatalog
+ .grabLock(operationContext(),
+ "",
+ OID::gen(),
+ "",
+ "",
+ Date_t::now(),
+ "",
+ DistLockCatalog::kMajorityWriteConcern)
+ .getStatus();
+ ASSERT_NOT_OK(status);
+}
+
+TEST_F(DistLockCatalogReplSetTest, GrabLockRunCmdError) {
+ shutdownExecutorPool();
+
+ auto status = _distLockCatalog
+ .grabLock(operationContext(),
+ "",
+ OID::gen(),
+ "",
+ "",
+ Date_t::now(),
+ "",
+ DistLockCatalog::kMajorityWriteConcern)
+ .getStatus();
+ ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, status.code());
+ ASSERT_FALSE(status.reason().empty());
+}
+
+TEST_F(DistLockCatalogReplSetTest, GrabLockCommandError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog
+ .grabLock(opCtx,
+ "",
+ OID::gen(),
+ "",
+ "",
+ Date_t::now(),
+ "",
+ DistLockCatalog::kMajorityWriteConcern)
+ .getStatus();
+ ASSERT_EQUALS(ErrorCodes::FailedToParse, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ ok: 0,
+ errmsg: "bad",
+ code: 9
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GrabLockDupKeyError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog
+ .grabLock(opCtx,
+ "",
+ OID::gen(),
+ "",
+ "",
+ Date_t::now(),
+ "",
+ DistLockCatalog::kMajorityWriteConcern)
+ .getStatus();
+ ASSERT_EQUALS(ErrorCodes::LockStateChangeFailed, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return Status(
+ {DuplicateKeyErrorInfo(BSON("x" << 1), BSON("" << 1)), "Mock duplicate key error"});
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GrabLockWriteError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog
+ .grabLock(opCtx,
+ "",
+ OID::gen(),
+ "",
+ "",
+ Date_t::now(),
+ "",
+ DistLockCatalog::kMajorityWriteConcern)
+ .getStatus();
+ ASSERT_EQUALS(ErrorCodes::Unauthorized, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ ok: 0,
+ code: 13,
+ errmsg: "not authorized"
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GrabLockWriteConcernError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog
+ .grabLock(operationContext(),
+ "",
+ OID::gen(),
+ "",
+ "",
+ Date_t::now(),
+ "",
+ DistLockCatalog::kMajorityWriteConcern)
+ .getStatus();
+ ASSERT_EQUALS(ErrorCodes::NotWritablePrimary, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ ok: 1,
+ value: null,
+ writeConcernError: {
+ code: 10107,
+ errmsg: "Not master while waiting for write concern"
+ }
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GrabLockWriteConcernErrorBadType) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog
+ .grabLock(operationContext(),
+ "",
+ OID::gen(),
+ "",
+ "",
+ Date_t::now(),
+ "",
+ DistLockCatalog::kMajorityWriteConcern)
+ .getStatus();
+ ASSERT_EQUALS(ErrorCodes::TypeMismatch, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ // return invalid non-object type for writeConcernError.
+ return fromjson(R"({
+ ok: 1,
+ value: null,
+ writeConcernError: "unexpected"
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GrabLockResponseMissingValueField) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog
+ .grabLock(operationContext(),
+ "",
+ OID::gen(),
+ "",
+ "",
+ Date_t::now(),
+ "",
+ DistLockCatalog::kMajorityWriteConcern)
+ .getStatus();
+ ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ ok: 1
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GrabLockUnsupportedWriteConcernResponse) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog
+ .grabLock(operationContext(),
+ "",
+ OID::gen(),
+ "",
+ "",
+ Date_t::now(),
+ "",
+ DistLockCatalog::kMajorityWriteConcern)
+ .getStatus();
+ ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ // return non numeric code for writeConcernError.code
+ return fromjson(R"({
+ ok: 1,
+ value: null,
+ writeConcernError: {
+ code: "bad format",
+ errmsg: "waiting for replication timed out"
+ }
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GrabLockUnsupportedResponseFormat) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog
+ .grabLock(operationContext(),
+ "",
+ OID::gen(),
+ "",
+ "",
+ Date_t::now(),
+ "",
+ DistLockCatalog::kMajorityWriteConcern)
+ .getStatus();
+ ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, status.code());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return BSON("ok" << 1 << "value"
+ << "NaN");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, OvertakeLockNoOp) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ OID myID("555f80be366c194b13fb0372");
+ OID currentOwner("555f99712c99a78c5b083358");
+ Date_t now(dateFromISOString("2015-05-22T19:17:18.098Z").getValue());
+ auto resultStatus =
+ _distLockCatalog
+ .overtakeLock(
+ operationContext(), "test", myID, currentOwner, "me", "mongos", now, "because")
+ .getStatus();
+
+ ASSERT_EQUALS(ErrorCodes::LockStateChangeFailed, resultStatus.code());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ ASSERT_EQUALS(dummyHost, request.target);
+ ASSERT_EQUALS("config", request.dbname);
+
+ BSONObj expectedCmd(fromjson(R"({
+ findAndModify: "locks",
+ query: {
+ $or: [
+ { _id: "test", state: 0 },
+ { _id: "test", ts: ObjectId("555f99712c99a78c5b083358") }
+ ]
+ },
+ update: {
+ $set: {
+ ts: ObjectId("555f80be366c194b13fb0372"),
+ state: 2,
+ who: "me",
+ process: "mongos",
+ when: { $date: "2015-05-22T19:17:18.098Z" },
+ why: "because"
+ }
+ },
+ new: true,
+ writeConcern: { w: "majority", wtimeout: 15000 },
+ maxTimeMS: 30000
+ })"));
+
+ ASSERT_BSONOBJ_EQ(expectedCmd, request.cmdObj);
+
+ return fromjson("{ ok: 1, value: null }");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, OvertakeLockWithNewDoc) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ OID myID("555f80be366c194b13fb0372");
+ OID currentOwner("555f99712c99a78c5b083358");
+ Date_t now(dateFromISOString("2015-05-22T19:17:18.098Z").getValue());
+ auto resultStatus = _distLockCatalog.overtakeLock(
+ operationContext(), "test", myID, currentOwner, "me", "mongos", now, "because");
+ ASSERT_OK(resultStatus.getStatus());
+
+ const auto& lockDoc = resultStatus.getValue();
+ ASSERT_OK(lockDoc.validate());
+ ASSERT_EQUALS("test", lockDoc.getName());
+ ASSERT_EQUALS(myID, lockDoc.getLockID());
+ ASSERT_EQUALS("me", lockDoc.getWho());
+ ASSERT_EQUALS("mongos", lockDoc.getProcess());
+ ASSERT_EQUALS("because", lockDoc.getWhy());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ ASSERT_EQUALS(dummyHost, request.target);
+ ASSERT_EQUALS("config", request.dbname);
+
+ BSONObj expectedCmd(fromjson(R"({
+ findAndModify: "locks",
+ query: {
+ $or: [
+ { _id: "test", state: 0 },
+ { _id: "test", ts: ObjectId("555f99712c99a78c5b083358") }
+ ]
+ },
+ update: {
+ $set: {
+ ts: ObjectId("555f80be366c194b13fb0372"),
+ state: 2,
+ who: "me",
+ process: "mongos",
+ when: { $date: "2015-05-22T19:17:18.098Z" },
+ why: "because"
+ }
+ },
+ new: true,
+ writeConcern: { w: "majority", wtimeout: 15000 },
+ maxTimeMS: 30000
+ })"));
+
+ ASSERT_BSONOBJ_EQ(expectedCmd, request.cmdObj);
+
+ return fromjson(R"({
+ lastErrorObject: {
+ updatedExisting: false,
+ n: 1,
+ upserted: 1
+ },
+ value: {
+ _id: "test",
+ ts: ObjectId("555f80be366c194b13fb0372"),
+ state: 2,
+ who: "me",
+ process: "mongos",
+ when: { $date: "2015-05-22T19:17:18.098Z" },
+ why: "because"
+ },
+ ok: 1
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, OvertakeLockWithBadLockDoc) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ Date_t now(dateFromISOString("2015-05-22T19:17:18.098Z").getValue());
+ auto resultStatus =
+ _distLockCatalog.overtakeLock(operationContext(), "test", OID(), OID(), "", "", now, "")
+ .getStatus();
+ ASSERT_EQUALS(ErrorCodes::FailedToParse, resultStatus.code());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ // Return an invalid lock document on value. This is theoretically impossible because
+ // the vital parts of the resulting doc are derived from the update request.
+ return fromjson(R"({
+ lastErrorObject: {
+ updatedExisting: false,
+ n: 1,
+ upserted: 1
+ },
+ value: {
+ _id: "test",
+ ts: ObjectId("555f80be366c194b13fb0372"),
+ state: "x",
+ who: "me",
+ process: "mongos",
+ when: { $date: "2015-05-22T19:17:18.098Z" },
+ why: "because"
+ },
+ ok: 1
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, OvertakeLockTargetError) {
+ configTargeter()->setFindHostReturnValue({ErrorCodes::InternalError, "can't target"});
+ auto status = _distLockCatalog
+ .overtakeLock(operationContext(), "", OID(), OID(), "", "", Date_t::now(), "")
+ .getStatus();
+ ASSERT_NOT_OK(status);
+}
+
+TEST_F(DistLockCatalogReplSetTest, OvertakeLockRunCmdError) {
+ shutdownExecutorPool();
+
+ auto status = _distLockCatalog
+ .overtakeLock(operationContext(), "", OID(), OID(), "", "", Date_t::now(), "")
+ .getStatus();
+ ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, status.code());
+ ASSERT_FALSE(status.reason().empty());
+}
+
+TEST_F(DistLockCatalogReplSetTest, OvertakeLockCommandError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status =
+ _distLockCatalog
+ .overtakeLock(operationContext(), "", OID(), OID(), "", "", Date_t::now(), "")
+ .getStatus();
+ ASSERT_EQUALS(ErrorCodes::FailedToParse, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ ok: 0,
+ errmsg: "bad",
+ code: 9
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, OvertakeLockWriteError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status =
+ _distLockCatalog
+ .overtakeLock(operationContext(), "", OID(), OID(), "", "", Date_t::now(), "")
+ .getStatus();
+ ASSERT_EQUALS(ErrorCodes::Unauthorized, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ ok: 0,
+ code: 13,
+ errmsg: "not authorized"
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, OvertakeLockWriteConcernError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status =
+ _distLockCatalog
+ .overtakeLock(operationContext(), "", OID(), OID(), "", "", Date_t::now(), "")
+ .getStatus();
+ ASSERT_EQUALS(ErrorCodes::WriteConcernFailed, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ ok: 1,
+ value: null,
+ writeConcernError: {
+ code: 64,
+ errmsg: "waiting for replication timed out"
+ }
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, OvertakeLockUnsupportedWriteConcernResponse) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status =
+ _distLockCatalog
+ .overtakeLock(operationContext(), "", OID(), OID(), "", "", Date_t::now(), "")
+ .getStatus();
+ ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ // return non numeric code for writeConcernError.code
+ return fromjson(R"({
+ ok: 1,
+ value: null,
+ writeConcernError: {
+ code: "bad format",
+ errmsg: "waiting for replication timed out"
+ }
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, OvertakeLockUnsupportedResponseFormat) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status =
+ _distLockCatalog
+ .overtakeLock(operationContext(), "", OID(), OID(), "", "", Date_t::now(), "")
+ .getStatus();
+ ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, status.code());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return BSON("ok" << 1 << "value"
+ << "NaN");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, BasicUnlock) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.unlock(operationContext(), OID("555f99712c99a78c5b083358"));
+ ASSERT_OK(status);
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ ASSERT_EQUALS(dummyHost, request.target);
+ ASSERT_EQUALS("config", request.dbname);
+
+ BSONObj expectedCmd(fromjson(R"({
+ findAndModify: "locks",
+ query: { ts: ObjectId("555f99712c99a78c5b083358") },
+ update: { $set: { state: 0 }},
+ writeConcern: { w: "majority", wtimeout: 15000 },
+ maxTimeMS: 30000
+ })"));
+
+ ASSERT_BSONOBJ_EQ(expectedCmd, request.cmdObj);
+
+ return fromjson(R"({
+ ok: 1,
+ value: {
+ _id: "",
+ ts: ObjectId("555f99712c99a78c5b083358"),
+ state: 0
+ }
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, BasicUnlockWithName) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.unlock(
+ operationContext(), OID("555f99712c99a78c5b083358"), "TestDB.TestColl");
+ ASSERT_OK(status);
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ ASSERT_EQUALS(dummyHost, request.target);
+ ASSERT_EQUALS("config", request.dbname);
+
+ BSONObj expectedCmd(fromjson(R"({
+ findAndModify: "locks",
+ query: { ts: ObjectId("555f99712c99a78c5b083358"), _id: "TestDB.TestColl" },
+ update: { $set: { state: 0 }},
+ writeConcern: { w: "majority", wtimeout: 15000 },
+ maxTimeMS: 30000
+ })"));
+
+ ASSERT_BSONOBJ_EQ(expectedCmd, request.cmdObj);
+
+ return fromjson(R"({
+ ok: 1,
+ value: {
+ _id: "TestDB.TestColl",
+ ts: ObjectId("555f99712c99a78c5b083358"),
+ state: 0
+ }
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, UnlockWithNoNewDoc) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.unlock(operationContext(), OID("555f99712c99a78c5b083358"));
+ ASSERT_OK(status);
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ ASSERT_EQUALS(dummyHost, request.target);
+ ASSERT_EQUALS("config", request.dbname);
+
+ BSONObj expectedCmd(fromjson(R"({
+ findAndModify: "locks",
+ query: { ts: ObjectId("555f99712c99a78c5b083358") },
+ update: { $set: { state: 0 }},
+ writeConcern: { w: "majority", wtimeout: 15000 },
+ maxTimeMS: 30000
+ })"));
+
+ ASSERT_BSONOBJ_EQ(expectedCmd, request.cmdObj);
+
+ return fromjson(R"({
+ ok: 1,
+ value: null
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, UnlockWithNameWithNoNewDoc) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.unlock(
+ operationContext(), OID("555f99712c99a78c5b083358"), "TestDB.TestColl");
+ ASSERT_OK(status);
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ ASSERT_EQUALS(dummyHost, request.target);
+ ASSERT_EQUALS("config", request.dbname);
+
+ BSONObj expectedCmd(fromjson(R"({
+ findAndModify: "locks",
+ query: { ts: ObjectId("555f99712c99a78c5b083358"), _id: "TestDB.TestColl" },
+ update: { $set: { state: 0 }},
+ writeConcern: { w: "majority", wtimeout: 15000 },
+ maxTimeMS: 30000
+ })"));
+
+ ASSERT_BSONOBJ_EQ(expectedCmd, request.cmdObj);
+
+ return fromjson(R"({
+ ok: 1,
+ value: null
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, UnlockTargetError) {
+ configTargeter()->setFindHostReturnValue({ErrorCodes::InternalError, "can't target"});
+ auto status = _distLockCatalog.unlock(operationContext(), OID());
+ ASSERT_NOT_OK(status);
+}
+
+TEST_F(DistLockCatalogReplSetTest, UnlockRunCmdError) {
+ shutdownExecutorPool();
+
+ auto status = _distLockCatalog.unlock(operationContext(), OID());
+ ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, status.code());
+ ASSERT_FALSE(status.reason().empty());
+}
+
+TEST_F(DistLockCatalogReplSetTest, UnlockCommandError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.unlock(operationContext(), OID());
+ ASSERT_EQUALS(ErrorCodes::FailedToParse, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ ok: 0,
+ errmsg: "bad",
+ code: 9
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, UnlockWriteError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.unlock(operationContext(), OID());
+ ASSERT_EQUALS(ErrorCodes::Unauthorized, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ ok: 0,
+ code: 13,
+ errmsg: "not authorized"
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, UnlockWriteConcernError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.unlock(operationContext(), OID());
+ ASSERT_EQUALS(ErrorCodes::WriteConcernFailed, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ BSONObj writeConcernFailedResponse = fromjson(R"({
+ ok: 1,
+ value: null,
+ writeConcernError: {
+ code: 64,
+ errmsg: "waiting for replication timed out"
+ }
+ })");
+
+ // The dist lock catalog calls into the ShardRegistry, which will retry 3 times for
+ // WriteConcernFailed errors
+ onCommand([&](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return writeConcernFailedResponse;
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, UnlockUnsupportedWriteConcernResponse) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.unlock(operationContext(), OID());
+ ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ // return non numeric code for writeConcernError.code
+ return fromjson(R"({
+ ok: 1,
+ value: null,
+ writeConcernError: {
+ code: "bad format",
+ errmsg: "waiting for replication timed out"
+ }
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, UnlockUnsupportedResponseFormat) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.unlock(operationContext(), OID());
+ ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, status.code());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return BSON("ok" << 1 << "value"
+ << "NaN");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, BasicUnlockAll) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.unlockAll(operationContext(), "processID");
+ ASSERT_OK(status);
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ ASSERT_EQUALS(dummyHost, request.target);
+ ASSERT_EQUALS("config", request.dbname);
+
+ const auto opMsgRequest(OpMsgRequest::fromDBAndBody(request.dbname, request.cmdObj));
+ const auto commandRequest(BatchedCommandRequest::parseUpdate(opMsgRequest));
+
+ ASSERT_BSONOBJ_EQ(BSON("w" << 1 << "wtimeout" << 0), commandRequest.getWriteConcern());
+
+ const auto& updateOp = commandRequest.getUpdateRequest();
+ ASSERT_EQUALS(LocksType::ConfigNS, updateOp.getNamespace());
+
+ const auto& updates = updateOp.getUpdates();
+ ASSERT_EQUALS(1U, updates.size());
+
+ const auto& update = updates.front();
+ ASSERT(!update.getUpsert());
+ ASSERT(update.getMulti());
+ ASSERT_BSONOBJ_EQ(BSON(LocksType::process("processID")), update.getQ());
+ ASSERT_BSONOBJ_EQ(BSON("$set" << BSON(LocksType::state(LocksType::UNLOCKED))),
+ update.getU().getUpdateClassic());
+
+ return BSON("ok" << 1);
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, UnlockAllWriteFailed) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.unlockAll(operationContext(), "processID");
+ ASSERT_EQUALS(ErrorCodes::IllegalOperation, status);
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return BSON("ok" << 0 << "code" << ErrorCodes::IllegalOperation << "errmsg"
+ << "something went wrong");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, UnlockAllNetworkError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.unlockAll(operationContext(), "processID");
+ ASSERT_EQUALS(ErrorCodes::NetworkTimeout, status);
+ });
+
+ for (int i = 0; i < 3; i++) { // ShardRegistry will retry 3 times on network errors
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return Status(ErrorCodes::NetworkTimeout, "network error");
+ });
+ }
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, BasicGetServerInfo) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ Date_t localTime(dateFromISOString("2015-05-26T13:06:27.293Z").getValue());
+ OID electionID("555fa85d4d8640862a0fc79b");
+ auto resultStatus = _distLockCatalog.getServerInfo(operationContext());
+ ASSERT_OK(resultStatus.getStatus());
+
+ const auto& serverInfo = resultStatus.getValue();
+ ASSERT_EQUALS(electionID, serverInfo.electionId);
+ ASSERT_EQUALS(localTime, serverInfo.serverTime);
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ ASSERT_EQUALS(dummyHost, request.target);
+ ASSERT_EQUALS("admin", request.dbname);
+ ASSERT_BSONOBJ_EQ(BSON("serverStatus" << 1 << "maxTimeMS" << 30000), request.cmdObj);
+
+ return fromjson(R"({
+ localTime: { $date: "2015-05-26T13:06:27.293Z" },
+ repl: {
+ electionId: ObjectId("555fa85d4d8640862a0fc79b")
+ },
+ ok: 1
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetServerTargetError) {
+ configTargeter()->setFindHostReturnValue({ErrorCodes::InternalError, "can't target"});
+ auto status = _distLockCatalog.getServerInfo(operationContext()).getStatus();
+ ASSERT_NOT_OK(status);
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetServerRunCmdError) {
+ shutdownExecutorPool();
+
+ auto status = _distLockCatalog.getServerInfo(operationContext()).getStatus();
+ ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, status.code());
+ ASSERT_FALSE(status.reason().empty());
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetServerCommandError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.getServerInfo(operationContext()).getStatus();
+ ASSERT_EQUALS(ErrorCodes::FailedToParse, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ ok: 0,
+ errmsg: "bad",
+ code: 9
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetServerBadElectionId) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.getServerInfo(operationContext()).getStatus();
+ ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ // return invalid non-oid electionId
+ return fromjson(R"({
+ localTime: { $date: "2015-05-26T13:06:27.293Z" },
+ repl: {
+ electionId: 34
+ },
+ ok: 1
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetServerBadLocalTime) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.getServerInfo(operationContext()).getStatus();
+ ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ // return invalid non date type for localTime field.
+ return fromjson(R"({
+ localTime: "2015-05-26T13:06:27.293Z",
+ repl: {
+ electionId: ObjectId("555fa85d4d8640862a0fc79b")
+ },
+ ok: 1
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetServerNoGLEStats) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.getServerInfo(operationContext()).getStatus();
+ ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ localTime: { $date: "2015-05-26T13:06:27.293Z" },
+ ok: 1
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetServerNoElectionId) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.getServerInfo(operationContext()).getStatus();
+ ASSERT_EQUALS(ErrorCodes::NotWritablePrimary, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ localTime: { $date: "2015-05-26T13:06:27.293Z" },
+ repl: {
+ ismaster: false,
+ me: "me:1234"
+ },
+ ok: 1
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetServerInvalidReplSubsectionShouldFail) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.getServerInfo(operationContext()).getStatus();
+ ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ localTime: { $date: "2015-05-26T13:06:27.293Z" },
+ repl: {
+ invalid: true
+ },
+ ok: 1
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetServerNoElectionIdButMasterShouldFail) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.getServerInfo(operationContext()).getStatus();
+ ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, status.code());
+ ASSERT_NOT_EQUALS(std::string::npos, status.reason().find("me:1234"));
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ localTime: { $date: "2015-05-26T13:06:27.293Z" },
+ repl: {
+ ismaster: true,
+ me: "me:1234"
+ },
+ ok: 1
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, BasicStopPing) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.stopPing(operationContext(), "test");
+ ASSERT_OK(status);
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ ASSERT_EQUALS(dummyHost, request.target);
+ ASSERT_EQUALS("config", request.dbname);
+
+ BSONObj expectedCmd(fromjson(R"({
+ findAndModify: "lockpings",
+ query: { _id: "test" },
+ remove: true,
+ writeConcern: { w: "majority", wtimeout: 15000 },
+ maxTimeMS: 30000
+ })"));
+
+ ASSERT_BSONOBJ_EQ(expectedCmd, request.cmdObj);
+
+ return fromjson(R"({
+ ok: 1,
+ value: {
+ _id: "test",
+ ping: { $date: "2014-03-11T09:17:18.098Z" }
+ }
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, StopPingTargetError) {
+ configTargeter()->setFindHostReturnValue({ErrorCodes::InternalError, "can't target"});
+ auto status = _distLockCatalog.stopPing(operationContext(), "");
+ ASSERT_NOT_OK(status);
+}
+
+TEST_F(DistLockCatalogReplSetTest, StopPingRunCmdError) {
+ shutdownExecutorPool();
+
+ auto status = _distLockCatalog.stopPing(operationContext(), "");
+ ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, status.code());
+ ASSERT_FALSE(status.reason().empty());
+}
+
+TEST_F(DistLockCatalogReplSetTest, StopPingCommandError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.stopPing(operationContext(), "");
+ ASSERT_EQUALS(ErrorCodes::FailedToParse, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ ok: 0,
+ errmsg: "bad",
+ code: 9
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, StopPingWriteError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.stopPing(operationContext(), "");
+ ASSERT_EQUALS(ErrorCodes::Unauthorized, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ ok: 0,
+ code: 13,
+ errmsg: "Unauthorized"
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, StopPingWriteConcernError) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.stopPing(operationContext(), "");
+ ASSERT_EQUALS(ErrorCodes::WriteConcernFailed, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return fromjson(R"({
+ ok: 1,
+ value: null,
+ writeConcernError: {
+ code: 64,
+ errmsg: "waiting for replication timed out"
+ }
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, StopPingUnsupportedWriteConcernResponse) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.stopPing(operationContext(), "");
+ ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ // return non numeric code for writeConcernError.code
+ return fromjson(R"({
+ ok: 1,
+ value: null,
+ writeConcernError: {
+ code: "bad format",
+ errmsg: "waiting for replication timed out"
+ }
+ })");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, StopPingUnsupportedResponseFormat) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.stopPing(operationContext(), "");
+ ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, status.code());
+ });
+
+ onCommand([](const RemoteCommandRequest& request) -> StatusWith<BSONObj> {
+ return BSON("ok" << 1 << "value"
+ << "NaN");
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, BasicGetPing) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ Date_t ping(dateFromISOString("2015-05-26T13:06:27.293Z").getValue());
+ auto resultStatus = _distLockCatalog.getPing(operationContext(), "test");
+ ASSERT_OK(resultStatus.getStatus());
+
+ const auto& pingDoc = resultStatus.getValue();
+ ASSERT_EQUALS("test", pingDoc.getProcess());
+ ASSERT_EQUALS(ping, pingDoc.getPing());
+ });
+
+ onFindCommand([](const RemoteCommandRequest& request) {
+ ASSERT_EQUALS(dummyHost, request.target);
+ ASSERT_EQUALS("config", request.dbname);
+
+ const auto& findCmd = request.cmdObj;
+ ASSERT_EQUALS("lockpings", findCmd["find"].str());
+ ASSERT_BSONOBJ_EQ(BSON("_id"
+ << "test"),
+ findCmd["filter"].Obj());
+ ASSERT_EQUALS(1, findCmd["limit"].numberLong());
+ checkReadConcern(findCmd);
+
+ BSONObj pingDoc(fromjson(R"({
+ _id: "test",
+ ping: { $date: "2015-05-26T13:06:27.293Z" }
+ })"));
+
+ std::vector<BSONObj> result;
+ result.push_back(pingDoc);
+
+ return result;
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetPingTargetError) {
+ configTargeter()->setFindHostReturnValue({ErrorCodes::InternalError, "can't target"});
+ auto status = _distLockCatalog.getPing(operationContext(), "").getStatus();
+ ASSERT_EQUALS(ErrorCodes::InternalError, status.code());
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetPingRunCmdError) {
+ shutdownExecutorPool();
+
+ auto status = _distLockCatalog.getPing(operationContext(), "").getStatus();
+ ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, status.code());
+ ASSERT_FALSE(status.reason().empty());
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetPingNotFound) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.getPing(operationContext(), "").getStatus();
+ ASSERT_EQUALS(ErrorCodes::NoMatchingDocument, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onFindCommand([](const RemoteCommandRequest& request) -> StatusWith<std::vector<BSONObj>> {
+ return std::vector<BSONObj>();
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetPingUnsupportedFormat) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.getPing(operationContext(), "test").getStatus();
+ ASSERT_EQUALS(ErrorCodes::FailedToParse, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onFindCommand([](const RemoteCommandRequest& request) -> StatusWith<std::vector<BSONObj>> {
+ // return non-date type for ping.
+ BSONObj pingDoc(fromjson(R"({
+ _id: "test",
+ ping: "bad"
+ })"));
+
+ std::vector<BSONObj> result;
+ result.push_back(pingDoc);
+
+ return result;
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, BasicGetLockByTS) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ OID ts("555f99712c99a78c5b083358");
+ auto resultStatus = _distLockCatalog.getLockByTS(operationContext(), ts);
+ ASSERT_OK(resultStatus.getStatus());
+
+ const auto& lockDoc = resultStatus.getValue();
+ ASSERT_EQUALS("test", lockDoc.getName());
+ ASSERT_EQUALS(ts, lockDoc.getLockID());
+ });
+
+ onFindCommand([](const RemoteCommandRequest& request) -> StatusWith<std::vector<BSONObj>> {
+ ASSERT_EQUALS(dummyHost, request.target);
+ ASSERT_EQUALS("config", request.dbname);
+
+ const auto& findCmd = request.cmdObj;
+ ASSERT_EQUALS("locks", findCmd["find"].str());
+ ASSERT_BSONOBJ_EQ(BSON("ts" << OID("555f99712c99a78c5b083358")), findCmd["filter"].Obj());
+ ASSERT_EQUALS(1, findCmd["limit"].numberLong());
+ checkReadConcern(findCmd);
+
+ BSONObj lockDoc(fromjson(R"({
+ _id: "test",
+ state: 2,
+ ts: ObjectId("555f99712c99a78c5b083358")
+ })"));
+
+ std::vector<BSONObj> result;
+ result.push_back(lockDoc);
+ return result;
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetLockByTSTargetError) {
+ configTargeter()->setFindHostReturnValue({ErrorCodes::InternalError, "can't target"});
+ auto status = _distLockCatalog.getLockByTS(operationContext(), OID()).getStatus();
+ ASSERT_EQUALS(ErrorCodes::InternalError, status.code());
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetLockByTSRunCmdError) {
+ shutdownExecutorPool();
+ auto status = _distLockCatalog.getLockByTS(operationContext(), OID()).getStatus();
+ ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, status.code());
+ ASSERT_FALSE(status.reason().empty());
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetLockByTSNotFound) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.getLockByTS(operationContext(), OID()).getStatus();
+ ASSERT_EQUALS(ErrorCodes::LockNotFound, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onFindCommand([](const RemoteCommandRequest& request) -> StatusWith<std::vector<BSONObj>> {
+ return std::vector<BSONObj>();
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetLockByTSUnsupportedFormat) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.getLockByTS(operationContext(), OID()).getStatus();
+ ASSERT_EQUALS(ErrorCodes::FailedToParse, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onFindCommand([](const RemoteCommandRequest& request) -> StatusWith<std::vector<BSONObj>> {
+ // return invalid non-numeric type for state.
+ BSONObj lockDoc(fromjson(R"({
+ _id: "test",
+ state: "bad"
+ })"));
+
+ std::vector<BSONObj> result;
+ result.push_back(lockDoc);
+
+ return result;
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, BasicGetLockByName) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ OID ts("555f99712c99a78c5b083358");
+ auto resultStatus = _distLockCatalog.getLockByName(operationContext(), "abc");
+ ASSERT_OK(resultStatus.getStatus());
+
+ const auto& lockDoc = resultStatus.getValue();
+ ASSERT_EQUALS("abc", lockDoc.getName());
+ ASSERT_EQUALS(ts, lockDoc.getLockID());
+ });
+
+ onFindCommand([](const RemoteCommandRequest& request) {
+ ASSERT_EQUALS(dummyHost, request.target);
+ ASSERT_EQUALS("config", request.dbname);
+
+ const auto& findCmd = request.cmdObj;
+ ASSERT_EQUALS("locks", findCmd["find"].str());
+ ASSERT_BSONOBJ_EQ(BSON("_id"
+ << "abc"),
+ findCmd["filter"].Obj());
+ ASSERT_EQUALS(1, findCmd["limit"].numberLong());
+ checkReadConcern(findCmd);
+
+ BSONObj lockDoc(fromjson(R"({
+ _id: "abc",
+ state: 2,
+ ts: ObjectId("555f99712c99a78c5b083358")
+ })"));
+
+ std::vector<BSONObj> result;
+ result.push_back(lockDoc);
+ return result;
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetLockByNameTargetError) {
+ configTargeter()->setFindHostReturnValue({ErrorCodes::InternalError, "can't target"});
+ auto status = _distLockCatalog.getLockByName(operationContext(), "x").getStatus();
+ ASSERT_EQUALS(ErrorCodes::InternalError, status.code());
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetLockByNameRunCmdError) {
+ shutdownExecutorPool();
+
+ auto status = _distLockCatalog.getLockByName(operationContext(), "x").getStatus();
+ ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, status.code());
+ ASSERT_FALSE(status.reason().empty());
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetLockByNameNotFound) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.getLockByName(operationContext(), "x").getStatus();
+ ASSERT_EQUALS(ErrorCodes::LockNotFound, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onFindCommand([](const RemoteCommandRequest& request) -> StatusWith<std::vector<BSONObj>> {
+ return std::vector<BSONObj>();
+ });
+
+ future.default_timed_get();
+}
+
+TEST_F(DistLockCatalogReplSetTest, GetLockByNameUnsupportedFormat) {
+ auto future = launchOnSeparateThread([this](OperationContext* opCtx) {
+ auto status = _distLockCatalog.getLockByName(operationContext(), "x").getStatus();
+ ASSERT_EQUALS(ErrorCodes::FailedToParse, status.code());
+ ASSERT_FALSE(status.reason().empty());
+ });
+
+ onFindCommand([](const RemoteCommandRequest& request) -> StatusWith<std::vector<BSONObj>> {
+ // Return non-numeric type for state.
+ BSONObj lockDoc(fromjson(R"({
+ _id: "x",
+ state: "bad"
+ })"));
+
+ std::vector<BSONObj> result;
+ result.push_back(lockDoc);
+
+ return result;
+ });
+
+ future.default_timed_get();
+}
+
+} // namespace
+} // namespace mongo
diff --git a/src/mongo/db/s/dist_lock_manager.cpp b/src/mongo/db/s/dist_lock_manager.cpp
new file mode 100644
index 00000000000..00893d912e4
--- /dev/null
+++ b/src/mongo/db/s/dist_lock_manager.cpp
@@ -0,0 +1,107 @@
+/**
+ * Copyright (C) 2018-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/s/dist_lock_manager.h"
+
+#include "mongo/db/operation_context.h"
+
+namespace mongo {
+namespace {
+
+const auto getDistLockManager =
+ ServiceContext::declareDecoration<std::unique_ptr<DistLockManager>>();
+
+} // namespace
+
+const Seconds DistLockManager::kDefaultLockTimeout(20);
+const Milliseconds DistLockManager::kSingleLockAttemptTimeout(0);
+
+DistLockManager::ScopedDistLock::ScopedDistLock(OperationContext* opCtx,
+ DistLockHandle lockHandle,
+ DistLockManager* lockManager)
+ : _opCtx(opCtx), _lockID(std::move(lockHandle)), _lockManager(lockManager) {}
+
+DistLockManager::ScopedDistLock::~ScopedDistLock() {
+ if (_lockManager) {
+ _lockManager->unlock(_opCtx, _lockID);
+ }
+}
+
+DistLockManager::ScopedDistLock::ScopedDistLock(ScopedDistLock&& other)
+ : _opCtx(nullptr), _lockManager(nullptr) {
+ *this = std::move(other);
+}
+
+DistLockManager::ScopedDistLock& DistLockManager::ScopedDistLock::operator=(
+ ScopedDistLock&& other) {
+ if (this != &other) {
+ invariant(_lockManager == nullptr);
+ invariant(_opCtx == nullptr);
+
+ _opCtx = other._opCtx;
+ _lockID = std::move(other._lockID);
+ _lockManager = other._lockManager;
+ other._lockManager = nullptr;
+ }
+
+ return *this;
+}
+
+Status DistLockManager::ScopedDistLock::checkStatus() {
+ invariant(_lockManager);
+ return _lockManager->checkStatus(_opCtx, _lockID);
+}
+
+DistLockManager* DistLockManager::get(OperationContext* opCtx) {
+ return getDistLockManager(opCtx->getServiceContext()).get();
+}
+
+void DistLockManager::create(ServiceContext* service,
+ std::unique_ptr<DistLockManager> distLockManager) {
+ invariant(!getDistLockManager(service));
+ getDistLockManager(service) = std::move(distLockManager);
+}
+
+StatusWith<DistLockManager::ScopedDistLock> DistLockManager::lock(OperationContext* opCtx,
+ StringData name,
+ StringData whyMessage,
+ Milliseconds waitFor) {
+ auto distLockHandleStatus = lockWithSessionID(opCtx, name, whyMessage, OID::gen(), waitFor);
+ if (!distLockHandleStatus.isOK()) {
+ return distLockHandleStatus.getStatus();
+ }
+
+ return DistLockManager::ScopedDistLock(opCtx, std::move(distLockHandleStatus.getValue()), this);
+}
+
+} // namespace mongo
diff --git a/src/mongo/db/s/dist_lock_manager.h b/src/mongo/db/s/dist_lock_manager.h
new file mode 100644
index 00000000000..61da53109b1
--- /dev/null
+++ b/src/mongo/db/s/dist_lock_manager.h
@@ -0,0 +1,189 @@
+/**
+ * Copyright (C) 2018-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/base/string_data.h"
+#include "mongo/bson/oid.h"
+#include "mongo/db/service_context.h"
+#include "mongo/stdx/chrono.h"
+
+namespace mongo {
+
+using DistLockHandle = OID;
+
+/**
+ * Interface for handling distributed locks.
+ *
+ * Usage:
+ *
+ * auto scopedDistLock = mgr->lock(...);
+ *
+ * if (!scopedDistLock.isOK()) {
+ * // Did not get lock. scopedLockStatus destructor will not call unlock.
+ * }
+ *
+ * // To check if lock is still owned:
+ * auto status = scopedDistLock.getValue().checkStatus();
+ *
+ * if (!status.isOK()) {
+ * // Someone took over the lock! Unlock will still be called at destructor, but will
+ * // practically be a no-op since it doesn't own the lock anymore.
+ * }
+ */
+class DistLockManager {
+public:
+ // Default timeout which will be used if one is not passed to the lock method.
+ static const Seconds kDefaultLockTimeout;
+
+ // Timeout value, which specifies that if the lock is not available immediately, no attempt
+ // should be made to wait for it to become free.
+ static const Milliseconds kSingleLockAttemptTimeout;
+
+ /**
+ * RAII type for distributed lock. Not meant to be shared across multiple threads.
+ */
+ class ScopedDistLock {
+ ScopedDistLock(const ScopedDistLock&) = delete;
+ ScopedDistLock& operator=(const ScopedDistLock&) = delete;
+
+ public:
+ ScopedDistLock(OperationContext* opCtx,
+ DistLockHandle lockHandle,
+ DistLockManager* lockManager);
+ ~ScopedDistLock();
+
+ ScopedDistLock(ScopedDistLock&& other);
+ ScopedDistLock& operator=(ScopedDistLock&& other);
+
+ /**
+ * Checks whether the lock is still being held by querying the config server.
+ */
+ Status checkStatus();
+
+ private:
+ OperationContext* _opCtx;
+ DistLockHandle _lockID;
+ DistLockManager* _lockManager; // Not owned here.
+ };
+
+ virtual ~DistLockManager() = default;
+
+ /**
+ * Retrieves the DistLockManager singleton for the node.
+ */
+ static DistLockManager* get(OperationContext* opCtx);
+ static void create(ServiceContext* service, std::unique_ptr<DistLockManager> distLockManager);
+
+ /**
+ * Performs bootstrapping for the manager. Implementation do not need to guarantee
+ * thread safety so callers should employ proper synchronization when calling this method.
+ */
+ virtual void startUp() = 0;
+
+ /**
+ * Cleanup the manager's resources. Implementations do not need to guarantee thread safety
+ * so callers should employ proper synchronization when calling this method.
+ */
+ virtual void shutDown(OperationContext* opCtx) = 0;
+
+ /**
+ * Returns the process ID for this DistLockManager.
+ */
+ virtual std::string getProcessID() = 0;
+
+ /**
+ * Tries multiple times to lock, using the specified lock try interval, until
+ * a certain amount of time has passed or when any error that is not LockBusy
+ * occurred.
+ *
+ * waitFor = 0 indicates there should only be one attempt to acquire the lock, and
+ * no waiting.
+ * waitFor = -1 indicates we should retry indefinitely.
+ *
+ * Returns OK if the lock was successfully acquired.
+ * Returns ErrorCodes::DistributedClockSkewed when a clock skew is detected.
+ * Returns ErrorCodes::LockBusy if the lock is being held.
+ */
+ StatusWith<ScopedDistLock> lock(OperationContext* opCtx,
+ StringData name,
+ StringData whyMessage,
+ Milliseconds waitFor);
+
+ /**
+ * Same behavior as lock(...) above, except takes a specific lock session ID "lockSessionID"
+ * instead of randomly generating one internally.
+ *
+ * This is useful for a process running on the config primary after a failover. A lock can be
+ * immediately reacquired if "lockSessionID" matches that of the lock, rather than waiting for
+ * the inactive lock to expire.
+ */
+ virtual StatusWith<DistLockHandle> lockWithSessionID(OperationContext* opCtx,
+ StringData name,
+ StringData whyMessage,
+ const OID& lockSessionID,
+ Milliseconds waitFor) = 0;
+
+ /**
+ * Specialized locking method, which only succeeds if the specified lock name is not held by
+ * anyone. Uses local write concern and does not attempt to overtake the lock or check whether
+ * the lock lease has expired.
+ */
+ virtual StatusWith<DistLockHandle> tryLockWithLocalWriteConcern(OperationContext* opCtx,
+ StringData name,
+ StringData whyMessage,
+ const OID& lockSessionID) = 0;
+
+ /**
+ * Unlocks the given lockHandle. Will attempt to retry again later if the config
+ * server is not reachable.
+ */
+ virtual void unlock(OperationContext* opCtx, const DistLockHandle& lockHandle) = 0;
+
+ /**
+ * Unlocks the lock specified by "lockHandle" and "name". Will attempt to retry again later if
+ * the config server is not reachable.
+ */
+ virtual void unlock(OperationContext* opCtx,
+ const DistLockHandle& lockHandle,
+ StringData name) = 0;
+
+ /**
+ * Makes a best-effort attempt to unlock all locks owned by the given processID.
+ */
+ virtual void unlockAll(OperationContext* opCtx, const std::string& processID) = 0;
+
+private:
+ /**
+ * Checks if the lockHandle still exists in the config server.
+ */
+ virtual Status checkStatus(OperationContext* opCtx, const DistLockHandle& lockHandle) = 0;
+};
+
+} // namespace mongo
diff --git a/src/mongo/db/s/dist_lock_manager_mock.cpp b/src/mongo/db/s/dist_lock_manager_mock.cpp
new file mode 100644
index 00000000000..951bd4e9c97
--- /dev/null
+++ b/src/mongo/db/s/dist_lock_manager_mock.cpp
@@ -0,0 +1,139 @@
+/**
+ * Copyright (C) 2018-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kTest
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/s/dist_lock_manager_mock.h"
+
+#include <algorithm>
+
+#include "mongo/unittest/unittest.h"
+#include "mongo/util/str.h"
+#include "mongo/util/time_support.h"
+
+namespace mongo {
+namespace {
+
+void noLockFuncSet(StringData name, StringData whyMessage, Milliseconds waitFor) {
+ FAIL(str::stream() << "Lock not expected to be called. "
+ << "Name: " << name << ", whyMessage: " << whyMessage
+ << ", waitFor: " << waitFor);
+}
+
+} // namespace
+
+DistLockManagerMock::DistLockManagerMock()
+ : _lockReturnStatus{Status::OK()}, _lockChecker{noLockFuncSet} {}
+
+DistLockManagerMock::~DistLockManagerMock() = default;
+
+void DistLockManagerMock::startUp() {}
+
+void DistLockManagerMock::shutDown(OperationContext* opCtx) {
+ uassert(28659, "DistLockManagerMock shut down with outstanding locks present", _locks.empty());
+}
+
+std::string DistLockManagerMock::getProcessID() {
+ return "Mock dist lock manager process id";
+}
+
+StatusWith<DistLockHandle> DistLockManagerMock::lockWithSessionID(OperationContext* opCtx,
+ StringData name,
+ StringData whyMessage,
+ const OID& lockSessionID,
+ Milliseconds waitFor) {
+ _lockChecker(name, whyMessage, waitFor);
+ _lockChecker = noLockFuncSet;
+
+ if (!_lockReturnStatus.isOK()) {
+ return _lockReturnStatus;
+ }
+
+ if (_locks.end() != std::find_if(_locks.begin(), _locks.end(), [name](LockInfo info) -> bool {
+ return info.name == name;
+ })) {
+ return Status(ErrorCodes::LockBusy,
+ str::stream() << "Lock \"" << name << "\" is already taken");
+ }
+
+ LockInfo info;
+ info.name = name.toString();
+ info.lockID = lockSessionID;
+ _locks.push_back(info);
+
+ return info.lockID;
+}
+
+StatusWith<DistLockHandle> DistLockManagerMock::tryLockWithLocalWriteConcern(
+ OperationContext* opCtx, StringData name, StringData whyMessage, const OID& lockSessionID) {
+ // Not yet implemented
+ MONGO_UNREACHABLE;
+}
+
+void DistLockManagerMock::unlockAll(OperationContext* opCtx, const std::string& processID) {
+ // Not yet implemented
+ MONGO_UNREACHABLE;
+}
+
+void DistLockManagerMock::unlock(OperationContext* opCtx, const DistLockHandle& lockHandle) {
+ std::vector<LockInfo>::iterator it =
+ std::find_if(_locks.begin(), _locks.end(), [&lockHandle](LockInfo info) -> bool {
+ return info.lockID == lockHandle;
+ });
+ if (it == _locks.end()) {
+ return;
+ }
+ _locks.erase(it);
+}
+
+void DistLockManagerMock::unlock(OperationContext* opCtx,
+ const DistLockHandle& lockHandle,
+ StringData name) {
+ std::vector<LockInfo>::iterator it =
+ std::find_if(_locks.begin(), _locks.end(), [&lockHandle, &name](LockInfo info) -> bool {
+ return ((info.lockID == lockHandle) && (info.name == name));
+ });
+ if (it == _locks.end()) {
+ return;
+ }
+ _locks.erase(it);
+}
+
+Status DistLockManagerMock::checkStatus(OperationContext* opCtx, const DistLockHandle& lockHandle) {
+ return Status::OK();
+}
+
+void DistLockManagerMock::expectLock(LockFunc checker, Status status) {
+ _lockReturnStatus = std::move(status);
+ _lockChecker = checker;
+}
+
+} // namespace mongo
diff --git a/src/mongo/db/s/dist_lock_manager_mock.h b/src/mongo/db/s/dist_lock_manager_mock.h
new file mode 100644
index 00000000000..824a7a3b0ca
--- /dev/null
+++ b/src/mongo/db/s/dist_lock_manager_mock.h
@@ -0,0 +1,86 @@
+/**
+ * Copyright (C) 2018-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#pragma once
+
+#include <functional>
+#include <vector>
+
+#include "mongo/db/s/dist_lock_manager.h"
+
+namespace mongo {
+
+class DistLockManagerMock : public DistLockManager {
+public:
+ DistLockManagerMock();
+ virtual ~DistLockManagerMock();
+
+ void startUp() override;
+ void shutDown(OperationContext* opCtx) override;
+
+ std::string getProcessID() override;
+
+ StatusWith<DistLockHandle> lockWithSessionID(OperationContext* opCtx,
+ StringData name,
+ StringData whyMessage,
+ const OID& lockSessionID,
+ Milliseconds waitFor) override;
+
+ StatusWith<DistLockHandle> tryLockWithLocalWriteConcern(OperationContext* opCtx,
+ StringData name,
+ StringData whyMessage,
+ const OID& lockSessionID) override;
+
+ void unlockAll(OperationContext* opCtx, const std::string& processID) override;
+
+ using LockFunc =
+ std::function<void(StringData name, StringData whyMessage, Milliseconds waitFor)>;
+
+ void expectLock(LockFunc checkerFunc, Status lockStatus);
+
+ void unlock(OperationContext* opCtx, const DistLockHandle& lockHandle) override;
+
+ void unlock(OperationContext* opCtx,
+ const DistLockHandle& lockHandle,
+ StringData name) override;
+
+private:
+ struct LockInfo {
+ DistLockHandle lockID;
+ std::string name;
+ };
+
+ Status checkStatus(OperationContext* opCtx, const DistLockHandle& lockHandle) override;
+
+ std::vector<LockInfo> _locks;
+ Status _lockReturnStatus;
+ LockFunc _lockChecker;
+};
+
+} // namespace mongo
diff --git a/src/mongo/db/s/dist_lock_manager_replset.cpp b/src/mongo/db/s/dist_lock_manager_replset.cpp
new file mode 100644
index 00000000000..a3c09d5fbdc
--- /dev/null
+++ b/src/mongo/db/s/dist_lock_manager_replset.cpp
@@ -0,0 +1,622 @@
+/**
+ * Copyright (C) 2018-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/s/dist_lock_manager_replset.h"
+
+#include "mongo/logv2/log.h"
+#include "mongo/s/catalog/type_lockpings.h"
+#include "mongo/s/catalog/type_locks.h"
+#include "mongo/s/client/shard_registry.h"
+#include "mongo/s/grid.h"
+#include "mongo/stdx/chrono.h"
+#include "mongo/util/concurrency/idle_thread_block.h"
+#include "mongo/util/concurrency/thread_name.h"
+#include "mongo/util/fail_point.h"
+#include "mongo/util/str.h"
+#include "mongo/util/time_support.h"
+#include "mongo/util/timer.h"
+
+namespace mongo {
+namespace {
+
+// How many times to retry acquiring the lock after the first attempt fails
+const int kMaxNumLockAcquireRetries = 2;
+
+// How frequently to poll the distributed lock when it is found to be locked
+const Milliseconds kLockRetryInterval(500);
+
+MONGO_FAIL_POINT_DEFINE(setDistLockTimeout);
+MONGO_FAIL_POINT_DEFINE(disableReplSetDistLockManager);
+
+} // namespace
+
+const Seconds ReplSetDistLockManager::kDistLockPingInterval{30};
+const Minutes ReplSetDistLockManager::kDistLockExpirationTime{15};
+
+ReplSetDistLockManager::ReplSetDistLockManager(ServiceContext* globalContext,
+ StringData processID,
+ std::unique_ptr<DistLockCatalog> catalog,
+ Milliseconds pingInterval,
+ Milliseconds lockExpiration)
+ : _serviceContext(globalContext),
+ _processID(processID.toString()),
+ _catalog(std::move(catalog)),
+ _pingInterval(pingInterval),
+ _lockExpiration(lockExpiration) {}
+
+ReplSetDistLockManager::~ReplSetDistLockManager() = default;
+
+void ReplSetDistLockManager::startUp() {
+ if (!_execThread) {
+ _execThread = std::make_unique<stdx::thread>(&ReplSetDistLockManager::doTask, this);
+ }
+}
+
+void ReplSetDistLockManager::shutDown(OperationContext* opCtx) {
+ {
+ stdx::lock_guard<Latch> lk(_mutex);
+ _isShutDown = true;
+ _shutDownCV.notify_all();
+ }
+
+ // Don't grab _mutex, otherwise will deadlock trying to join. Safe to read
+ // _execThread since it is modified only at statrUp().
+ if (_execThread && _execThread->joinable()) {
+ _execThread->join();
+ _execThread.reset();
+ }
+
+ auto status = _catalog->stopPing(opCtx, _processID);
+ if (!status.isOK()) {
+ LOGV2_WARNING(22667,
+ "Error cleaning up distributed ping entry for {processId} caused by {error}",
+ "Error cleaning up distributed ping entry",
+ "processId"_attr = _processID,
+ "error"_attr = redact(status));
+ }
+}
+
+std::string ReplSetDistLockManager::getProcessID() {
+ return _processID;
+}
+
+bool ReplSetDistLockManager::isShutDown() {
+ stdx::lock_guard<Latch> lk(_mutex);
+ return _isShutDown;
+}
+
+void ReplSetDistLockManager::doTask() {
+ LOGV2(22649,
+ "Creating distributed lock ping thread for process {processId} with ping interval "
+ "{pingInterval}",
+ "Creating distributed lock ping thread",
+ "processId"_attr = _processID,
+ "pingInterval"_attr = _pingInterval);
+
+ Timer elapsedSincelastPing(_serviceContext->getTickSource());
+ Client::initThread("replSetDistLockPinger");
+
+ while (!isShutDown()) {
+ if (MONGO_unlikely(disableReplSetDistLockManager.shouldFail())) {
+ LOGV2(426321,
+ "The distributed lock ping thread is disabled for testing",
+ "processId"_attr = _processID,
+ "pingInterval"_attr = _pingInterval);
+ return;
+ }
+ {
+ auto opCtx = cc().makeOperationContext();
+ auto pingStatus = _catalog->ping(opCtx.get(), _processID, Date_t::now());
+
+ if (!pingStatus.isOK() && pingStatus != ErrorCodes::NotWritablePrimary) {
+ LOGV2_WARNING(22668,
+ "Pinging failed for distributed lock pinger caused by {error}",
+ "Pinging failed for distributed lock pinger",
+ "error"_attr = pingStatus);
+ }
+
+ const Milliseconds elapsed(elapsedSincelastPing.millis());
+ if (elapsed > 10 * _pingInterval) {
+ LOGV2_WARNING(22669,
+ "Lock pinger for process {processId} was inactive for {duration}",
+ "Lock pinger was inactive for multiple intervals",
+ "processId"_attr = _processID,
+ "duration"_attr = elapsed);
+ }
+ elapsedSincelastPing.reset();
+
+ std::deque<std::pair<DistLockHandle, boost::optional<std::string>>> toUnlockBatch;
+ {
+ stdx::unique_lock<Latch> lk(_mutex);
+ toUnlockBatch.swap(_unlockList);
+ }
+
+ for (const auto& toUnlock : toUnlockBatch) {
+ Status unlockStatus(ErrorCodes::NotYetInitialized,
+ "status unlock not initialized!");
+ if (toUnlock.second) {
+ // A non-empty _id (name) field was provided, unlock by ts (sessionId) and _id.
+ unlockStatus = _catalog->unlock(opCtx.get(), toUnlock.first, *toUnlock.second);
+ } else {
+ unlockStatus = _catalog->unlock(opCtx.get(), toUnlock.first);
+ }
+
+ if (!unlockStatus.isOK()) {
+ LOGV2_WARNING(22670,
+ "Error unlocking distributed lock {lockName} with sessionID "
+ "{lockSessionId} caused by {error}",
+ "Error unlocking distributed lock",
+ "lockName"_attr = toUnlock.second,
+ "lockSessionId"_attr = toUnlock.first,
+ "error"_attr = unlockStatus);
+ // Queue another attempt, unless the problem was no longer being primary.
+ if (unlockStatus != ErrorCodes::NotWritablePrimary) {
+ queueUnlock(toUnlock.first, toUnlock.second);
+ }
+ } else {
+ LOGV2(22650,
+ "Unlocked distributed lock {lockName} with sessionID {lockSessionId}",
+ "Unlocked distributed lock",
+ "lockName"_attr = toUnlock.second,
+ "lockSessionId"_attr = toUnlock.first);
+ }
+
+ if (isShutDown()) {
+ return;
+ }
+ }
+ }
+
+ MONGO_IDLE_THREAD_BLOCK;
+ stdx::unique_lock<Latch> lk(_mutex);
+ _shutDownCV.wait_for(lk, _pingInterval.toSystemDuration(), [this] { return _isShutDown; });
+ }
+}
+
+StatusWith<bool> ReplSetDistLockManager::isLockExpired(OperationContext* opCtx,
+ LocksType lockDoc,
+ const Milliseconds& lockExpiration) {
+ const auto& processID = lockDoc.getProcess();
+ auto pingStatus = _catalog->getPing(opCtx, processID);
+
+ Date_t pingValue;
+ if (pingStatus.isOK()) {
+ const auto& pingDoc = pingStatus.getValue();
+ Status pingDocValidationStatus = pingDoc.validate();
+ if (!pingDocValidationStatus.isOK()) {
+ return {ErrorCodes::UnsupportedFormat,
+ str::stream() << "invalid ping document for " << processID << ": "
+ << pingDocValidationStatus.toString()};
+ }
+
+ pingValue = pingDoc.getPing();
+ } else if (pingStatus.getStatus() != ErrorCodes::NoMatchingDocument) {
+ return pingStatus.getStatus();
+ } // else use default pingValue if ping document does not exist.
+
+ Timer timer(_serviceContext->getTickSource());
+ auto serverInfoStatus = _catalog->getServerInfo(opCtx);
+ if (!serverInfoStatus.isOK()) {
+ if (serverInfoStatus.getStatus() == ErrorCodes::NotWritablePrimary) {
+ return false;
+ }
+
+ return serverInfoStatus.getStatus();
+ }
+
+ // Be conservative when determining that lock expiration has elapsed by
+ // taking into account the roundtrip delay of trying to get the local
+ // time from the config server.
+ Milliseconds delay(timer.millis() / 2); // Assuming symmetrical delay.
+
+ const auto& serverInfo = serverInfoStatus.getValue();
+
+ stdx::lock_guard<Latch> lk(_mutex);
+ auto pingIter = _pingHistory.find(lockDoc.getName());
+
+ if (pingIter == _pingHistory.end()) {
+ // We haven't seen this lock before so we don't have any point of reference
+ // to compare and determine the elapsed time. Save the current ping info
+ // for this lock.
+ _pingHistory.emplace(std::piecewise_construct,
+ std::forward_as_tuple(lockDoc.getName()),
+ std::forward_as_tuple(processID,
+ pingValue,
+ serverInfo.serverTime,
+ lockDoc.getLockID(),
+ serverInfo.electionId));
+ return false;
+ }
+
+ auto configServerLocalTime = serverInfo.serverTime - delay;
+
+ auto* pingInfo = &pingIter->second;
+
+ LOGV2_DEBUG(22651,
+ 1,
+ "Checking last ping for lock {lockName} against last seen process {processId} and "
+ "ping {lastPing}",
+ "Checking last ping for lock",
+ "lockName"_attr = lockDoc.getName(),
+ "processId"_attr = pingInfo->processId,
+ "lastPing"_attr = pingInfo->lastPing);
+
+ if (pingInfo->lastPing != pingValue || // ping is active
+
+ // Owner of this lock is now different from last time so we can't
+ // use the ping data.
+ pingInfo->lockSessionId != lockDoc.getLockID() ||
+
+ // Primary changed, we can't trust that clocks are synchronized so
+ // treat as if this is a new entry.
+ pingInfo->electionId != serverInfo.electionId) {
+ pingInfo->lastPing = pingValue;
+ pingInfo->electionId = serverInfo.electionId;
+ pingInfo->configLocalTime = configServerLocalTime;
+ pingInfo->lockSessionId = lockDoc.getLockID();
+ return false;
+ }
+
+ if (configServerLocalTime < pingInfo->configLocalTime) {
+ LOGV2_WARNING(22671,
+ "Config server local time went backwards, new value "
+ "{newConfigServerLocalTime}, old value {oldConfigServerLocalTime}",
+ "Config server local time went backwards",
+ "newConfigServerLocalTime"_attr = configServerLocalTime,
+ "oldConfigServerLocalTime"_attr = pingInfo->configLocalTime);
+ return false;
+ }
+
+ Milliseconds elapsedSinceLastPing(configServerLocalTime - pingInfo->configLocalTime);
+ if (elapsedSinceLastPing >= lockExpiration) {
+ LOGV2(22652,
+ "Forcing lock {lockName} because elapsed time {elapsedSinceLastPing} >= "
+ "takeover time {lockExpirationTimeout}",
+ "Forcing lock because too much time has passed from last ping",
+ "lockName"_attr = lockDoc.getName(),
+ "elapsedSinceLastPing"_attr = elapsedSinceLastPing,
+ "lockExpirationTimeout"_attr = lockExpiration);
+ return true;
+ }
+
+ LOGV2_DEBUG(22653,
+ 1,
+ "Could not force lock of {lockName} because elapsed time {elapsedSinceLastPing} < "
+ "takeover time {lockExpirationTimeout}",
+ "Could not force lock because too little time has passed from last ping",
+ "lockName"_attr = lockDoc.getName(),
+ "elapsedSinceLastPing"_attr = elapsedSinceLastPing,
+ "lockExpirationTimeout"_attr = lockExpiration);
+ return false;
+}
+
+StatusWith<DistLockHandle> ReplSetDistLockManager::lockWithSessionID(OperationContext* opCtx,
+ StringData name,
+ StringData whyMessage,
+ const OID& lockSessionID,
+ Milliseconds waitFor) {
+ Timer timer(_serviceContext->getTickSource());
+ Timer msgTimer(_serviceContext->getTickSource());
+
+ // Counts how many attempts have been made to grab the lock, which have failed with network
+ // error. This value is reset for each lock acquisition attempt because these are
+ // independent write operations.
+ int networkErrorRetries = 0;
+
+ auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
+
+ // Distributed lock acquisition works by tring to update the state of the lock to 'taken'. If
+ // the lock is currently taken, we will back off and try the acquisition again, repeating this
+ // until the lockTryInterval has been reached. If a network error occurs at each lock
+ // acquisition attempt, the lock acquisition will be retried immediately.
+ while (waitFor <= Milliseconds::zero() || Milliseconds(timer.millis()) < waitFor) {
+ const std::string who = str::stream() << _processID << ":" << getThreadName();
+
+ auto lockExpiration = _lockExpiration;
+ setDistLockTimeout.execute([&](const BSONObj& data) {
+ lockExpiration = Milliseconds(data["timeoutMs"].numberInt());
+ });
+
+ LOGV2_DEBUG(22654,
+ 1,
+ "Trying to acquire new distributed lock for {lockName} ( "
+ "lockSessionID: {lockSessionId}, "
+ "process : {processId}, "
+ "lock timeout : {lockExpirationTimeout}, "
+ "ping interval : {pingInterval}, "
+ "reason: {reason} )",
+ "Trying to acquire new distributed lock",
+ "lockName"_attr = name,
+ "lockSessionId"_attr = lockSessionID,
+ "processId"_attr = _processID,
+ "lockExpirationTimeout"_attr = lockExpiration,
+ "pingInterval"_attr = _pingInterval,
+ "reason"_attr = whyMessage);
+
+ auto lockResult = _catalog->grabLock(
+ opCtx, name, lockSessionID, who, _processID, Date_t::now(), whyMessage.toString());
+
+ auto status = lockResult.getStatus();
+
+ if (status.isOK()) {
+ // Lock is acquired since findAndModify was able to successfully modify
+ // the lock document.
+ LOGV2(22655,
+ "Acquired distributed lock {lockName} with session ID {lockSessionId} for "
+ "{reason}",
+ "Acquired distributed lock",
+ "lockName"_attr = name,
+ "lockSessionId"_attr = lockSessionID,
+ "reason"_attr = whyMessage);
+ return lockSessionID;
+ }
+
+ // If a network error occurred, unlock the lock synchronously and try again
+ if (configShard->isRetriableError(status.code(), Shard::RetryPolicy::kIdempotent) &&
+ networkErrorRetries < kMaxNumLockAcquireRetries) {
+ LOGV2_DEBUG(22656,
+ 1,
+ "Error acquiring distributed lock because of retryable error. "
+ "Retrying acquisition by first unlocking the stale entry, which possibly "
+ "exists now. Caused by {error}",
+ "Error acquiring distributed lock because of retryable error. "
+ "Retrying acquisition by first unlocking the stale entry, which possibly "
+ "exists now",
+ "error"_attr = redact(status));
+
+ networkErrorRetries++;
+
+ status = _catalog->unlock(opCtx, lockSessionID, name);
+ if (status.isOK()) {
+ // We certainly do not own the lock, so we can retry
+ continue;
+ }
+
+ // Fall-through to the error checking logic below
+ invariant(status != ErrorCodes::LockStateChangeFailed);
+
+ LOGV2_DEBUG(22657,
+ 1,
+ "Last attempt to acquire distributed lock failed with {error}",
+ "Last attempt to acquire distributed lock failed",
+ "error"_attr = redact(status));
+ }
+
+ if (status != ErrorCodes::LockStateChangeFailed) {
+ // An error occurred but the write might have actually been applied on the
+ // other side. Schedule an unlock to clean it up just in case.
+ queueUnlock(lockSessionID, name.toString());
+ return status;
+ }
+
+ // Get info from current lock and check if we can overtake it.
+ auto getLockStatusResult = _catalog->getLockByName(opCtx, name);
+ const auto& getLockStatus = getLockStatusResult.getStatus();
+
+ if (!getLockStatusResult.isOK() && getLockStatus != ErrorCodes::LockNotFound) {
+ return getLockStatus;
+ }
+
+ // Note: Only attempt to overtake locks that actually exists. If lock was not
+ // found, use the normal grab lock path to acquire it.
+ if (getLockStatusResult.isOK()) {
+ auto currentLock = getLockStatusResult.getValue();
+ auto isLockExpiredResult = isLockExpired(opCtx, currentLock, lockExpiration);
+
+ if (!isLockExpiredResult.isOK()) {
+ return isLockExpiredResult.getStatus();
+ }
+
+ if (isLockExpiredResult.getValue() || (lockSessionID == currentLock.getLockID())) {
+ auto overtakeResult = _catalog->overtakeLock(opCtx,
+ name,
+ lockSessionID,
+ currentLock.getLockID(),
+ who,
+ _processID,
+ Date_t::now(),
+ whyMessage);
+
+ const auto& overtakeStatus = overtakeResult.getStatus();
+
+ if (overtakeResult.isOK()) {
+ // Lock is acquired since findAndModify was able to successfully modify
+ // the lock document.
+
+ LOGV2(22658,
+ "Acquired distributed lock {lockName} with sessionId {lockSessionId}",
+ "Acquired distributed lock",
+ "lockName"_attr = name,
+ "lockSessionId"_attr = lockSessionID);
+ return lockSessionID;
+ }
+
+ if (overtakeStatus != ErrorCodes::LockStateChangeFailed) {
+ // An error occurred but the write might have actually been applied on the
+ // other side. Schedule an unlock to clean it up just in case.
+ queueUnlock(lockSessionID, boost::none);
+ return overtakeStatus;
+ }
+ }
+ }
+
+ LOGV2_DEBUG(22660,
+ 1,
+ "Distributed lock {lockName} was not acquired",
+ "Distributed lock was not acquired",
+ "lockName"_attr = name);
+
+ if (waitFor == Milliseconds::zero()) {
+ break;
+ }
+
+ // Periodically message for debugging reasons
+ if (msgTimer.seconds() > 10) {
+ LOGV2(22661,
+ "Waited {elapsed} for distributed lock {lockName} for {reason}",
+ "Waiting for distributed lock",
+ "lockName"_attr = name,
+ "elapsed"_attr = Seconds(timer.seconds()),
+ "reason"_attr = whyMessage);
+
+ msgTimer.reset();
+ }
+
+ // A new lock acquisition attempt will begin now (because the previous found the lock to be
+ // busy, so reset the retries counter)
+ networkErrorRetries = 0;
+
+ const Milliseconds timeRemaining =
+ std::max(Milliseconds::zero(), waitFor - Milliseconds(timer.millis()));
+ sleepFor(std::min(kLockRetryInterval, timeRemaining));
+ }
+
+ return {ErrorCodes::LockBusy, str::stream() << "timed out waiting for " << name};
+}
+
+StatusWith<DistLockHandle> ReplSetDistLockManager::tryLockWithLocalWriteConcern(
+ OperationContext* opCtx, StringData name, StringData whyMessage, const OID& lockSessionID) {
+ const std::string who = str::stream() << _processID << ":" << getThreadName();
+
+ LOGV2_DEBUG(22662,
+ 1,
+ "Trying to acquire new distributed lock for {lockName} ( "
+ "process : {processId}, "
+ "lockSessionID: {lockSessionId}, "
+ "lock timeout : {lockExpirationTimeout}, "
+ "ping interval : {pingInterval}, "
+ "reason: {reason} )",
+ "Trying to acquire new distributed lock",
+ "lockName"_attr = name,
+ "lockSessionId"_attr = lockSessionID,
+ "processId"_attr = _processID,
+ "lockExpirationTimeout"_attr = _lockExpiration,
+ "pingInterval"_attr = _pingInterval,
+ "reason"_attr = whyMessage);
+
+ auto lockStatus = _catalog->grabLock(opCtx,
+ name,
+ lockSessionID,
+ who,
+ _processID,
+ Date_t::now(),
+ whyMessage.toString(),
+ DistLockCatalog::kLocalWriteConcern);
+
+ if (lockStatus.isOK()) {
+ LOGV2(22663,
+ "Acquired distributed lock {lockName} with session ID {lockSessionId} for "
+ "{reason}",
+ "Acquired distributed lock",
+ "lockName"_attr = name,
+ "lockSessionId"_attr = lockSessionID,
+ "reason"_attr = whyMessage);
+ return lockSessionID;
+ }
+
+ LOGV2_DEBUG(22664,
+ 1,
+ "Distributed lock {lockName} was not acquired",
+ "Distributed lock was not acquired",
+ "lockName"_attr = name);
+
+ if (lockStatus == ErrorCodes::LockStateChangeFailed) {
+ return {ErrorCodes::LockBusy, str::stream() << "Unable to acquire " << name};
+ }
+
+ return lockStatus.getStatus();
+}
+
+void ReplSetDistLockManager::unlock(OperationContext* opCtx, const DistLockHandle& lockSessionID) {
+ auto unlockStatus = _catalog->unlock(opCtx, lockSessionID);
+
+ if (!unlockStatus.isOK()) {
+ queueUnlock(lockSessionID, boost::none);
+ } else {
+ LOGV2(22665,
+ "Unlocked distributed lock with sessionID {lockSessionId}",
+ "Unlocked distributed lock",
+ "lockSessionId"_attr = lockSessionID);
+ }
+}
+
+void ReplSetDistLockManager::unlock(OperationContext* opCtx,
+ const DistLockHandle& lockSessionID,
+ StringData name) {
+ auto unlockStatus = _catalog->unlock(opCtx, lockSessionID, name);
+
+ if (!unlockStatus.isOK()) {
+ queueUnlock(lockSessionID, name.toString());
+ } else {
+ LOGV2(22666,
+ "Unlocked distributed lock {lockName} with sessionID {lockSessionId}",
+ "Unlocked distributed lock",
+ "lockName"_attr = name,
+ "lockSessionId"_attr = lockSessionID);
+ }
+}
+
+void ReplSetDistLockManager::unlockAll(OperationContext* opCtx, const std::string& processID) {
+ Status status = _catalog->unlockAll(opCtx, processID);
+ if (!status.isOK()) {
+ LOGV2_WARNING(
+ 22672,
+ "Error unlocking all distributed locks for process {processId} caused by {error}",
+ "Error unlocking all existing distributed locks for a process",
+ "processId"_attr = processID,
+ "error"_attr = redact(status));
+ }
+}
+
+Status ReplSetDistLockManager::checkStatus(OperationContext* opCtx,
+ const DistLockHandle& lockHandle) {
+ return _catalog->getLockByTS(opCtx, lockHandle).getStatus();
+}
+
+void ReplSetDistLockManager::queueUnlock(const DistLockHandle& lockSessionID,
+ const boost::optional<std::string>& name) {
+ stdx::unique_lock<Latch> lk(_mutex);
+ _unlockList.push_back(std::make_pair(lockSessionID, name));
+}
+
+ReplSetDistLockManager::DistLockPingInfo::DistLockPingInfo() = default;
+
+ReplSetDistLockManager::DistLockPingInfo::DistLockPingInfo(
+ StringData idArg, Date_t lastPingArg, Date_t remoteArg, OID tsArg, OID electionIdArg)
+ : processId(idArg.toString()),
+ lastPing(lastPingArg),
+ configLocalTime(remoteArg),
+ lockSessionId(std::move(tsArg)),
+ electionId(std::move(electionIdArg)) {}
+
+} // namespace mongo
diff --git a/src/mongo/db/s/dist_lock_manager_replset.h b/src/mongo/db/s/dist_lock_manager_replset.h
new file mode 100644
index 00000000000..3c0ce75c6c2
--- /dev/null
+++ b/src/mongo/db/s/dist_lock_manager_replset.h
@@ -0,0 +1,173 @@
+/**
+ * Copyright (C) 2018-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#pragma once
+
+#include <deque>
+
+#include "mongo/db/s/dist_lock_catalog.h"
+#include "mongo/db/s/dist_lock_manager.h"
+#include "mongo/platform/mutex.h"
+#include "mongo/stdx/condition_variable.h"
+#include "mongo/stdx/thread.h"
+#include "mongo/stdx/unordered_map.h"
+
+namespace mongo {
+
+class ReplSetDistLockManager : public DistLockManager {
+public:
+ // How frequently should the dist lock pinger thread run and write liveness information about
+ // this instance of the dist lock manager
+ static const Seconds kDistLockPingInterval;
+
+ // How long should the lease on a distributed lock last
+ static const Minutes kDistLockExpirationTime;
+
+ ReplSetDistLockManager(ServiceContext* globalContext,
+ StringData processID,
+ std::unique_ptr<DistLockCatalog> catalog,
+ Milliseconds pingInterval,
+ Milliseconds lockExpiration);
+
+ virtual ~ReplSetDistLockManager();
+
+ void startUp() override;
+ void shutDown(OperationContext* opCtx) override;
+
+ std::string getProcessID() override;
+
+ StatusWith<DistLockHandle> lockWithSessionID(OperationContext* opCtx,
+ StringData name,
+ StringData whyMessage,
+ const OID& lockSessionID,
+ Milliseconds waitFor) override;
+
+ StatusWith<DistLockHandle> tryLockWithLocalWriteConcern(OperationContext* opCtx,
+ StringData name,
+ StringData whyMessage,
+ const OID& lockSessionID) override;
+
+ void unlock(OperationContext* opCtx, const DistLockHandle& lockSessionID) override;
+
+ void unlock(OperationContext* opCtx,
+ const DistLockHandle& lockSessionID,
+ StringData name) override;
+
+ void unlockAll(OperationContext* opCtx, const std::string& processID) override;
+
+private:
+ Status checkStatus(OperationContext* opCtx, const DistLockHandle& lockSessionID) override;
+
+ /**
+ * Queue a lock to be unlocked asynchronously with retry until it doesn't error.
+ */
+ void queueUnlock(const DistLockHandle& lockSessionID, const boost::optional<std::string>& name);
+
+ /**
+ * Periodically pings and checks if there are locks queued that needs unlocking.
+ */
+ void doTask();
+
+ /**
+ * Returns true if shutDown was called.
+ */
+ bool isShutDown();
+
+ /**
+ * Returns true if the current process that owns the lock has no fresh pings since
+ * the lock expiration threshold.
+ */
+ StatusWith<bool> isLockExpired(OperationContext* opCtx,
+ const LocksType lockDoc,
+ const Milliseconds& lockExpiration);
+
+ /**
+ * Data structure for storing information about distributed lock pings.
+ */
+ struct DistLockPingInfo {
+ DistLockPingInfo();
+ DistLockPingInfo(StringData processId,
+ Date_t lastPing,
+ Date_t configLocalTime,
+ OID lockSessionId,
+ OID electionId);
+
+ // the process processId of the last known owner of the lock.
+ std::string processId;
+
+ // the ping value from the last owner of the lock.
+ Date_t lastPing;
+
+ // the config server local time when this object was updated.
+ Date_t configLocalTime;
+
+ // last known owner of the lock.
+ OID lockSessionId;
+
+ // the election id of the config server when this object was updated.
+ // Note: unused by legacy dist lock.
+ OID electionId;
+ };
+
+ //
+ // All member variables are labeled with one of the following codes indicating the
+ // synchronization rules for accessing them.
+ //
+ // (F) Self synchronizing.
+ // (M) Must hold _mutex for access.
+ // (I) Immutable, no synchronization needed.
+ // (S) Can only be called inside startUp/shutDown.
+ //
+
+ ServiceContext* const _serviceContext; // (F)
+
+ const std::string _processID; // (I)
+ const std::unique_ptr<DistLockCatalog> _catalog; // (I)
+ const Milliseconds _pingInterval; // (I)
+ const Milliseconds _lockExpiration; // (I)
+
+ Mutex _mutex = MONGO_MAKE_LATCH("ReplSetDistLockManager::_mutex");
+ std::unique_ptr<stdx::thread> _execThread; // (S)
+
+ // Contains the list of locks queued for unlocking. Cases when unlock operation can
+ // be queued include:
+ // 1. First attempt on unlocking resulted in an error.
+ // 2. Attempting to grab or overtake a lock resulted in an error where we are uncertain
+ // whether the modification was actually applied or not, and call unlock to make
+ // sure that it was cleaned up.
+ std::deque<std::pair<DistLockHandle, boost::optional<std::string>>> _unlockList; // (M)
+
+ bool _isShutDown = false; // (M)
+ stdx::condition_variable _shutDownCV; // (M)
+
+ // Map of lockName to last ping information.
+ stdx::unordered_map<std::string, DistLockPingInfo> _pingHistory; // (M)
+};
+
+} // namespace mongo
diff --git a/src/mongo/db/s/dist_lock_manager_replset_test.cpp b/src/mongo/db/s/dist_lock_manager_replset_test.cpp
new file mode 100644
index 00000000000..94e0a7be9a9
--- /dev/null
+++ b/src/mongo/db/s/dist_lock_manager_replset_test.cpp
@@ -0,0 +1,2135 @@
+/**
+ * Copyright (C) 2018-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include <boost/optional.hpp>
+#include <map>
+#include <string>
+#include <vector>
+
+#include "mongo/bson/json.h"
+#include "mongo/db/s/dist_lock_catalog_mock.h"
+#include "mongo/db/s/dist_lock_manager_replset.h"
+#include "mongo/db/s/shard_server_test_fixture.h"
+#include "mongo/platform/mutex.h"
+#include "mongo/s/balancer_configuration.h"
+#include "mongo/s/catalog/sharding_catalog_client_mock.h"
+#include "mongo/s/catalog/type_lockpings.h"
+#include "mongo/s/catalog/type_locks.h"
+#include "mongo/s/grid.h"
+#include "mongo/stdx/condition_variable.h"
+#include "mongo/unittest/unittest.h"
+#include "mongo/util/system_tick_source.h"
+#include "mongo/util/tick_source_mock.h"
+#include "mongo/util/time_support.h"
+
+/**
+ * Tests for ReplSetDistLockManager. Note that unlock and ping operations are executed on a separate
+ * thread. And since this thread cannot capture the assertion exceptions, all the assertion calls
+ * should be performed on the main thread.
+ */
+namespace mongo {
+namespace {
+
+// Max duration to wait to satisfy test invariant before joining with main test thread.
+const Seconds kJoinTimeout(30);
+const Milliseconds kPingInterval(2);
+const Seconds kLockExpiration(10);
+
+std::string mapToString(const std::map<OID, int>& map) {
+ StringBuilder str;
+ for (const auto& entry : map) {
+ str << "(" << entry.first.toString() << ": " << entry.second << ")";
+ }
+
+ return str.str();
+}
+
+std::string vectorToString(const std::vector<OID>& list) {
+ StringBuilder str;
+ for (const auto& entry : list) {
+ str << "(" << entry.toString() << ")";
+ }
+
+ return str.str();
+}
+
+/**
+ * Basic fixture for ReplSetDistLockManager that starts it up before the test begins
+ * and shuts it down when a test finishes.
+ */
+class DistLockManagerReplSetTest : public ShardServerTestFixture {
+protected:
+ void tearDown() override {
+ // Don't care about what shutDown passes to stopPing here.
+ getMockCatalog()->expectStopPing([](StringData) {}, Status::OK());
+
+ ShardServerTestFixture::tearDown();
+ }
+
+ std::unique_ptr<DistLockManager> makeDistLockManager() override {
+ auto distLockCatalogMock = std::make_unique<DistLockCatalogMock>();
+ _distLockCatalogMock = distLockCatalogMock.get();
+ return std::make_unique<ReplSetDistLockManager>(getServiceContext(),
+ _processID,
+ std::move(distLockCatalogMock),
+ kPingInterval,
+ kLockExpiration);
+ }
+
+ std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient() override {
+ return std::make_unique<ShardingCatalogClientMock>();
+ }
+
+ std::unique_ptr<BalancerConfiguration> makeBalancerConfiguration() override {
+ return std::make_unique<BalancerConfiguration>();
+ }
+
+ DistLockCatalogMock* getMockCatalog() const {
+ return _distLockCatalogMock;
+ }
+
+ /**
+ * Get the process id that was initialized with the lock manager being tested.
+ */
+ std::string getProcessID() const {
+ return _processID;
+ }
+
+private:
+ const std::string _processID = "test";
+
+ DistLockCatalogMock* _distLockCatalogMock{nullptr};
+};
+
+/**
+ * Test scenario:
+ * 1. Grab lock.
+ * 2. Unlock (on destructor of ScopedDistLock).
+ * 3. Check lock id used in lock and unlock are the same.
+ */
+TEST_F(DistLockManagerReplSetTest, BasicLockLifeCycle) {
+ std::string lockName("test");
+ Date_t now(Date_t::now());
+ std::string whyMsg("because");
+
+ LocksType retLockDoc;
+ retLockDoc.setName(lockName);
+ retLockDoc.setState(LocksType::LOCKED);
+ retLockDoc.setProcess(getProcessID());
+ retLockDoc.setWho("me");
+ retLockDoc.setWhy(whyMsg);
+ // Will be different from the actual lock session id. For testing only.
+ retLockDoc.setLockID(OID::gen());
+
+ OID lockSessionIDPassed;
+
+ getMockCatalog()->expectGrabLock(
+ [this, &lockName, &now, &whyMsg, &lockSessionIDPassed](StringData lockID,
+ const OID& lockSessionID,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ ASSERT_EQUALS(lockName, lockID);
+ ASSERT_TRUE(lockSessionID.isSet());
+ ASSERT_EQUALS(getProcessID(), processId);
+ ASSERT_GREATER_THAN_OR_EQUALS(time, now);
+ ASSERT_EQUALS(whyMsg, why);
+
+ lockSessionIDPassed = lockSessionID;
+ getMockCatalog()->expectNoGrabLock(); // Call only once.
+ },
+ retLockDoc);
+
+ int unlockCallCount = 0;
+ OID unlockSessionIDPassed;
+
+ {
+ auto lockStatus = DistLockManager::get(operationContext())
+ ->lock(operationContext(),
+ lockName,
+ whyMsg,
+ DistLockManager::kSingleLockAttemptTimeout);
+ ASSERT_OK(lockStatus.getStatus());
+
+ getMockCatalog()->expectNoGrabLock();
+ getMockCatalog()->expectUnLock(
+ [&unlockCallCount, &unlockSessionIDPassed](const OID& lockSessionID) {
+ unlockCallCount++;
+ unlockSessionIDPassed = lockSessionID;
+ },
+ Status::OK());
+ }
+
+ ASSERT_EQUALS(1, unlockCallCount);
+ ASSERT_EQUALS(lockSessionIDPassed, unlockSessionIDPassed);
+}
+
+/**
+ * Test scenario:
+ * 1. Set mock to error on grab lock.
+ * 2. Grab lock attempted.
+ * 3. Wait for unlock to be called.
+ * 4. Check that lockSessionID used on all unlock is the same as the one used to grab lock.
+ */
+TEST_F(DistLockManagerReplSetTest, MustUnlockOnLockError) {
+ std::string lockName("test");
+ std::string me("me");
+ OID lastTS;
+ std::string whyMsg("because");
+
+ getMockCatalog()->expectGrabLock(
+ [this, &lockName, &lastTS, &me, &whyMsg](StringData lockID,
+ const OID& lockSessionID,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ ASSERT_EQUALS(lockName, lockID);
+ // Every attempt should have a unique sesssion ID.
+ ASSERT_TRUE(lockSessionID.isSet());
+ ASSERT_EQUALS(getProcessID(), processId);
+ ASSERT_EQUALS(whyMsg, why);
+
+ lastTS = lockSessionID;
+ getMockCatalog()->expectNoGrabLock();
+ },
+ {ErrorCodes::ExceededMemoryLimit, "bad remote server"});
+
+ auto unlockMutex = MONGO_MAKE_LATCH();
+ stdx::condition_variable unlockCV;
+ int unlockCallCount = 0;
+ OID unlockSessionIDPassed;
+
+ getMockCatalog()->expectUnLock(
+ [&unlockMutex, &unlockCV, &unlockCallCount, &unlockSessionIDPassed](
+ const OID& lockSessionID) {
+ stdx::unique_lock<Latch> lk(unlockMutex);
+ unlockCallCount++;
+ unlockSessionIDPassed = lockSessionID;
+ unlockCV.notify_all();
+ },
+ Status::OK());
+
+ auto lockStatus = DistLockManager::get(operationContext())
+ ->lock(operationContext(), lockName, whyMsg, Milliseconds(10))
+ .getStatus();
+ ASSERT_NOT_OK(lockStatus);
+ ASSERT_EQUALS(ErrorCodes::ExceededMemoryLimit, lockStatus.code());
+
+ bool didTimeout = false;
+ {
+ stdx::unique_lock<Latch> lk(unlockMutex);
+ if (unlockCallCount == 0) {
+ didTimeout =
+ unlockCV.wait_for(lk, kJoinTimeout.toSystemDuration()) == stdx::cv_status::timeout;
+ }
+ }
+
+ // Join the background thread before trying to call asserts. Shutdown calls
+ // stopPing and we don't care in this test.
+ getMockCatalog()->expectStopPing([](StringData) {}, Status::OK());
+ DistLockManager::get(operationContext())->shutDown(operationContext());
+
+ // No assert until shutDown has been called to make sure that the background thread
+ // won't be trying to access the local variables that were captured by lamdas that
+ // may have gone out of scope when the assert unwinds the stack.
+ // No need to grab unlockMutex since there is only one thread running at this point.
+
+ ASSERT_FALSE(didTimeout);
+ ASSERT_EQUALS(1, unlockCallCount);
+ ASSERT_EQUALS(lastTS, unlockSessionIDPassed);
+}
+
+/**
+ * Test scenario:
+ * 1. Ping thread started during setUp of fixture.
+ * 2. Wait until ping was called at least 3 times.
+ * 3. Check that correct process is being pinged.
+ */
+TEST_F(DistLockManagerReplSetTest, LockPinging) {
+ auto testMutex = MONGO_MAKE_LATCH();
+ stdx::condition_variable ping3TimesCV;
+ std::vector<std::string> processIDList;
+
+ getMockCatalog()->expectPing(
+ [&testMutex, &ping3TimesCV, &processIDList](StringData processIDArg, Date_t ping) {
+ stdx::lock_guard<Latch> lk(testMutex);
+ processIDList.push_back(processIDArg.toString());
+
+ if (processIDList.size() >= 3) {
+ ping3TimesCV.notify_all();
+ }
+ },
+ Status::OK());
+
+ bool didTimeout = false;
+ {
+ stdx::unique_lock<Latch> lk(testMutex);
+ if (processIDList.size() < 3) {
+ didTimeout = ping3TimesCV.wait_for(lk, kJoinTimeout.toSystemDuration()) ==
+ stdx::cv_status::timeout;
+ }
+ }
+
+ // Join the background thread before trying to call asserts. Shutdown calls
+ // stopPing and we don't care in this test.
+ getMockCatalog()->expectStopPing([](StringData) {}, Status::OK());
+ DistLockManager::get(operationContext())->shutDown(operationContext());
+
+ // No assert until shutDown has been called to make sure that the background thread
+ // won't be trying to access the local variables that were captured by lamdas that
+ // may have gone out of scope when the assert unwinds the stack.
+ // No need to grab testMutex since there is only one thread running at this point.
+
+ ASSERT_FALSE(didTimeout);
+
+ ASSERT_FALSE(processIDList.empty());
+ for (const auto& processIDArg : processIDList) {
+ ASSERT_EQUALS(getProcessID(), processIDArg);
+ }
+}
+
+/**
+ * Test scenario:
+ * 1. Grab lock.
+ * 2. Unlock fails 3 times.
+ * 3. Unlock finally succeeds at the 4th time.
+ * 4. Check that lockSessionID used on all unlock is the same as the one used to grab lock.
+ */
+TEST_F(DistLockManagerReplSetTest, UnlockUntilNoError) {
+ auto unlockMutex = MONGO_MAKE_LATCH();
+ stdx::condition_variable unlockCV;
+ const unsigned int kUnlockErrorCount = 3;
+ std::vector<OID> lockSessionIDPassed;
+
+ getMockCatalog()->expectUnLock(
+ [this, &unlockMutex, &unlockCV, &kUnlockErrorCount, &lockSessionIDPassed](
+ const OID& lockSessionID) {
+ stdx::unique_lock<Latch> lk(unlockMutex);
+ lockSessionIDPassed.push_back(lockSessionID);
+
+ if (lockSessionIDPassed.size() >= kUnlockErrorCount) {
+ getMockCatalog()->expectUnLock(
+ [&lockSessionIDPassed, &unlockMutex, &unlockCV](const OID& lockSessionID) {
+ stdx::unique_lock<Latch> lk(unlockMutex);
+ lockSessionIDPassed.push_back(lockSessionID);
+ unlockCV.notify_all();
+ },
+ Status::OK());
+ }
+ },
+ {ErrorCodes::NetworkTimeout, "bad test network"});
+
+ OID lockSessionID;
+ LocksType retLockDoc;
+ retLockDoc.setName("test");
+ retLockDoc.setState(LocksType::LOCKED);
+ retLockDoc.setProcess(getProcessID());
+ retLockDoc.setWho("me");
+ retLockDoc.setWhy("why");
+ // Will be different from the actual lock session id. For testing only.
+ retLockDoc.setLockID(OID::gen());
+
+ getMockCatalog()->expectGrabLock(
+ [&lockSessionID](StringData lockID,
+ const OID& lockSessionIDArg,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) { lockSessionID = lockSessionIDArg; },
+ retLockDoc);
+
+ {
+ auto lockStatus = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "test", "why", Milliseconds(0));
+ }
+
+ bool didTimeout = false;
+ {
+ stdx::unique_lock<Latch> lk(unlockMutex);
+ if (lockSessionIDPassed.size() < kUnlockErrorCount) {
+ didTimeout =
+ unlockCV.wait_for(lk, kJoinTimeout.toSystemDuration()) == stdx::cv_status::timeout;
+ }
+ }
+
+ // Join the background thread before trying to call asserts. Shutdown calls
+ // stopPing and we don't care in this test.
+ getMockCatalog()->expectStopPing([](StringData) {}, Status::OK());
+ DistLockManager::get(operationContext())->shutDown(operationContext());
+
+ // No assert until shutDown has been called to make sure that the background thread
+ // won't be trying to access the local variables that were captured by lamdas that
+ // may have gone out of scope when the assert unwinds the stack.
+ // No need to grab testMutex since there is only one thread running at this point.
+
+ ASSERT_FALSE(didTimeout);
+
+ for (const auto& id : lockSessionIDPassed) {
+ ASSERT_EQUALS(lockSessionID, id);
+ }
+}
+
+/**
+ * Test scenario:
+ * 1. Grab 2 locks.
+ * 2. Trigger unlocks by making ScopedDistLock go out of scope.
+ * 3. Unlocks fail and will be queued for retry.
+ * 4. Unlocks will keep on failing until we see at least 3 unique ids being unlocked more
+ * than once. This implies that both ids have been retried at least 3 times.
+ * 5. Check that the lock session id used when lock was called matches with unlock.
+ */
+TEST_F(DistLockManagerReplSetTest, MultipleQueuedUnlock) {
+ auto testMutex = MONGO_MAKE_LATCH();
+ stdx::condition_variable unlockCV;
+ std::vector<OID> lockSessionIDPassed;
+ std::map<OID, int> unlockIDMap; // id -> count
+
+ /**
+ * Returns true if all values in the map are greater than 2.
+ */
+ auto mapEntriesGreaterThanTwo = [](const decltype(unlockIDMap)& map) -> bool {
+ auto iter = find_if(
+ map.begin(),
+ map.end(),
+ [](const std::remove_reference<decltype(map)>::type::value_type& entry) -> bool {
+ return entry.second < 3;
+ });
+
+ return iter == map.end();
+ };
+
+ getMockCatalog()->expectUnLock(
+ [this, &unlockIDMap, &testMutex, &unlockCV, &mapEntriesGreaterThanTwo](
+ const OID& lockSessionID) {
+ stdx::unique_lock<Latch> lk(testMutex);
+ unlockIDMap[lockSessionID]++;
+
+ // Wait until we see at least 2 unique lockSessionID more than twice.
+ if (unlockIDMap.size() >= 2 && mapEntriesGreaterThanTwo(unlockIDMap)) {
+ getMockCatalog()->expectUnLock(
+ [&testMutex, &unlockCV](const OID& lockSessionID) {
+ stdx::unique_lock<Latch> lk(testMutex);
+ unlockCV.notify_all();
+ },
+ Status::OK());
+ }
+ },
+ {ErrorCodes::NetworkTimeout, "bad test network"});
+
+ LocksType retLockDoc;
+ retLockDoc.setName("test");
+ retLockDoc.setState(LocksType::LOCKED);
+ retLockDoc.setProcess(getProcessID());
+ retLockDoc.setWho("me");
+ retLockDoc.setWhy("why");
+ // Will be different from the actual lock session id. For testing only.
+ retLockDoc.setLockID(OID::gen());
+
+ getMockCatalog()->expectGrabLock(
+ [&testMutex, &lockSessionIDPassed](StringData lockID,
+ const OID& lockSessionIDArg,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ stdx::unique_lock<Latch> lk(testMutex);
+ lockSessionIDPassed.push_back(lockSessionIDArg);
+ },
+ retLockDoc);
+
+ {
+ auto lockStatus = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "test", "why", Milliseconds(0));
+ auto otherStatus = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "lock", "why", Milliseconds(0));
+ }
+
+ bool didTimeout = false;
+ {
+ stdx::unique_lock<Latch> lk(testMutex);
+
+ if (unlockIDMap.size() < 2 || !mapEntriesGreaterThanTwo(unlockIDMap)) {
+ didTimeout =
+ unlockCV.wait_for(lk, kJoinTimeout.toSystemDuration()) == stdx::cv_status::timeout;
+ }
+ }
+
+ // Join the background thread before trying to call asserts. Shutdown calls
+ // stopPing and we don't care in this test.
+ getMockCatalog()->expectStopPing([](StringData) {}, Status::OK());
+ DistLockManager::get(operationContext())->shutDown(operationContext());
+
+ // No assert until shutDown has been called to make sure that the background thread
+ // won't be trying to access the local variables that were captured by lamdas that
+ // may have gone out of scope when the assert unwinds the stack.
+ // No need to grab testMutex since there is only one thread running at this point.
+
+ ASSERT_FALSE(didTimeout);
+ ASSERT_EQUALS(2u, lockSessionIDPassed.size());
+
+ for (const auto& id : lockSessionIDPassed) {
+ ASSERT_GREATER_THAN(unlockIDMap[id], 2)
+ << "lockIDList: " << vectorToString(lockSessionIDPassed)
+ << ", map: " << mapToString(unlockIDMap);
+ }
+}
+
+TEST_F(DistLockManagerReplSetTest, CleanupPingOnShutdown) {
+ bool stopPingCalled = false;
+ getMockCatalog()->expectStopPing(
+ [this, &stopPingCalled](StringData processID) {
+ ASSERT_EQUALS(getProcessID(), processID);
+ stopPingCalled = true;
+ },
+ Status::OK());
+
+ DistLockManager::get(operationContext())->shutDown(operationContext());
+ ASSERT_TRUE(stopPingCalled);
+}
+
+TEST_F(DistLockManagerReplSetTest, CheckLockStatusOK) {
+ LocksType retLockDoc;
+ retLockDoc.setName("test");
+ retLockDoc.setState(LocksType::LOCKED);
+ retLockDoc.setProcess(getProcessID());
+ retLockDoc.setWho("me");
+ retLockDoc.setWhy("why");
+ // Will be different from the actual lock session id. For testing only.
+ retLockDoc.setLockID(OID::gen());
+
+ OID lockSessionID;
+
+ getMockCatalog()->expectGrabLock(
+ [&lockSessionID](StringData, const OID& ts, StringData, StringData, Date_t, StringData) {
+ lockSessionID = ts;
+ },
+ retLockDoc);
+
+
+ auto lockStatus = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "a", "", Milliseconds(0));
+ ASSERT_OK(lockStatus.getStatus());
+
+ getMockCatalog()->expectNoGrabLock();
+ getMockCatalog()->expectUnLock(
+ [](const OID&) {
+ // Don't care
+ },
+ Status::OK());
+
+ auto& scopedLock = lockStatus.getValue();
+
+ getMockCatalog()->expectNoGrabLock();
+ getMockCatalog()->expectGetLockByTS(
+ [&lockSessionID](const OID& ts) { ASSERT_EQUALS(lockSessionID, ts); }, retLockDoc);
+
+ ASSERT_OK(scopedLock.checkStatus());
+}
+
+TEST_F(DistLockManagerReplSetTest, CheckLockStatusNoLongerOwn) {
+ LocksType retLockDoc;
+ retLockDoc.setName("test");
+ retLockDoc.setState(LocksType::LOCKED);
+ retLockDoc.setProcess(getProcessID());
+ retLockDoc.setWho("me");
+ retLockDoc.setWhy("why");
+ // Will be different from the actual lock session id. For testing only.
+ retLockDoc.setLockID(OID::gen());
+
+ OID lockSessionID;
+
+ getMockCatalog()->expectGrabLock(
+ [&lockSessionID](StringData, const OID& ts, StringData, StringData, Date_t, StringData) {
+ lockSessionID = ts;
+ },
+ retLockDoc);
+
+
+ auto lockStatus = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "a", "", Milliseconds(0));
+ ASSERT_OK(lockStatus.getStatus());
+
+ getMockCatalog()->expectNoGrabLock();
+ getMockCatalog()->expectUnLock(
+ [](const OID&) {
+ // Don't care
+ },
+ Status::OK());
+
+ auto& scopedLock = lockStatus.getValue();
+
+ getMockCatalog()->expectNoGrabLock();
+ getMockCatalog()->expectGetLockByTS(
+ [&lockSessionID](const OID& ts) { ASSERT_EQUALS(lockSessionID, ts); },
+ {ErrorCodes::LockNotFound, "no lock"});
+
+ ASSERT_NOT_OK(scopedLock.checkStatus());
+}
+
+TEST_F(DistLockManagerReplSetTest, CheckLockStatusError) {
+ LocksType retLockDoc;
+ retLockDoc.setName("test");
+ retLockDoc.setState(LocksType::LOCKED);
+ retLockDoc.setProcess(getProcessID());
+ retLockDoc.setWho("me");
+ retLockDoc.setWhy("why");
+ // Will be different from the actual lock session id. For testing only.
+ retLockDoc.setLockID(OID::gen());
+
+ OID lockSessionID;
+
+ getMockCatalog()->expectGrabLock(
+ [&lockSessionID](StringData, const OID& ts, StringData, StringData, Date_t, StringData) {
+ lockSessionID = ts;
+ },
+ retLockDoc);
+
+
+ auto lockStatus = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "a", "", Milliseconds(0));
+ ASSERT_OK(lockStatus.getStatus());
+
+ getMockCatalog()->expectNoGrabLock();
+ getMockCatalog()->expectUnLock(
+ [](const OID&) {
+ // Don't care
+ },
+ Status::OK());
+
+ auto& scopedLock = lockStatus.getValue();
+
+ getMockCatalog()->expectNoGrabLock();
+ getMockCatalog()->expectGetLockByTS(
+ [&lockSessionID](const OID& ts) { ASSERT_EQUALS(lockSessionID, ts); },
+ {ErrorCodes::NetworkTimeout, "bad test network"});
+
+ ASSERT_NOT_OK(scopedLock.checkStatus());
+}
+
+/**
+ * Test scenario:
+ * 1. Attempt to grab lock fails because lock is already owned.
+ * 2. Try to get ping data and config server clock.
+ * 3. Since we don't have previous ping data to compare with, we cannot
+ * decide whether it's ok to overtake, so we can't.
+ * 4. Lock expiration has elapsed and the ping has not been updated since.
+ * 5. 2nd attempt to grab lock still fails for the same reason.
+ * 6. But since the ping is not fresh anymore, dist lock manager should overtake lock.
+ */
+TEST_F(DistLockManagerReplSetTest, LockOvertakingAfterLockExpiration) {
+ OID lastTS;
+
+ getMockCatalog()->expectGrabLock(
+ [&lastTS](
+ StringData, const OID& lockSessionID, StringData, StringData, Date_t, StringData) {
+ lastTS = lockSessionID;
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ LocksType currentLockDoc;
+ currentLockDoc.setName("bar");
+ currentLockDoc.setState(LocksType::LOCKED);
+ currentLockDoc.setProcess("otherProcess");
+ currentLockDoc.setLockID(OID("5572007fda9e476582bf3716"));
+ currentLockDoc.setWho("me");
+ currentLockDoc.setWhy("why");
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("bar", name); },
+ currentLockDoc);
+
+ LockpingsType pingDoc;
+ pingDoc.setProcess("otherProcess");
+ pingDoc.setPing(Date_t());
+
+ getMockCatalog()->expectGetPing(
+ [](StringData process) { ASSERT_EQUALS("otherProcess", process); }, pingDoc);
+
+ getMockCatalog()->expectGetServerInfo([]() {}, DistLockCatalog::ServerInfo(Date_t(), OID()));
+
+ // First attempt will record the ping data.
+ {
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::LockBusy, status.code());
+ }
+
+ // Advance config server time to exceed lock expiration.
+ getMockCatalog()->expectGetServerInfo(
+ []() {}, DistLockCatalog::ServerInfo(Date_t() + kLockExpiration + Milliseconds(1), OID()));
+
+ getMockCatalog()->expectOvertakeLock(
+ [this, &lastTS, &currentLockDoc](StringData lockID,
+ const OID& lockSessionID,
+ const OID& currentHolderTS,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ ASSERT_EQUALS("bar", lockID);
+ ASSERT_EQUALS(lastTS, lockSessionID);
+ ASSERT_EQUALS(currentLockDoc.getLockID(), currentHolderTS);
+ ASSERT_EQUALS(getProcessID(), processId);
+ ASSERT_EQUALS("foo", why);
+ },
+ currentLockDoc); // return arbitrary valid lock document, for testing purposes only.
+
+ int unlockCallCount = 0;
+ OID unlockSessionIDPassed;
+
+ // Second attempt should overtake lock.
+ {
+ auto lockStatus = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "foo", Milliseconds(0));
+
+ ASSERT_OK(lockStatus.getStatus());
+
+ getMockCatalog()->expectNoGrabLock();
+ getMockCatalog()->expectUnLock(
+ [&unlockCallCount, &unlockSessionIDPassed](const OID& lockSessionID) {
+ unlockCallCount++;
+ unlockSessionIDPassed = lockSessionID;
+ },
+ Status::OK());
+ }
+
+ ASSERT_EQUALS(1, unlockCallCount);
+ ASSERT_EQUALS(lastTS, unlockSessionIDPassed);
+}
+
+/**
+ * Test scenario:
+ * 1. Attempt to grab lock with lockSessionID fails because lock is already owned.
+ * 2. Then the the lock is overtaken because the lockSessionID matches the lock owner.
+ */
+TEST_F(DistLockManagerReplSetTest, LockOvertakingWithSessionID) {
+ OID passedLockSessionID("5572007fda9e476582bf3716");
+
+ LocksType currentLockDoc;
+ currentLockDoc.setName("bar");
+ currentLockDoc.setState(LocksType::LOCKED);
+ currentLockDoc.setProcess("otherProcess");
+ currentLockDoc.setLockID(passedLockSessionID);
+ currentLockDoc.setWho("me");
+ currentLockDoc.setWhy("why");
+
+ getMockCatalog()->expectGrabLock(
+ [&passedLockSessionID, &currentLockDoc](
+ StringData, const OID& lockSessionID, StringData, StringData, Date_t, StringData) {
+ ASSERT_EQUALS(passedLockSessionID, lockSessionID);
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("bar", name); },
+ currentLockDoc);
+
+ LockpingsType pingDoc;
+ pingDoc.setProcess("otherProcess");
+ pingDoc.setPing(Date_t());
+
+ getMockCatalog()->expectGetPing(
+ [](StringData process) { ASSERT_EQUALS("otherProcess", process); }, pingDoc);
+
+ getMockCatalog()->expectGetServerInfo([]() {}, DistLockCatalog::ServerInfo(Date_t(), OID()));
+
+ getMockCatalog()->expectOvertakeLock(
+ [this, &passedLockSessionID, &currentLockDoc](StringData lockID,
+ const OID& lockSessionID,
+ const OID& currentHolderTS,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ ASSERT_EQUALS("bar", lockID);
+ ASSERT_EQUALS(passedLockSessionID, lockSessionID);
+ ASSERT_EQUALS(currentLockDoc.getLockID(), currentHolderTS);
+ ASSERT_EQUALS(getProcessID(), processId);
+ ASSERT_EQUALS("foo", why);
+ },
+ currentLockDoc);
+
+ auto distLockHandleStatus =
+ DistLockManager::get(operationContext())
+ ->lockWithSessionID(
+ operationContext(), "bar", "foo", passedLockSessionID, Milliseconds(0));
+ ASSERT_OK(distLockHandleStatus.getStatus());
+
+ getMockCatalog()->expectNoGrabLock();
+}
+
+TEST_F(DistLockManagerReplSetTest, CannotOvertakeIfExpirationHasNotElapsed) {
+ getMockCatalog()->expectGrabLock(
+ [](StringData, const OID&, StringData, StringData, Date_t, StringData) {
+ // Don't care.
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ LocksType currentLockDoc;
+ currentLockDoc.setName("bar");
+ currentLockDoc.setState(LocksType::LOCKED);
+ currentLockDoc.setProcess("otherProcess");
+ currentLockDoc.setLockID(OID("5572007fda9e476582bf3716"));
+ currentLockDoc.setWho("me");
+ currentLockDoc.setWhy("why");
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("bar", name); },
+ currentLockDoc);
+
+ LockpingsType pingDoc;
+ pingDoc.setProcess("otherProcess");
+ pingDoc.setPing(Date_t());
+
+ getMockCatalog()->expectGetPing(
+ [](StringData process) { ASSERT_EQUALS("otherProcess", process); }, pingDoc);
+
+ getMockCatalog()->expectGetServerInfo([]() {}, DistLockCatalog::ServerInfo(Date_t(), OID()));
+
+ // First attempt will record the ping data.
+ {
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::LockBusy, status.code());
+ }
+
+ // Advance config server time to 1 millisecond before lock expiration.
+ getMockCatalog()->expectGetServerInfo(
+ []() {}, DistLockCatalog::ServerInfo(Date_t() + kLockExpiration - Milliseconds(1), OID()));
+
+ // Second attempt should still not overtake lock.
+ {
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::LockBusy, status.code());
+ }
+}
+
+TEST_F(DistLockManagerReplSetTest, GetPingErrorWhileOvertaking) {
+ getMockCatalog()->expectGrabLock(
+ [](StringData, const OID&, StringData, StringData, Date_t, StringData) {
+ // Don't care
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ LocksType currentLockDoc;
+ currentLockDoc.setName("bar");
+ currentLockDoc.setState(LocksType::LOCKED);
+ currentLockDoc.setProcess("otherProcess");
+ currentLockDoc.setLockID(OID("5572007fda9e476582bf3716"));
+ currentLockDoc.setWho("me");
+ currentLockDoc.setWhy("why");
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("bar", name); },
+ currentLockDoc);
+
+ getMockCatalog()->expectGetPing(
+ [](StringData process) { ASSERT_EQUALS("otherProcess", process); },
+ {ErrorCodes::NetworkTimeout, "bad test network"});
+
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::NetworkTimeout, status.code());
+}
+
+TEST_F(DistLockManagerReplSetTest, GetInvalidPingDocumentWhileOvertaking) {
+ getMockCatalog()->expectGrabLock(
+ [](StringData, const OID&, StringData, StringData, Date_t, StringData) {
+ // Don't care
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ LocksType currentLockDoc;
+ currentLockDoc.setName("bar");
+ currentLockDoc.setState(LocksType::LOCKED);
+ currentLockDoc.setProcess("otherProcess");
+ currentLockDoc.setLockID(OID("5572007fda9e476582bf3716"));
+ currentLockDoc.setWho("me");
+ currentLockDoc.setWhy("why");
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("bar", name); },
+ currentLockDoc);
+
+ LockpingsType invalidPing;
+ getMockCatalog()->expectGetPing(
+ [](StringData process) { ASSERT_EQUALS("otherProcess", process); }, invalidPing);
+
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::UnsupportedFormat, status.code());
+}
+
+TEST_F(DistLockManagerReplSetTest, GetServerInfoErrorWhileOvertaking) {
+ getMockCatalog()->expectGrabLock(
+ [](StringData, const OID&, StringData, StringData, Date_t, StringData) {
+ // Don't care
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ LocksType currentLockDoc;
+ currentLockDoc.setName("bar");
+ currentLockDoc.setState(LocksType::LOCKED);
+ currentLockDoc.setProcess("otherProcess");
+ currentLockDoc.setLockID(OID("5572007fda9e476582bf3716"));
+ currentLockDoc.setWho("me");
+ currentLockDoc.setWhy("why");
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("bar", name); },
+ currentLockDoc);
+
+ LockpingsType pingDoc;
+ pingDoc.setProcess("otherProcess");
+ pingDoc.setPing(Date_t());
+
+ getMockCatalog()->expectGetPing(
+ [](StringData process) { ASSERT_EQUALS("otherProcess", process); }, pingDoc);
+
+ getMockCatalog()->expectGetServerInfo([]() {},
+ {ErrorCodes::NetworkTimeout, "bad test network"});
+
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::NetworkTimeout, status.code());
+}
+
+TEST_F(DistLockManagerReplSetTest, GetLockErrorWhileOvertaking) {
+ getMockCatalog()->expectGrabLock(
+ [](StringData, const OID&, StringData, StringData, Date_t, StringData) {
+ // Don't care
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("bar", name); },
+ {ErrorCodes::NetworkTimeout, "bad test network"});
+
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::NetworkTimeout, status.code());
+}
+
+TEST_F(DistLockManagerReplSetTest, GetLockDisappearedWhileOvertaking) {
+ getMockCatalog()->expectGrabLock(
+ [](StringData, const OID&, StringData, StringData, Date_t, StringData) {
+ // Don't care
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("bar", name); },
+ {ErrorCodes::LockNotFound, "disappeared!"});
+
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::LockBusy, status.code());
+}
+
+/**
+ * 1. Try to grab lock multiple times.
+ * 2. For each attempt, the ping is updated and the config server clock is advanced
+ * by increments of lock expiration duration.
+ * 3. All of the previous attempt should result in lock busy.
+ * 4. Try to grab lock again when the ping was not updated and lock expiration has elapsed.
+ */
+TEST_F(DistLockManagerReplSetTest, CannotOvertakeIfPingIsActive) {
+ getMockCatalog()->expectGrabLock(
+ [](StringData, const OID&, StringData, StringData, Date_t, StringData) {
+ // Don't care
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ LocksType currentLockDoc;
+ currentLockDoc.setName("bar");
+ currentLockDoc.setState(LocksType::LOCKED);
+ currentLockDoc.setProcess("otherProcess");
+ currentLockDoc.setLockID(OID("5572007fda9e476582bf3716"));
+ currentLockDoc.setWho("me");
+ currentLockDoc.setWhy("why");
+
+ Date_t currentPing;
+ LockpingsType pingDoc;
+ pingDoc.setProcess("otherProcess");
+
+ Date_t configServerLocalTime;
+ int getServerInfoCallCount = 0;
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("bar", name); },
+ currentLockDoc);
+
+ const int kLoopCount = 5;
+ for (int x = 0; x < kLoopCount; x++) {
+ // Advance config server time to reach lock expiration.
+ configServerLocalTime += kLockExpiration;
+
+ currentPing += Milliseconds(1);
+ pingDoc.setPing(currentPing);
+
+ getMockCatalog()->expectGetPing(
+ [](StringData process) { ASSERT_EQUALS("otherProcess", process); }, pingDoc);
+
+ getMockCatalog()->expectGetServerInfo(
+ [&getServerInfoCallCount]() { getServerInfoCallCount++; },
+ DistLockCatalog::ServerInfo(configServerLocalTime, OID()));
+
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::LockBusy, status.code());
+ }
+
+ ASSERT_EQUALS(kLoopCount, getServerInfoCallCount);
+
+ configServerLocalTime += kLockExpiration;
+ getMockCatalog()->expectGetServerInfo(
+ [&getServerInfoCallCount]() { getServerInfoCallCount++; },
+ DistLockCatalog::ServerInfo(configServerLocalTime, OID()));
+
+ OID lockTS;
+ // Make sure that overtake is now ok since ping is no longer updated.
+ getMockCatalog()->expectOvertakeLock(
+ [this, &lockTS, &currentLockDoc](StringData lockID,
+ const OID& lockSessionID,
+ const OID& currentHolderTS,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ ASSERT_EQUALS("bar", lockID);
+ lockTS = lockSessionID;
+ ASSERT_EQUALS(currentLockDoc.getLockID(), currentHolderTS);
+ ASSERT_EQUALS(getProcessID(), processId);
+ ASSERT_EQUALS("foo", why);
+ },
+ currentLockDoc); // return arbitrary valid lock document, for testing purposes only.
+
+ int unlockCallCount = 0;
+ OID unlockSessionIDPassed;
+
+ {
+ auto lockStatus = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "foo", Milliseconds(0));
+
+ ASSERT_OK(lockStatus.getStatus());
+
+ getMockCatalog()->expectNoGrabLock();
+ getMockCatalog()->expectUnLock(
+ [&unlockCallCount, &unlockSessionIDPassed](const OID& lockSessionID) {
+ unlockCallCount++;
+ unlockSessionIDPassed = lockSessionID;
+ },
+ Status::OK());
+ }
+
+ ASSERT_EQUALS(1, unlockCallCount);
+ ASSERT_EQUALS(lockTS, unlockSessionIDPassed);
+}
+
+/**
+ * 1. Try to grab lock multiple times.
+ * 2. For each attempt, the owner of the lock is different and the config server clock is
+ * advanced by increments of lock expiration duration.
+ * 3. All of the previous attempt should result in lock busy.
+ * 4. Try to grab lock again when the ping was not updated and lock expiration has elapsed.
+ */
+TEST_F(DistLockManagerReplSetTest, CannotOvertakeIfOwnerJustChanged) {
+ getMockCatalog()->expectGrabLock(
+ [](StringData, const OID&, StringData, StringData, Date_t, StringData) {
+ // Don't care
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ LocksType currentLockDoc;
+ currentLockDoc.setName("bar");
+ currentLockDoc.setState(LocksType::LOCKED);
+ currentLockDoc.setProcess("otherProcess");
+ currentLockDoc.setLockID(OID("5572007fda9e476582bf3716"));
+ currentLockDoc.setWho("me");
+ currentLockDoc.setWhy("why");
+
+ LockpingsType pingDoc;
+ pingDoc.setProcess("otherProcess");
+ pingDoc.setPing(Date_t());
+
+ Date_t configServerLocalTime;
+ int getServerInfoCallCount = 0;
+
+ getMockCatalog()->expectGetPing(
+ [](StringData process) { ASSERT_EQUALS("otherProcess", process); }, pingDoc);
+
+ const int kLoopCount = 5;
+ for (int x = 0; x < kLoopCount; x++) {
+ // Advance config server time to reach lock expiration.
+ configServerLocalTime += kLockExpiration;
+
+ currentLockDoc.setLockID(OID::gen());
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("bar", name); },
+ currentLockDoc);
+
+ getMockCatalog()->expectGetServerInfo(
+ [&getServerInfoCallCount]() { getServerInfoCallCount++; },
+ DistLockCatalog::ServerInfo(configServerLocalTime, OID()));
+
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::LockBusy, status.code());
+ }
+
+ ASSERT_EQUALS(kLoopCount, getServerInfoCallCount);
+
+ configServerLocalTime += kLockExpiration;
+ getMockCatalog()->expectGetServerInfo(
+ [&getServerInfoCallCount]() { getServerInfoCallCount++; },
+ DistLockCatalog::ServerInfo(configServerLocalTime, OID()));
+
+ OID lockTS;
+ // Make sure that overtake is now ok since lock owner didn't change.
+ getMockCatalog()->expectOvertakeLock(
+ [this, &lockTS, &currentLockDoc](StringData lockID,
+ const OID& lockSessionID,
+ const OID& currentHolderTS,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ ASSERT_EQUALS("bar", lockID);
+ lockTS = lockSessionID;
+ ASSERT_EQUALS(currentLockDoc.getLockID(), currentHolderTS);
+ ASSERT_EQUALS(getProcessID(), processId);
+ ASSERT_EQUALS("foo", why);
+ },
+ currentLockDoc); // return arbitrary valid lock document, for testing purposes only.
+
+ int unlockCallCount = 0;
+ OID unlockSessionIDPassed;
+
+ {
+ auto lockStatus = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "foo", Milliseconds(0));
+
+ ASSERT_OK(lockStatus.getStatus());
+
+ getMockCatalog()->expectNoGrabLock();
+ getMockCatalog()->expectUnLock(
+ [&unlockCallCount, &unlockSessionIDPassed](const OID& lockSessionID) {
+ unlockCallCount++;
+ unlockSessionIDPassed = lockSessionID;
+ },
+ Status::OK());
+ }
+
+ ASSERT_EQUALS(1, unlockCallCount);
+ ASSERT_EQUALS(lockTS, unlockSessionIDPassed);
+}
+
+/**
+ * 1. Try to grab lock multiple times.
+ * 2. For each attempt, the electionId of the config server is different and the
+ * config server clock is advanced by increments of lock expiration duration.
+ * 3. All of the previous attempt should result in lock busy.
+ * 4. Try to grab lock again when the ping was not updated and lock expiration has elapsed.
+ */
+TEST_F(DistLockManagerReplSetTest, CannotOvertakeIfElectionIdChanged) {
+ getMockCatalog()->expectGrabLock(
+ [](StringData, const OID&, StringData, StringData, Date_t, StringData) {
+ // Don't care
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ LocksType currentLockDoc;
+ currentLockDoc.setName("bar");
+ currentLockDoc.setState(LocksType::LOCKED);
+ currentLockDoc.setProcess("otherProcess");
+ currentLockDoc.setLockID(OID("5572007fda9e476582bf3716"));
+ currentLockDoc.setWho("me");
+ currentLockDoc.setWhy("why");
+
+ LockpingsType pingDoc;
+ pingDoc.setProcess("otherProcess");
+ pingDoc.setPing(Date_t());
+
+ Date_t configServerLocalTime;
+ int getServerInfoCallCount = 0;
+
+ const LocksType& fixedLockDoc = currentLockDoc;
+ const LockpingsType& fixedPingDoc = pingDoc;
+
+ const int kLoopCount = 5;
+ OID lastElectionId;
+ for (int x = 0; x < kLoopCount; x++) {
+ // Advance config server time to reach lock expiration.
+ configServerLocalTime += kLockExpiration;
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("bar", name); },
+ fixedLockDoc);
+
+ getMockCatalog()->expectGetPing(
+ [](StringData process) { ASSERT_EQUALS("otherProcess", process); }, fixedPingDoc);
+
+ lastElectionId = OID::gen();
+ getMockCatalog()->expectGetServerInfo(
+ [&getServerInfoCallCount]() { getServerInfoCallCount++; },
+ DistLockCatalog::ServerInfo(configServerLocalTime, lastElectionId));
+
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::LockBusy, status.code());
+ }
+
+ ASSERT_EQUALS(kLoopCount, getServerInfoCallCount);
+
+ configServerLocalTime += kLockExpiration;
+ getMockCatalog()->expectGetServerInfo(
+ [&getServerInfoCallCount]() { getServerInfoCallCount++; },
+ DistLockCatalog::ServerInfo(configServerLocalTime, lastElectionId));
+
+ OID lockTS;
+ // Make sure that overtake is now ok since electionId didn't change.
+ getMockCatalog()->expectOvertakeLock(
+ [this, &lockTS, &currentLockDoc](StringData lockID,
+ const OID& lockSessionID,
+ const OID& currentHolderTS,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ ASSERT_EQUALS("bar", lockID);
+ lockTS = lockSessionID;
+ ASSERT_EQUALS(currentLockDoc.getLockID(), currentHolderTS);
+ ASSERT_EQUALS(getProcessID(), processId);
+ ASSERT_EQUALS("foo", why);
+ },
+ currentLockDoc); // return arbitrary valid lock document, for testing purposes only.
+
+ int unlockCallCount = 0;
+ OID unlockSessionIDPassed;
+
+ {
+ auto lockStatus = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "foo", Milliseconds(0));
+
+ ASSERT_OK(lockStatus.getStatus());
+
+ getMockCatalog()->expectNoGrabLock();
+ getMockCatalog()->expectUnLock(
+ [&unlockCallCount, &unlockSessionIDPassed](const OID& lockSessionID) {
+ unlockCallCount++;
+ unlockSessionIDPassed = lockSessionID;
+ },
+ Status::OK());
+ }
+
+ ASSERT_EQUALS(1, unlockCallCount);
+ ASSERT_EQUALS(lockTS, unlockSessionIDPassed);
+}
+
+/**
+ * 1. Try to grab lock multiple times.
+ * 2. For each attempt, attempting to check the ping document results in NotWritablePrimary error.
+ * 3. All of the previous attempt should result in lock busy.
+ * 4. Try to grab lock again when the ping was not updated and lock expiration has elapsed.
+ */
+TEST_F(DistLockManagerReplSetTest, CannotOvertakeIfNoMaster) {
+ getMockCatalog()->expectGrabLock(
+ [](StringData, const OID&, StringData, StringData, Date_t, StringData) {
+ // Don't care
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ LocksType currentLockDoc;
+ currentLockDoc.setName("bar");
+ currentLockDoc.setState(LocksType::LOCKED);
+ currentLockDoc.setProcess("otherProcess");
+ currentLockDoc.setLockID(OID("5572007fda9e476582bf3716"));
+ currentLockDoc.setWho("me");
+ currentLockDoc.setWhy("why");
+
+ LockpingsType pingDoc;
+ pingDoc.setProcess("otherProcess");
+ pingDoc.setPing(Date_t());
+
+ int getServerInfoCallCount = 0;
+
+ const LocksType& fixedLockDoc = currentLockDoc;
+ const LockpingsType& fixedPingDoc = pingDoc;
+
+ Date_t configServerLocalTime;
+ const int kLoopCount = 4;
+ OID lastElectionId;
+ for (int x = 0; x < kLoopCount; x++) {
+ configServerLocalTime += kLockExpiration;
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("bar", name); },
+ fixedLockDoc);
+
+ getMockCatalog()->expectGetPing(
+ [](StringData process) { ASSERT_EQUALS("otherProcess", process); }, fixedPingDoc);
+
+ if (x == 0) {
+ // initialize internal ping history first.
+ lastElectionId = OID::gen();
+ getMockCatalog()->expectGetServerInfo(
+ [&getServerInfoCallCount]() { getServerInfoCallCount++; },
+ DistLockCatalog::ServerInfo(configServerLocalTime, lastElectionId));
+ } else {
+ getMockCatalog()->expectGetServerInfo(
+ [&getServerInfoCallCount]() { getServerInfoCallCount++; },
+ {ErrorCodes::NotWritablePrimary, "not master"});
+ }
+
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::LockBusy, status.code());
+ }
+
+ ASSERT_EQUALS(kLoopCount, getServerInfoCallCount);
+
+ getMockCatalog()->expectGetServerInfo(
+ [&getServerInfoCallCount]() { getServerInfoCallCount++; },
+ DistLockCatalog::ServerInfo(configServerLocalTime, lastElectionId));
+
+ OID lockTS;
+ // Make sure that overtake is now ok since electionId didn't change.
+ getMockCatalog()->expectOvertakeLock(
+ [this, &lockTS, &currentLockDoc](StringData lockID,
+ const OID& lockSessionID,
+ const OID& currentHolderTS,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ ASSERT_EQUALS("bar", lockID);
+ lockTS = lockSessionID;
+ ASSERT_EQUALS(currentLockDoc.getLockID(), currentHolderTS);
+ ASSERT_EQUALS(getProcessID(), processId);
+ ASSERT_EQUALS("foo", why);
+ },
+ currentLockDoc); // return arbitrary valid lock document, for testing purposes only.
+
+ int unlockCallCount = 0;
+ OID unlockSessionIDPassed;
+
+ {
+ auto lockStatus = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "foo", Milliseconds(0));
+
+ ASSERT_OK(lockStatus.getStatus());
+
+ getMockCatalog()->expectNoGrabLock();
+ getMockCatalog()->expectUnLock(
+ [&unlockCallCount, &unlockSessionIDPassed](const OID& lockSessionID) {
+ unlockCallCount++;
+ unlockSessionIDPassed = lockSessionID;
+ },
+ Status::OK());
+ }
+
+ ASSERT_EQUALS(1, unlockCallCount);
+ ASSERT_EQUALS(lockTS, unlockSessionIDPassed);
+}
+
+/**
+ * Test scenario:
+ * 1. Attempt to grab lock fails because lock is already owned.
+ * 2. Try to get ping data and config server clock.
+ * 3. Since we don't have previous ping data to compare with, we cannot
+ * decide whether it's ok to overtake, so we can't.
+ * 4. Lock expiration has elapsed and the ping has not been updated since.
+ * 5. 2nd attempt to grab lock still fails for the same reason.
+ * 6. But since the ping is not fresh anymore, dist lock manager should overtake lock.
+ * 7. Attempt to overtake resulted in an error.
+ * 8. Check that unlock was called.
+ */
+TEST_F(DistLockManagerReplSetTest, LockOvertakingResultsInError) {
+ getMockCatalog()->expectGrabLock(
+ [](StringData, const OID&, StringData, StringData, Date_t, StringData) {
+ // Don't care
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ LocksType currentLockDoc;
+ currentLockDoc.setName("bar");
+ currentLockDoc.setState(LocksType::LOCKED);
+ currentLockDoc.setProcess("otherProcess");
+ currentLockDoc.setLockID(OID("5572007fda9e476582bf3716"));
+ currentLockDoc.setWho("me");
+ currentLockDoc.setWhy("why");
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("bar", name); },
+ currentLockDoc);
+
+ LockpingsType pingDoc;
+ pingDoc.setProcess("otherProcess");
+ pingDoc.setPing(Date_t());
+
+ getMockCatalog()->expectGetPing(
+ [](StringData process) { ASSERT_EQUALS("otherProcess", process); }, pingDoc);
+
+ getMockCatalog()->expectGetServerInfo([]() {}, DistLockCatalog::ServerInfo(Date_t(), OID()));
+
+ // First attempt will record the ping data.
+ {
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::LockBusy, status.code());
+ }
+
+ // Advance config server time to exceed lock expiration.
+ getMockCatalog()->expectGetServerInfo(
+ []() {}, DistLockCatalog::ServerInfo(Date_t() + kLockExpiration + Milliseconds(1), OID()));
+
+ OID lastTS;
+ getMockCatalog()->expectOvertakeLock(
+ [this, &lastTS, &currentLockDoc](StringData lockID,
+ const OID& lockSessionID,
+ const OID& currentHolderTS,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ ASSERT_EQUALS("bar", lockID);
+ lastTS = lockSessionID;
+ ASSERT_EQUALS(currentLockDoc.getLockID(), currentHolderTS);
+ ASSERT_EQUALS(getProcessID(), processId);
+ ASSERT_EQUALS("foo", why);
+ },
+ {ErrorCodes::NetworkTimeout, "bad test network"});
+
+ OID unlockSessionIDPassed;
+
+ auto unlockMutex = MONGO_MAKE_LATCH();
+ stdx::condition_variable unlockCV;
+ getMockCatalog()->expectUnLock(
+ [&unlockSessionIDPassed, &unlockMutex, &unlockCV](const OID& lockSessionID) {
+ stdx::unique_lock<Latch> lk(unlockMutex);
+ unlockSessionIDPassed = lockSessionID;
+ unlockCV.notify_all();
+ },
+ Status::OK());
+
+ // Second attempt should overtake lock.
+ auto lockStatus = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "foo", Milliseconds(0));
+
+ ASSERT_NOT_OK(lockStatus.getStatus());
+
+ bool didTimeout = false;
+ {
+ stdx::unique_lock<Latch> lk(unlockMutex);
+ if (!unlockSessionIDPassed.isSet()) {
+ didTimeout =
+ unlockCV.wait_for(lk, kJoinTimeout.toSystemDuration()) == stdx::cv_status::timeout;
+ }
+ }
+
+ // Join the background thread before trying to call asserts. Shutdown calls
+ // stopPing and we don't care in this test.
+ getMockCatalog()->expectStopPing([](StringData) {}, Status::OK());
+ DistLockManager::get(operationContext())->shutDown(operationContext());
+
+ // No assert until shutDown has been called to make sure that the background thread
+ // won't be trying to access the local variables that were captured by lamdas that
+ // may have gone out of scope when the assert unwinds the stack.
+ // No need to grab testMutex since there is only one thread running at this point.
+
+ ASSERT_FALSE(didTimeout);
+ ASSERT_EQUALS(lastTS, unlockSessionIDPassed);
+}
+
+/**
+ * Test scenario:
+ * 1. Attempt to grab lock fails because lock is already owned.
+ * 2. Try to get ping data and config server clock.
+ * 3. Since we don't have previous ping data to compare with, we cannot
+ * decide whether it's ok to overtake, so we can't.
+ * 4. Lock expiration has elapsed and the ping has not been updated since.
+ * 5. 2nd attempt to grab lock still fails for the same reason.
+ * 6. But since the ping is not fresh anymore, dist lock manager should overtake lock.
+ * 7. Attempt to overtake resulted failed because someone beat us into it.
+ */
+TEST_F(DistLockManagerReplSetTest, LockOvertakingFailed) {
+ getMockCatalog()->expectGrabLock(
+ [](StringData, const OID&, StringData, StringData, Date_t, StringData) {
+ // Don't care
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ LocksType currentLockDoc;
+ currentLockDoc.setName("bar");
+ currentLockDoc.setState(LocksType::LOCKED);
+ currentLockDoc.setProcess("otherProcess");
+ currentLockDoc.setLockID(OID("5572007fda9e476582bf3716"));
+ currentLockDoc.setWho("me");
+ currentLockDoc.setWhy("why");
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("bar", name); },
+ currentLockDoc);
+
+ LockpingsType pingDoc;
+ pingDoc.setProcess("otherProcess");
+ pingDoc.setPing(Date_t());
+
+ getMockCatalog()->expectGetPing(
+ [](StringData process) { ASSERT_EQUALS("otherProcess", process); }, pingDoc);
+
+ getMockCatalog()->expectGetServerInfo([]() {}, DistLockCatalog::ServerInfo(Date_t(), OID()));
+
+ // First attempt will record the ping data.
+ {
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::LockBusy, status.code());
+ }
+
+ // Advance config server time to exceed lock expiration.
+ getMockCatalog()->expectGetServerInfo(
+ []() {}, DistLockCatalog::ServerInfo(Date_t() + kLockExpiration + Milliseconds(1), OID()));
+
+ // Second attempt should overtake lock.
+ getMockCatalog()->expectOvertakeLock(
+ [this, &currentLockDoc](StringData lockID,
+ const OID& lockSessionID,
+ const OID& currentHolderTS,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ ASSERT_EQUALS("bar", lockID);
+ ASSERT_EQUALS(currentLockDoc.getLockID(), currentHolderTS);
+ ASSERT_EQUALS(getProcessID(), processId);
+ ASSERT_EQUALS("foo", why);
+ },
+ {ErrorCodes::LockStateChangeFailed, "nmod 0"});
+
+ {
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "foo", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::LockBusy, status.code());
+ }
+}
+
+/**
+ * Test scenario:
+ * 1. Attempt to grab lock fails because lock is already owned.
+ * 2. Try to get ping data and config server clock.
+ * 3. Since we don't have previous ping data to compare with, we cannot
+ * decide whether it's ok to overtake, so we can't.
+ * 4. Lock expiration has elapsed and the ping has not been updated since.
+ * 5. 2nd attempt to grab lock still fails for the same reason.
+ * 6. But since the ping is not fresh anymore, dist lock manager should overtake lock.
+ * 7. Attempt to overtake resulted failed because someone beat us into it.
+ */
+TEST_F(DistLockManagerReplSetTest, CannotOvertakeIfConfigServerClockGoesBackwards) {
+ getMockCatalog()->expectGrabLock(
+ [](StringData, const OID&, StringData, StringData, Date_t, StringData) {
+ // Don't care
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ LocksType currentLockDoc;
+ currentLockDoc.setName("bar");
+ currentLockDoc.setState(LocksType::LOCKED);
+ currentLockDoc.setProcess("otherProcess");
+ currentLockDoc.setLockID(OID("5572007fda9e476582bf3716"));
+ currentLockDoc.setWho("me");
+ currentLockDoc.setWhy("why");
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("bar", name); },
+ currentLockDoc);
+
+ LockpingsType pingDoc;
+ pingDoc.setProcess("otherProcess");
+ pingDoc.setPing(Date_t());
+
+ getMockCatalog()->expectGetPing(
+ [](StringData process) { ASSERT_EQUALS("otherProcess", process); }, pingDoc);
+
+ Date_t configClock(Date_t::now());
+ getMockCatalog()->expectGetServerInfo([]() {}, DistLockCatalog::ServerInfo(configClock, OID()));
+
+ // First attempt will record the ping data.
+ {
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::LockBusy, status.code());
+ }
+
+ // Make config server time go backwards by lock expiration duration.
+ getMockCatalog()->expectGetServerInfo(
+ []() {},
+ DistLockCatalog::ServerInfo(configClock - kLockExpiration - Milliseconds(1), OID()));
+
+ // Second attempt should not overtake lock.
+ {
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "foo", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::LockBusy, status.code());
+ }
+}
+
+TEST_F(DistLockManagerReplSetTest, LockAcquisitionRetriesOnNetworkErrorSuccess) {
+ getMockCatalog()->expectGrabLock(
+ [&](StringData, const OID&, StringData, StringData, Date_t, StringData) {
+ // Next acquisition should be successful
+ LocksType currentLockDoc;
+ currentLockDoc.setName("LockName");
+ currentLockDoc.setState(LocksType::LOCKED);
+ currentLockDoc.setProcess("otherProcess");
+ currentLockDoc.setLockID(OID("5572007fda9e476582bf3716"));
+ currentLockDoc.setWho("me");
+ currentLockDoc.setWhy("Lock reason");
+
+ getMockCatalog()->expectGrabLock(
+ [&](StringData, const OID&, StringData, StringData, Date_t, StringData) {},
+ currentLockDoc);
+ },
+ {ErrorCodes::NetworkTimeout, "network error"});
+
+ getMockCatalog()->expectUnLock([&](const OID& lockSessionID) {}, Status::OK());
+
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "LockName", "Lock reason", Milliseconds(0))
+ .getStatus();
+ ASSERT_OK(status);
+}
+
+TEST_F(DistLockManagerReplSetTest, LockAcquisitionRetriesOnInterruptionNeverSucceeds) {
+ getMockCatalog()->expectGrabLock(
+ [&](StringData, const OID&, StringData, StringData, Date_t, StringData) {},
+ {ErrorCodes::Interrupted, "operation interrupted"});
+
+ getMockCatalog()->expectUnLock([&](const OID& lockSessionID) {}, Status::OK());
+
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "foo", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+}
+
+class RSDistLockMgrWithMockTickSource : public DistLockManagerReplSetTest {
+protected:
+ RSDistLockMgrWithMockTickSource() {
+ getServiceContext()->setTickSource(std::make_unique<TickSourceMock<>>());
+ }
+
+ /**
+ * Returns the mock tick source.
+ */
+ TickSourceMock<>* getMockTickSource() {
+ return dynamic_cast<TickSourceMock<>*>(getServiceContext()->getTickSource());
+ }
+};
+
+/**
+ * Test scenario:
+ * 1. Grab lock fails up to 3 times.
+ * 2. Check that each subsequent attempt uses the same lock session id.
+ * 3. Unlock (on destructor of ScopedDistLock).
+ * 4. Check lock id used in lock and unlock are the same.
+ */
+TEST_F(RSDistLockMgrWithMockTickSource, LockSuccessAfterRetry) {
+ std::string lockName("test");
+ std::string me("me");
+ boost::optional<OID> lastTS;
+ Date_t lastTime(Date_t::now());
+ std::string whyMsg("because");
+
+ int retryAttempt = 0;
+ const int kMaxRetryAttempt = 3;
+
+ LocksType goodLockDoc;
+ goodLockDoc.setName(lockName);
+ goodLockDoc.setState(LocksType::LOCKED);
+ goodLockDoc.setProcess(getProcessID());
+ goodLockDoc.setWho("me");
+ goodLockDoc.setWhy(whyMsg);
+ goodLockDoc.setLockID(OID::gen());
+
+ getMockCatalog()->expectGrabLock(
+ [this,
+ &lockName,
+ &lastTS,
+ &me,
+ &lastTime,
+ &whyMsg,
+ &retryAttempt,
+ &kMaxRetryAttempt,
+ &goodLockDoc](StringData lockID,
+ const OID& lockSessionID,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ ASSERT_EQUALS(lockName, lockID);
+ // Lock session ID should be the same after first attempt.
+ if (lastTS) {
+ ASSERT_EQUALS(*lastTS, lockSessionID);
+ }
+ ASSERT_EQUALS(getProcessID(), processId);
+ ASSERT_GREATER_THAN_OR_EQUALS(time, lastTime);
+ ASSERT_EQUALS(whyMsg, why);
+
+ lastTS = lockSessionID;
+ lastTime = time;
+
+ getMockTickSource()->advance(Milliseconds(1));
+
+ if (++retryAttempt >= kMaxRetryAttempt) {
+ getMockCatalog()->expectGrabLock(
+ [this, &lockName, &lastTS, &me, &lastTime, &whyMsg](StringData lockID,
+ const OID& lockSessionID,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ ASSERT_EQUALS(lockName, lockID);
+ // Lock session ID should be the same after first attempt.
+ if (lastTS) {
+ ASSERT_EQUALS(*lastTS, lockSessionID);
+ }
+ ASSERT_TRUE(lockSessionID.isSet());
+ ASSERT_EQUALS(getProcessID(), processId);
+ ASSERT_GREATER_THAN_OR_EQUALS(time, lastTime);
+ ASSERT_EQUALS(whyMsg, why);
+
+ getMockCatalog()->expectNoGrabLock();
+
+ getMockCatalog()->expectGetLockByName(
+ [](StringData name) {
+ FAIL("should not attempt to overtake lock after successful lock");
+ },
+ LocksType());
+ },
+ goodLockDoc);
+ }
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ //
+ // Setup mock for lock overtaking.
+ //
+
+ LocksType currentLockDoc;
+ currentLockDoc.setName("test");
+ currentLockDoc.setState(LocksType::LOCKED);
+ currentLockDoc.setProcess("otherProcess");
+ currentLockDoc.setLockID(OID::gen());
+ currentLockDoc.setWho("me");
+ currentLockDoc.setWhy("why");
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("test", name); },
+ currentLockDoc);
+
+ LockpingsType pingDoc;
+ pingDoc.setProcess("otherProcess");
+ pingDoc.setPing(Date_t());
+
+ getMockCatalog()->expectGetPing(
+ [](StringData process) { ASSERT_EQUALS("otherProcess", process); }, pingDoc);
+
+ // Config server time is fixed, so overtaking will never succeed.
+ getMockCatalog()->expectGetServerInfo([]() {}, DistLockCatalog::ServerInfo(Date_t(), OID()));
+
+ //
+ // Try grabbing lock.
+ //
+
+ int unlockCallCount = 0;
+ OID unlockSessionIDPassed;
+
+ {
+ auto lockStatus = DistLockManager::get(operationContext())
+ ->lock(operationContext(), lockName, whyMsg, Milliseconds(10));
+ ASSERT_OK(lockStatus.getStatus());
+
+ getMockCatalog()->expectNoGrabLock();
+ getMockCatalog()->expectUnLock(
+ [&unlockCallCount, &unlockSessionIDPassed](const OID& lockSessionID) {
+ unlockCallCount++;
+ unlockSessionIDPassed = lockSessionID;
+ },
+ Status::OK());
+ }
+
+ ASSERT_EQUALS(1, unlockCallCount);
+ ASSERT(lastTS);
+ ASSERT_EQUALS(*lastTS, unlockSessionIDPassed);
+}
+
+/**
+ * Test scenario:
+ * 1. Grab lock fails up to 3 times.
+ * 2. Check that each subsequent attempt uses the same lock session id.
+ * 3. Grab lock errors out on the fourth try.
+ * 4. Make sure that unlock is called to cleanup the last lock attempted that error out.
+ */
+TEST_F(RSDistLockMgrWithMockTickSource, LockFailsAfterRetry) {
+ std::string lockName("test");
+ std::string me("me");
+ boost::optional<OID> lastTS;
+ Date_t lastTime(Date_t::now());
+ std::string whyMsg("because");
+
+ int retryAttempt = 0;
+ const int kMaxRetryAttempt = 3;
+
+ getMockCatalog()->expectGrabLock(
+ [this, &lockName, &lastTS, &me, &lastTime, &whyMsg, &retryAttempt, &kMaxRetryAttempt](
+ StringData lockID,
+ const OID& lockSessionID,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ ASSERT_EQUALS(lockName, lockID);
+ // Lock session ID should be the same after first attempt.
+ if (lastTS) {
+ ASSERT_EQUALS(*lastTS, lockSessionID);
+ }
+ ASSERT_EQUALS(getProcessID(), processId);
+ ASSERT_GREATER_THAN_OR_EQUALS(time, lastTime);
+ ASSERT_EQUALS(whyMsg, why);
+
+ lastTS = lockSessionID;
+ lastTime = time;
+
+ getMockTickSource()->advance(Milliseconds(1));
+
+ if (++retryAttempt >= kMaxRetryAttempt) {
+ getMockCatalog()->expectGrabLock(
+ [this, &lockName, &lastTS, &me, &lastTime, &whyMsg](StringData lockID,
+ const OID& lockSessionID,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ ASSERT_EQUALS(lockName, lockID);
+ // Lock session ID should be the same after first attempt.
+ if (lastTS) {
+ ASSERT_EQUALS(*lastTS, lockSessionID);
+ }
+ lastTS = lockSessionID;
+ ASSERT_TRUE(lockSessionID.isSet());
+ ASSERT_EQUALS(getProcessID(), processId);
+ ASSERT_GREATER_THAN_OR_EQUALS(time, lastTime);
+ ASSERT_EQUALS(whyMsg, why);
+
+ getMockCatalog()->expectNoGrabLock();
+ },
+ {ErrorCodes::ExceededMemoryLimit, "bad remote server"});
+ }
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ // Make mock return lock not found to skip lock overtaking.
+ getMockCatalog()->expectGetLockByName([](StringData) {},
+ {ErrorCodes::LockNotFound, "not found!"});
+
+ auto unlockMutex = MONGO_MAKE_LATCH();
+ stdx::condition_variable unlockCV;
+ OID unlockSessionIDPassed;
+ int unlockCallCount = 0;
+
+ getMockCatalog()->expectUnLock(
+ [&unlockMutex, &unlockCV, &unlockCallCount, &unlockSessionIDPassed](
+ const OID& lockSessionID) {
+ stdx::unique_lock<Latch> lk(unlockMutex);
+ unlockCallCount++;
+ unlockSessionIDPassed = lockSessionID;
+ unlockCV.notify_all();
+ },
+ Status::OK());
+
+ {
+ auto lockStatus = DistLockManager::get(operationContext())
+ ->lock(operationContext(), lockName, whyMsg, Milliseconds(10));
+ ASSERT_NOT_OK(lockStatus.getStatus());
+ }
+
+ bool didTimeout = false;
+ {
+ stdx::unique_lock<Latch> lk(unlockMutex);
+ if (unlockCallCount == 0) {
+ didTimeout =
+ unlockCV.wait_for(lk, kJoinTimeout.toSystemDuration()) == stdx::cv_status::timeout;
+ }
+ }
+
+ // Join the background thread before trying to call asserts. Shutdown calls
+ // stopPing and we don't care in this test.
+ getMockCatalog()->expectStopPing([](StringData) {}, Status::OK());
+ DistLockManager::get(operationContext())->shutDown(operationContext());
+
+ // No assert until shutDown has been called to make sure that the background thread
+ // won't be trying to access the local variables that were captured by lamdas that
+ // may have gone out of scope when the assert unwinds the stack.
+ // No need to grab unlockMutex since there is only one thread running at this point.
+
+ ASSERT_FALSE(didTimeout);
+ ASSERT_EQUALS(1, unlockCallCount);
+ ASSERT(lastTS);
+ ASSERT_EQUALS(*lastTS, unlockSessionIDPassed);
+}
+
+TEST_F(DistLockManagerReplSetTest, LockBusyNoRetry) {
+ getMockCatalog()->expectGrabLock(
+ [this](StringData, const OID&, StringData, StringData, Date_t, StringData) {
+ getMockCatalog()->expectNoGrabLock(); // Call only once.
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ // Make mock return lock not found to skip lock overtaking.
+ getMockCatalog()->expectGetLockByName([](StringData) {},
+ {ErrorCodes::LockNotFound, "not found!"});
+
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "", "", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::LockBusy, status.code());
+}
+
+/**
+ * Test scenario:
+ * 1. Attempt to grab lock.
+ * 2. Check that each subsequent attempt uses the same lock session id.
+ * 3. Times out trying.
+ * 4. Checks result is error.
+ * 5. Implicitly check that unlock is not called (default setting of mock catalog).
+ */
+TEST_F(RSDistLockMgrWithMockTickSource, LockRetryTimeout) {
+ std::string lockName("test");
+ std::string me("me");
+ boost::optional<OID> lastTS;
+ Date_t lastTime(Date_t::now());
+ std::string whyMsg("because");
+
+ int retryAttempt = 0;
+
+ getMockCatalog()->expectGrabLock(
+ [this, &lockName, &lastTS, &me, &lastTime, &whyMsg, &retryAttempt](StringData lockID,
+ const OID& lockSessionID,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ ASSERT_EQUALS(lockName, lockID);
+ // Lock session ID should be the same after first attempt.
+ if (lastTS) {
+ ASSERT_EQUALS(*lastTS, lockSessionID);
+ }
+ ASSERT_EQUALS(getProcessID(), processId);
+ ASSERT_GREATER_THAN_OR_EQUALS(time, lastTime);
+ ASSERT_EQUALS(whyMsg, why);
+
+ lastTS = lockSessionID;
+ lastTime = time;
+ retryAttempt++;
+
+ getMockTickSource()->advance(Milliseconds(1));
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ // Make mock return lock not found to skip lock overtaking.
+ getMockCatalog()->expectGetLockByName([](StringData) {},
+ {ErrorCodes::LockNotFound, "not found!"});
+
+ auto lockStatus = DistLockManager::get(operationContext())
+ ->lock(operationContext(), lockName, whyMsg, Milliseconds(5))
+ .getStatus();
+ ASSERT_NOT_OK(lockStatus);
+
+ ASSERT_EQUALS(ErrorCodes::LockBusy, lockStatus.code());
+ ASSERT_GREATER_THAN(retryAttempt, 1);
+}
+
+/**
+ * Test scenario:
+ * 1. Attempt to grab lock fails because lock is already owned.
+ * 2. Try to get ping data (does not exist) and config server clock.
+ * 3. Since we don't have previous ping data to compare with, we cannot
+ * decide whether it's ok to overtake, so we can't.
+ * 4. Lock expiration has elapsed and the ping still does not exist.
+ * 5. 2nd attempt to grab lock still fails for the same reason.
+ * 6. But since the ping has not been updated, dist lock manager should overtake lock.
+ */
+TEST_F(RSDistLockMgrWithMockTickSource, CanOvertakeIfNoPingDocument) {
+ getMockCatalog()->expectGrabLock(
+ [](StringData, const OID&, StringData, StringData, Date_t, StringData) {
+ // Don't care
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ LocksType currentLockDoc;
+ currentLockDoc.setName("bar");
+ currentLockDoc.setState(LocksType::LOCKED);
+ currentLockDoc.setProcess("otherProcess");
+ currentLockDoc.setLockID(OID("5572007fda9e476582bf3716"));
+ currentLockDoc.setWho("me");
+ currentLockDoc.setWhy("why");
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("bar", name); },
+ currentLockDoc);
+
+ getMockCatalog()->expectGetPing(
+ [](StringData process) { ASSERT_EQUALS("otherProcess", process); },
+ {ErrorCodes::NoMatchingDocument, "no ping"});
+
+ getMockCatalog()->expectGetServerInfo([]() {}, DistLockCatalog::ServerInfo(Date_t(), OID()));
+
+ // First attempt will record the ping data.
+ {
+ auto status = DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "", Milliseconds(0))
+ .getStatus();
+ ASSERT_NOT_OK(status);
+ ASSERT_EQUALS(ErrorCodes::LockBusy, status.code());
+ }
+
+ OID lastTS;
+ getMockCatalog()->expectGrabLock(
+ [&lastTS](StringData, const OID& newTS, StringData, StringData, Date_t, StringData) {
+ lastTS = newTS;
+ },
+ {ErrorCodes::LockStateChangeFailed, "nMod 0"});
+
+ getMockCatalog()->expectGetLockByName([](StringData name) { ASSERT_EQUALS("bar", name); },
+ currentLockDoc);
+
+ getMockCatalog()->expectGetPing(
+ [](StringData process) { ASSERT_EQUALS("otherProcess", process); },
+ {ErrorCodes::NoMatchingDocument, "no ping"});
+
+ getMockCatalog()->expectGetServerInfo(
+ []() {}, DistLockCatalog::ServerInfo(Date_t() + kLockExpiration + Milliseconds(1), OID()));
+
+ getMockCatalog()->expectOvertakeLock(
+ [this, &lastTS, &currentLockDoc](StringData lockID,
+ const OID& lockSessionID,
+ const OID& currentHolderTS,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ ASSERT_EQUALS("bar", lockID);
+ ASSERT_EQUALS(lastTS, lockSessionID);
+ ASSERT_EQUALS(currentLockDoc.getLockID(), currentHolderTS);
+ ASSERT_EQUALS(getProcessID(), processId);
+ ASSERT_EQUALS("foo", why);
+ },
+ currentLockDoc); // return arbitrary valid lock document, for testing purposes only.
+
+ getMockCatalog()->expectUnLock(
+ [](const OID&) {
+ // Don't care
+ },
+ Status::OK());
+
+ // Second attempt should overtake lock.
+ {
+ ASSERT_OK(DistLockManager::get(operationContext())
+ ->lock(operationContext(), "bar", "foo", Milliseconds(0))
+ .getStatus());
+ }
+}
+
+TEST_F(DistLockManagerReplSetTest, TryLockWithLocalWriteConcernBusy) {
+ std::string lockName("test");
+ Date_t now(Date_t::now());
+ std::string whyMsg("because");
+
+ LocksType retLockDoc;
+ retLockDoc.setName(lockName);
+ retLockDoc.setState(LocksType::LOCKED);
+ retLockDoc.setProcess(getProcessID());
+ retLockDoc.setWho("me");
+ retLockDoc.setWhy(whyMsg);
+ // Will be different from the actual lock session id. For testing only.
+ retLockDoc.setLockID(OID::gen());
+
+ OID lockSessionIDPassed = OID::gen();
+
+ getMockCatalog()->expectGrabLock(
+ [this, &lockName, &now, &whyMsg, &lockSessionIDPassed](StringData lockID,
+ const OID& lockSessionID,
+ StringData who,
+ StringData processId,
+ Date_t time,
+ StringData why) {
+ ASSERT_EQUALS(lockName, lockID);
+ ASSERT_TRUE(lockSessionID.isSet());
+ ASSERT_EQUALS(getProcessID(), processId);
+ ASSERT_GREATER_THAN_OR_EQUALS(time, now);
+ ASSERT_EQUALS(whyMsg, why);
+ ASSERT_EQUALS(lockSessionIDPassed, lockSessionID);
+
+ getMockCatalog()->expectNoGrabLock(); // Call only once.
+ },
+ {ErrorCodes::LockStateChangeFailed, "Unable to take lock"});
+
+ auto lockStatus = DistLockManager::get(operationContext())
+ ->tryLockWithLocalWriteConcern(
+ operationContext(), lockName, whyMsg, lockSessionIDPassed);
+ ASSERT_EQ(ErrorCodes::LockBusy, lockStatus.getStatus());
+}
+
+} // unnamed namespace
+} // namespace mongo
diff --git a/src/mongo/db/s/merge_chunks_command.cpp b/src/mongo/db/s/merge_chunks_command.cpp
index 0eb7a19ac65..5bcdcaff9b9 100644
--- a/src/mongo/db/s/merge_chunks_command.cpp
+++ b/src/mongo/db/s/merge_chunks_command.cpp
@@ -39,6 +39,7 @@
#include "mongo/db/field_parser.h"
#include "mongo/db/namespace_string.h"
#include "mongo/db/s/collection_sharding_runtime.h"
+#include "mongo/db/s/dist_lock_manager.h"
#include "mongo/db/s/shard_filtering_metadata_refresh.h"
#include "mongo/db/s/sharding_state.h"
#include "mongo/db/vector_clock.h"
@@ -80,7 +81,7 @@ void mergeChunks(OperationContext* opCtx,
const std::string whyMessage = str::stream() << "merging chunks in " << nss.ns() << " from "
<< redact(minKey) << " to " << redact(maxKey);
auto scopedDistLock = uassertStatusOKWithContext(
- Grid::get(opCtx)->catalogClient()->getDistLockManager()->lock(
+ DistLockManager::get(opCtx)->lock(
opCtx, nss.ns(), whyMessage, DistLockManager::kSingleLockAttemptTimeout),
str::stream() << "could not acquire collection lock for " << nss.ns()
<< " to merge chunks in [" << redact(minKey) << ", " << redact(maxKey)
diff --git a/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp b/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp
index 80380de60f8..25d1f2b598a 100644
--- a/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp
+++ b/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp
@@ -213,11 +213,10 @@ protected:
TxnNumber _txnNumber{0};
private:
- std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient(
- std::unique_ptr<DistLockManager> distLockManager) override {
+ std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient() override {
class StaticCatalogClient final : public ShardingCatalogClientMock {
public:
- StaticCatalogClient() : ShardingCatalogClientMock(nullptr) {}
+ StaticCatalogClient() = default;
StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards(
OperationContext* opCtx, repl::ReadConcernLevel readConcern) override {
diff --git a/src/mongo/db/s/migration_util_test.cpp b/src/mongo/db/s/migration_util_test.cpp
index 35838eb6953..fbe7cf9a40d 100644
--- a/src/mongo/db/s/migration_util_test.cpp
+++ b/src/mongo/db/s/migration_util_test.cpp
@@ -372,8 +372,7 @@ public:
// and loading all collections when a database is loaded for the first time by the CatalogCache.
class StaticCatalogClient final : public ShardingCatalogClientMock {
public:
- StaticCatalogClient(std::vector<ShardType> shards)
- : ShardingCatalogClientMock(nullptr), _shards(std::move(shards)) {}
+ StaticCatalogClient(std::vector<ShardType> shards) : _shards(std::move(shards)) {}
StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards(
OperationContext* opCtx, repl::ReadConcernLevel readConcern) override {
@@ -404,8 +403,7 @@ public:
return autoColl.getCollection()->uuid();
}
- std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient(
- std::unique_ptr<DistLockManager> distLockManager) override {
+ std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient() override {
auto mockCatalogClient = std::make_unique<StaticCatalogClient>(kShardList);
// Stash a pointer to the mock so its return values can be set.
_mockCatalogClient = mockCatalogClient.get();
diff --git a/src/mongo/db/s/resharding/resharding_donor_oplog_iterator_test.cpp b/src/mongo/db/s/resharding/resharding_donor_oplog_iterator_test.cpp
index 669da0f7ee0..5dbdba020d1 100644
--- a/src/mongo/db/s/resharding/resharding_donor_oplog_iterator_test.cpp
+++ b/src/mongo/db/s/resharding/resharding_donor_oplog_iterator_test.cpp
@@ -37,12 +37,10 @@
#include "mongo/db/s/resharding/resharding_donor_oplog_iterator.h"
#include "mongo/db/s/resharding/resharding_server_parameters_gen.h"
#include "mongo/db/s/resharding_util.h"
-#include "mongo/db/s/sharding_mongod_test_fixture.h"
-#include "mongo/db/service_context_d_test_fixture.h"
+#include "mongo/db/s/shard_server_test_fixture.h"
#include "mongo/executor/thread_pool_task_executor_test_fixture.h"
-#include "mongo/unittest/unittest.h"
-
#include "mongo/logv2/log.h"
+#include "mongo/unittest/unittest.h"
namespace mongo {
namespace {
@@ -93,7 +91,7 @@ private:
const int _originalValue;
};
-class ReshardingDonorOplogIterTest : public ShardingMongodTestFixture {
+class ReshardingDonorOplogIterTest : public ShardServerTestFixture {
public:
repl::MutableOplogEntry makeInsertOplog(Timestamp ts, BSONObj doc) {
ReshardingDonorOplogId oplogId(ts, ts);
diff --git a/src/mongo/db/s/resharding/resharding_donor_service_test.cpp b/src/mongo/db/s/resharding/resharding_donor_service_test.cpp
index b16c9c44dc6..8fb8f0c69ad 100644
--- a/src/mongo/db/s/resharding/resharding_donor_service_test.cpp
+++ b/src/mongo/db/s/resharding/resharding_donor_service_test.cpp
@@ -58,6 +58,7 @@
namespace mongo {
namespace {
+
auto reshardingTempNss(const UUID& existingUUID) {
return NamespaceString(fmt::format("db.system.resharding.{}", existingUUID.toString()));
}
@@ -67,9 +68,7 @@ protected:
class ThreeRecipientsCatalogClient final : public ShardingCatalogClientMock {
public:
ThreeRecipientsCatalogClient(UUID existingUUID, std::vector<ShardId> recipients)
- : ShardingCatalogClientMock(nullptr),
- _existingUUID(std::move(existingUUID)),
- _recipients(std::move(recipients)) {}
+ : _existingUUID(std::move(existingUUID)), _recipients(std::move(recipients)) {}
// Makes one chunk object per shard; the actual key ranges not relevant for the test.
// The output is deterministic since this function is also used to provide data to the
@@ -134,12 +133,9 @@ protected:
NamespaceString(kReshardNs), reshardingTempNss(kExistingUUID), kRecipientShards);
}
-
- std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient(
- std::unique_ptr<DistLockManager> distLockManager) {
- auto mockClient = std::make_unique<ThreeRecipientsCatalogClient>(
+ std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient() override {
+ return std::make_unique<ThreeRecipientsCatalogClient>(
uassertStatusOK(UUID::parse(kExistingUUID.toString())), kRecipientShards);
- return mockClient;
}
std::shared_ptr<ReshardingDonorService::DonorStateMachine> getStateMachineInstace(
diff --git a/src/mongo/db/s/resharding/resharding_oplog_applier_test.cpp b/src/mongo/db/s/resharding/resharding_oplog_applier_test.cpp
index 068519269b2..eca4bdd56b6 100644
--- a/src/mongo/db/s/resharding/resharding_oplog_applier_test.cpp
+++ b/src/mongo/db/s/resharding/resharding_oplog_applier_test.cpp
@@ -47,7 +47,6 @@
#include "mongo/db/s/resharding_util.h"
#include "mongo/db/s/sharding_mongod_test_fixture.h"
#include "mongo/db/s/sharding_state.h"
-#include "mongo/db/service_context_d_test_fixture.h"
#include "mongo/db/session_catalog_mongod.h"
#include "mongo/db/transaction_participant.h"
#include "mongo/executor/thread_pool_task_executor_test_fixture.h"
diff --git a/src/mongo/db/s/resharding/resharding_oplog_fetcher_test.cpp b/src/mongo/db/s/resharding/resharding_oplog_fetcher_test.cpp
index 79b029a9a92..2b9aed54b82 100644
--- a/src/mongo/db/s/resharding/resharding_oplog_fetcher_test.cpp
+++ b/src/mongo/db/s/resharding/resharding_oplog_fetcher_test.cpp
@@ -128,13 +128,11 @@ public:
* ShardRegistry reload is done over DBClient, not the NetworkInterface, and there is no
* DBClientMock analogous to the NetworkInterfaceMock.
*/
- std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient(
- std::unique_ptr<DistLockManager> distLockManager) {
+ std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient() {
class StaticCatalogClient final : public ShardingCatalogClientMock {
public:
- StaticCatalogClient(std::vector<ShardId> shardIds)
- : ShardingCatalogClientMock(nullptr), _shardIds(std::move(shardIds)) {}
+ StaticCatalogClient(std::vector<ShardId> shardIds) : _shardIds(std::move(shardIds)) {}
StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards(
OperationContext* opCtx, repl::ReadConcernLevel readConcern) override {
diff --git a/src/mongo/db/s/resharding/resharding_txn_cloner_test.cpp b/src/mongo/db/s/resharding/resharding_txn_cloner_test.cpp
index c493e4ce51e..db10b0a6bf6 100644
--- a/src/mongo/db/s/resharding/resharding_txn_cloner_test.cpp
+++ b/src/mongo/db/s/resharding/resharding_txn_cloner_test.cpp
@@ -134,13 +134,11 @@ class ReshardingTxnClonerTest : public ShardServerTestFixture {
* ShardRegistry reload is done over DBClient, not the NetworkInterface, and there is no
* DBClientMock analogous to the NetworkInterfaceMock.
*/
- std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient(
- std::unique_ptr<DistLockManager> distLockManager) {
+ std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient() {
class StaticCatalogClient final : public ShardingCatalogClientMock {
public:
- StaticCatalogClient(std::vector<ShardId> shardIds)
- : ShardingCatalogClientMock(nullptr), _shardIds(std::move(shardIds)) {}
+ StaticCatalogClient(std::vector<ShardId> shardIds) : _shardIds(std::move(shardIds)) {}
StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards(
OperationContext* opCtx, repl::ReadConcernLevel readConcern) override {
diff --git a/src/mongo/db/s/resharding_destined_recipient_test.cpp b/src/mongo/db/s/resharding_destined_recipient_test.cpp
index 78108531fbb..d9d51ae12a2 100644
--- a/src/mongo/db/s/resharding_destined_recipient_test.cpp
+++ b/src/mongo/db/s/resharding_destined_recipient_test.cpp
@@ -125,8 +125,7 @@ public:
class StaticCatalogClient final : public ShardingCatalogClientMock {
public:
- StaticCatalogClient(std::vector<ShardType> shards)
- : ShardingCatalogClientMock(nullptr), _shards(std::move(shards)) {}
+ StaticCatalogClient(std::vector<ShardType> shards) : _shards(std::move(shards)) {}
StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards(
OperationContext* opCtx, repl::ReadConcernLevel readConcern) override {
@@ -149,8 +148,7 @@ public:
std::vector<CollectionType> _colls;
};
- std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient(
- std::unique_ptr<DistLockManager> distLockManager) override {
+ std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient() override {
return std::make_unique<StaticCatalogClient>(kShardList);
}
diff --git a/src/mongo/db/s/resharding_util.cpp b/src/mongo/db/s/resharding_util.cpp
index 467fe1b5654..9cdfd2ea121 100644
--- a/src/mongo/db/s/resharding_util.cpp
+++ b/src/mongo/db/s/resharding_util.cpp
@@ -49,7 +49,6 @@
#include "mongo/db/pipeline/document_source_sort.h"
#include "mongo/db/pipeline/document_source_unwind.h"
#include "mongo/db/s/collection_sharding_state.h"
-#include "mongo/db/s/config/sharding_catalog_manager.h"
#include "mongo/db/storage/write_unit_of_work.h"
#include "mongo/logv2/log.h"
#include "mongo/rpc/get_status_from_command_result.h"
diff --git a/src/mongo/db/s/session_catalog_migration_destination_test.cpp b/src/mongo/db/s/session_catalog_migration_destination_test.cpp
index 28cac823d42..aba60895ed0 100644
--- a/src/mongo/db/s/session_catalog_migration_destination_test.cpp
+++ b/src/mongo/db/s/session_catalog_migration_destination_test.cpp
@@ -278,11 +278,10 @@ public:
}
private:
- std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient(
- std::unique_ptr<DistLockManager> distLockManager) override {
+ std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient() override {
class StaticCatalogClient final : public ShardingCatalogClientMock {
public:
- StaticCatalogClient() : ShardingCatalogClientMock(nullptr) {}
+ StaticCatalogClient() = default;
StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards(
OperationContext* opCtx, repl::ReadConcernLevel readConcern) override {
diff --git a/src/mongo/db/s/shard_collection_legacy.cpp b/src/mongo/db/s/shard_collection_legacy.cpp
index 494e8a01771..7952367766a 100644
--- a/src/mongo/db/s/shard_collection_legacy.cpp
+++ b/src/mongo/db/s/shard_collection_legacy.cpp
@@ -47,7 +47,7 @@
#include "mongo/db/s/active_shard_collection_registry.h"
#include "mongo/db/s/collection_sharding_runtime.h"
#include "mongo/db/s/config/initial_split_policy.h"
-#include "mongo/db/s/config/sharding_catalog_manager.h"
+#include "mongo/db/s/dist_lock_manager.h"
#include "mongo/db/s/shard_collection_legacy.h"
#include "mongo/db/s/shard_filtering_metadata_refresh.h"
#include "mongo/db/s/shard_key_util.h"
@@ -589,10 +589,9 @@ CreateCollectionResponse shardCollection(OperationContext* opCtx,
boost::optional<DistLockManager::ScopedDistLock> dbDistLock;
boost::optional<DistLockManager::ScopedDistLock> collDistLock;
if (!mustTakeDistLock) {
- auto const catalogClient = Grid::get(opCtx)->catalogClient();
- dbDistLock.emplace(uassertStatusOK(catalogClient->getDistLockManager()->lock(
+ dbDistLock.emplace(uassertStatusOK(DistLockManager::get(opCtx)->lock(
opCtx, nss.db(), "shardCollection", DistLockManager::kDefaultLockTimeout)));
- collDistLock.emplace(uassertStatusOK(catalogClient->getDistLockManager()->lock(
+ collDistLock.emplace(uassertStatusOK(DistLockManager::get(opCtx)->lock(
opCtx, nss.ns(), "shardCollection", DistLockManager::kDefaultLockTimeout)));
}
diff --git a/src/mongo/db/s/shard_server_test_fixture.cpp b/src/mongo/db/s/shard_server_test_fixture.cpp
index 9677193045a..2c65e268b1b 100644
--- a/src/mongo/db/s/shard_server_test_fixture.cpp
+++ b/src/mongo/db/s/shard_server_test_fixture.cpp
@@ -36,8 +36,6 @@
#include "mongo/db/repl/replication_coordinator_mock.h"
#include "mongo/db/s/shard_server_catalog_cache_loader.h"
#include "mongo/db/s/sharding_state.h"
-#include "mongo/s/catalog/dist_lock_catalog_mock.h"
-#include "mongo/s/catalog/dist_lock_manager_mock.h"
#include "mongo/s/catalog/sharding_catalog_client_impl.h"
#include "mongo/s/catalog_cache.h"
#include "mongo/s/config_server_catalog_cache_loader.h"
@@ -91,20 +89,8 @@ void ShardServerTestFixture::tearDown() {
ShardingMongodTestFixture::tearDown();
}
-std::unique_ptr<DistLockCatalog> ShardServerTestFixture::makeDistLockCatalog() {
- return std::make_unique<DistLockCatalogMock>();
-}
-
-std::unique_ptr<DistLockManager> ShardServerTestFixture::makeDistLockManager(
- std::unique_ptr<DistLockCatalog> distLockCatalog) {
- invariant(distLockCatalog);
- return std::make_unique<DistLockManagerMock>(std::move(distLockCatalog));
-}
-
-std::unique_ptr<ShardingCatalogClient> ShardServerTestFixture::makeShardingCatalogClient(
- std::unique_ptr<DistLockManager> distLockManager) {
- invariant(distLockManager);
- return std::make_unique<ShardingCatalogClientImpl>(std::move(distLockManager));
+std::unique_ptr<ShardingCatalogClient> ShardServerTestFixture::makeShardingCatalogClient() {
+ return std::make_unique<ShardingCatalogClientImpl>();
}
} // namespace mongo
diff --git a/src/mongo/db/s/shard_server_test_fixture.h b/src/mongo/db/s/shard_server_test_fixture.h
index 27da7d834f2..57adbc030ba 100644
--- a/src/mongo/db/s/shard_server_test_fixture.h
+++ b/src/mongo/db/s/shard_server_test_fixture.h
@@ -57,22 +57,7 @@ protected:
*/
std::shared_ptr<RemoteCommandTargeterMock> configTargeterMock();
- /**
- * Creates a DistLockCatalogMock.
- */
- std::unique_ptr<DistLockCatalog> makeDistLockCatalog() override;
-
- /**
- * Creates a DistLockManagerMock.
- */
- std::unique_ptr<DistLockManager> makeDistLockManager(
- std::unique_ptr<DistLockCatalog> distLockCatalog) override;
-
- /**
- * Creates a real ShardingCatalogClient.
- */
- std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient(
- std::unique_ptr<DistLockManager> distLockManager) override;
+ std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient() override;
void setCatalogCacheLoader(std::unique_ptr<CatalogCacheLoader> loader);
diff --git a/src/mongo/db/s/sharding_initialization_mongod.cpp b/src/mongo/db/s/sharding_initialization_mongod.cpp
index 64f628e315f..a8d9cf60ed9 100644
--- a/src/mongo/db/s/sharding_initialization_mongod.cpp
+++ b/src/mongo/db/s/sharding_initialization_mongod.cpp
@@ -35,7 +35,6 @@
#include "mongo/client/connection_string.h"
#include "mongo/client/global_conn_pool.h"
-#include "mongo/client/remote_command_targeter.h"
#include "mongo/client/remote_command_targeter_factory_impl.h"
#include "mongo/client/replica_set_monitor.h"
#include "mongo/db/catalog_raii.h"
@@ -47,6 +46,8 @@
#include "mongo/db/ops/update.h"
#include "mongo/db/repl/replication_coordinator.h"
#include "mongo/db/s/chunk_splitter.h"
+#include "mongo/db/s/dist_lock_catalog_replset.h"
+#include "mongo/db/s/dist_lock_manager_replset.h"
#include "mongo/db/s/periodic_balancer_config_refresher.h"
#include "mongo/db/s/read_only_catalog_cache_loader.h"
#include "mongo/db/s/shard_local.h"
@@ -258,42 +259,9 @@ private:
} // namespace
-void ShardingInitializationMongoD::initializeShardingEnvironmentOnShardServer(
- OperationContext* opCtx, const ShardIdentity& shardIdentity, StringData distLockProcessId) {
-
- _replicaSetChangeListener =
- ReplicaSetMonitor::getNotifier().makeListener<ShardingReplicaSetChangeListener>(
- opCtx->getServiceContext());
-
- initializeGlobalShardingStateForMongoD(
- opCtx, shardIdentity.getConfigsvrConnectionString(), distLockProcessId);
-
-
- // Determine primary/secondary/standalone state in order to properly initialize sharding
- // components.
- const auto replCoord = repl::ReplicationCoordinator::get(opCtx);
- bool isReplSet = replCoord->getReplicationMode() == repl::ReplicationCoordinator::modeReplSet;
- bool isStandaloneOrPrimary =
- !isReplSet || (replCoord->getMemberState() == repl::MemberState::RS_PRIMARY);
-
- CatalogCacheLoader::get(opCtx).initializeReplicaSetRole(isStandaloneOrPrimary);
- ChunkSplitter::get(opCtx).onShardingInitialization(isStandaloneOrPrimary);
- PeriodicBalancerConfigRefresher::get(opCtx).onShardingInitialization(opCtx->getServiceContext(),
- isStandaloneOrPrimary);
-
- // Start the transaction coordinator service only if the node is the primary of a replica set
- TransactionCoordinatorService::get(opCtx)->onShardingInitialization(
- opCtx, isReplSet && isStandaloneOrPrimary);
-
- LOGV2(22071,
- "Finished initializing sharding components for {memberState} node.",
- "Finished initializing sharding components",
- "memberState"_attr = (isStandaloneOrPrimary ? "primary" : "secondary"));
-}
-
ShardingInitializationMongoD::ShardingInitializationMongoD()
: _initFunc([this](auto... args) {
- this->initializeShardingEnvironmentOnShardServer(std::forward<decltype(args)>(args)...);
+ _initializeShardingEnvironmentOnShardServer(std::forward<decltype(args)>(args)...);
}) {}
ShardingInitializationMongoD::~ShardingInitializationMongoD() = default;
@@ -308,13 +276,14 @@ ShardingInitializationMongoD* ShardingInitializationMongoD::get(ServiceContext*
void ShardingInitializationMongoD::shutDown(OperationContext* opCtx) {
auto const shardingState = ShardingState::get(opCtx);
- auto const grid = Grid::get(opCtx);
-
if (!shardingState->enabled())
return;
- grid->catalogClient()->shutDown(opCtx);
+ DistLockManager::get(opCtx)->shutDown(opCtx);
+
+ auto const grid = Grid::get(opCtx);
grid->shardRegistry()->shutdown();
+
_replicaSetChangeListener.reset();
}
@@ -466,6 +435,7 @@ void ShardingInitializationMongoD::initializeFromShardIdentity(
} catch (const DBException& ex) {
shardingState->setInitialized(ex.toStatus());
}
+
Grid::get(opCtx)->setShardingInitialized();
}
@@ -514,36 +484,33 @@ void ShardingInitializationMongoD::updateShardIdentityConfigString(
void initializeGlobalShardingStateForMongoD(OperationContext* opCtx,
const ConnectionString& configCS,
StringData distLockProcessId) {
+ uassert(ErrorCodes::BadValue, "Unrecognized connection string.", configCS);
+
auto targeterFactory = std::make_unique<RemoteCommandTargeterFactoryImpl>();
auto targeterFactoryPtr = targeterFactory.get();
- ShardFactory::BuilderCallable setBuilder = [targeterFactoryPtr](
- const ShardId& shardId,
- const ConnectionString& connStr) {
- return std::make_unique<ShardRemote>(shardId, connStr, targeterFactoryPtr->create(connStr));
- };
-
- ShardFactory::BuilderCallable masterBuilder = [targeterFactoryPtr](
- const ShardId& shardId,
- const ConnectionString& connStr) {
- return std::make_unique<ShardRemote>(shardId, connStr, targeterFactoryPtr->create(connStr));
- };
-
- ShardFactory::BuilderCallable localBuilder = [](const ShardId& shardId,
- const ConnectionString& connStr) {
- return std::make_unique<ShardLocal>(shardId);
- };
-
ShardFactory::BuildersMap buildersMap{
- {ConnectionString::ConnectionType::kReplicaSet, std::move(setBuilder)},
- {ConnectionString::ConnectionType::kStandalone, std::move(masterBuilder)},
- {ConnectionString::ConnectionType::kLocal, std::move(localBuilder)},
+ {ConnectionString::ConnectionType::kReplicaSet,
+ [targeterFactoryPtr](const ShardId& shardId, const ConnectionString& connStr) {
+ return std::make_unique<ShardRemote>(
+ shardId, connStr, targeterFactoryPtr->create(connStr));
+ }},
+ {ConnectionString::ConnectionType::kLocal,
+ [](const ShardId& shardId, const ConnectionString& connStr) {
+ return std::make_unique<ShardLocal>(shardId);
+ }},
+ {ConnectionString::ConnectionType::kStandalone,
+ [targeterFactoryPtr](const ShardId& shardId, const ConnectionString& connStr) {
+ return std::make_unique<ShardRemote>(
+ shardId, connStr, targeterFactoryPtr->create(connStr));
+ }},
};
auto shardFactory =
std::make_unique<ShardFactory>(std::move(buildersMap), std::move(targeterFactory));
auto const service = opCtx->getServiceContext();
+
if (serverGlobalParams.clusterRole == ClusterRole::ShardServer) {
if (storageGlobalParams.readOnly) {
CatalogCacheLoader::set(service, std::make_unique<ReadOnlyCatalogCacheLoader>());
@@ -552,8 +519,10 @@ void initializeGlobalShardingStateForMongoD(OperationContext* opCtx,
std::make_unique<ShardServerCatalogCacheLoader>(
std::make_unique<ConfigServerCatalogCacheLoader>()));
}
- } else {
+ } else if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer) {
CatalogCacheLoader::set(service, std::make_unique<ConfigServerCatalogCacheLoader>());
+ } else {
+ MONGO_UNREACHABLE;
}
auto validator = LogicalTimeValidator::get(service);
@@ -575,8 +544,6 @@ void initializeGlobalShardingStateForMongoD(OperationContext* opCtx,
catCache->invalidateEntriesThatReferenceShard(removedShard);
}};
- uassert(ErrorCodes::BadValue, "Unrecognized connection string.", configCS);
-
auto shardRegistry = std::make_unique<ShardRegistry>(
std::move(shardFactory), configCS, std::move(shardRemovalHooks));
@@ -590,6 +557,15 @@ void initializeGlobalShardingStateForMongoD(OperationContext* opCtx,
// executors aren't used for user queries in mongod.
1));
+ DistLockManager::create(
+ service,
+ std::make_unique<ReplSetDistLockManager>(service,
+ distLockProcessId,
+ std::make_unique<DistLockCatalogImpl>(),
+ ReplSetDistLockManager::kDistLockPingInterval,
+ ReplSetDistLockManager::kDistLockExpirationTime));
+ DistLockManager::get(opCtx)->startUp();
+
auto const replCoord = repl::ReplicationCoordinator::get(service);
if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer &&
replCoord->getMemberState().primary()) {
@@ -597,4 +573,37 @@ void initializeGlobalShardingStateForMongoD(OperationContext* opCtx,
}
}
+void ShardingInitializationMongoD::_initializeShardingEnvironmentOnShardServer(
+ OperationContext* opCtx, const ShardIdentity& shardIdentity, StringData distLockProcessId) {
+
+ _replicaSetChangeListener =
+ ReplicaSetMonitor::getNotifier().makeListener<ShardingReplicaSetChangeListener>(
+ opCtx->getServiceContext());
+
+ initializeGlobalShardingStateForMongoD(
+ opCtx, shardIdentity.getConfigsvrConnectionString(), distLockProcessId);
+
+
+ // Determine primary/secondary/standalone state in order to properly initialize sharding
+ // components.
+ const auto replCoord = repl::ReplicationCoordinator::get(opCtx);
+ bool isReplSet = replCoord->getReplicationMode() == repl::ReplicationCoordinator::modeReplSet;
+ bool isStandaloneOrPrimary =
+ !isReplSet || (replCoord->getMemberState() == repl::MemberState::RS_PRIMARY);
+
+ CatalogCacheLoader::get(opCtx).initializeReplicaSetRole(isStandaloneOrPrimary);
+ ChunkSplitter::get(opCtx).onShardingInitialization(isStandaloneOrPrimary);
+ PeriodicBalancerConfigRefresher::get(opCtx).onShardingInitialization(opCtx->getServiceContext(),
+ isStandaloneOrPrimary);
+
+ // Start the transaction coordinator service only if the node is the primary of a replica set
+ TransactionCoordinatorService::get(opCtx)->onShardingInitialization(
+ opCtx, isReplSet && isStandaloneOrPrimary);
+
+ LOGV2(22071,
+ "Finished initializing sharding components for {memberState} node.",
+ "Finished initializing sharding components",
+ "memberState"_attr = (isStandaloneOrPrimary ? "primary" : "secondary"));
+}
+
} // namespace mongo
diff --git a/src/mongo/db/s/sharding_initialization_mongod.h b/src/mongo/db/s/sharding_initialization_mongod.h
index a8e8eec3c72..f02d5d43c69 100644
--- a/src/mongo/db/s/sharding_initialization_mongod.h
+++ b/src/mongo/db/s/sharding_initialization_mongod.h
@@ -38,10 +38,6 @@
namespace mongo {
-class ConnectionString;
-class OperationContext;
-class ServiceContext;
-
/**
* This class serves as a bootstrap and shutdown for the sharding subsystem and also controls the
* persisted cluster identity. The default ShardingEnvironmentInitFunc instantiates all the sharding
@@ -62,10 +58,6 @@ public:
static ShardingInitializationMongoD* get(OperationContext* opCtx);
static ShardingInitializationMongoD* get(ServiceContext* service);
- void initializeShardingEnvironmentOnShardServer(OperationContext* opCtx,
- const ShardIdentity& shardIdentity,
- StringData distLockProcessId);
-
/**
* If started with --shardsvr, initializes sharding awareness from the shardIdentity document on
* disk, if there is one.
@@ -112,6 +104,10 @@ public:
}
private:
+ void _initializeShardingEnvironmentOnShardServer(OperationContext* opCtx,
+ const ShardIdentity& shardIdentity,
+ StringData distLockProcessId);
+
// This mutex ensures that only one thread at a time executes the sharding
// initialization/teardown sequence
Mutex _initSynchronizationMutex =
diff --git a/src/mongo/db/s/sharding_initialization_mongod_test.cpp b/src/mongo/db/s/sharding_initialization_mongod_test.cpp
index 44b90529a89..f1359e75be1 100644
--- a/src/mongo/db/s/sharding_initialization_mongod_test.cpp
+++ b/src/mongo/db/s/sharding_initialization_mongod_test.cpp
@@ -44,7 +44,6 @@
#include "mongo/db/s/sharding_initialization_mongod.h"
#include "mongo/db/s/type_shard_identity.h"
#include "mongo/db/server_options.h"
-#include "mongo/s/catalog/dist_lock_manager_mock.h"
#include "mongo/s/catalog/sharding_catalog_client_impl.h"
#include "mongo/s/client/shard_registry.h"
#include "mongo/s/config_server_catalog_cache_loader.h"
@@ -60,9 +59,6 @@ const std::string kShardName("TestShard");
*/
class ShardingInitializationMongoDTest : public ShardingMongodTestFixture {
protected:
- // Used to write to set up local collections before exercising server logic.
- std::unique_ptr<DBDirectClient> _dbDirectClient;
-
void setUp() override {
serverGlobalParams.clusterRole = ClusterRole::None;
ShardingMongodTestFixture::setUp();
@@ -109,15 +105,8 @@ protected:
ShardingMongodTestFixture::tearDown();
}
- std::unique_ptr<DistLockManager> makeDistLockManager(
- std::unique_ptr<DistLockCatalog> distLockCatalog) override {
- return std::make_unique<DistLockManagerMock>(nullptr);
- }
-
- std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient(
- std::unique_ptr<DistLockManager> distLockManager) override {
- invariant(distLockManager);
- return std::make_unique<ShardingCatalogClientImpl>(std::move(distLockManager));
+ std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient() override {
+ return std::make_unique<ShardingCatalogClientImpl>();
}
auto* shardingInitialization() {
@@ -127,6 +116,9 @@ protected:
auto* shardingState() {
return ShardingState::get(getServiceContext());
}
+
+ // Used to write to set up local collections before exercising server logic.
+ std::unique_ptr<DBDirectClient> _dbDirectClient;
};
/**
diff --git a/src/mongo/db/s/sharding_initialization_op_observer_test.cpp b/src/mongo/db/s/sharding_initialization_op_observer_test.cpp
index 019394f91e9..e029b043c01 100644
--- a/src/mongo/db/s/sharding_initialization_op_observer_test.cpp
+++ b/src/mongo/db/s/sharding_initialization_op_observer_test.cpp
@@ -43,7 +43,6 @@
#include "mongo/db/s/sharding_mongod_test_fixture.h"
#include "mongo/db/s/type_shard_identity.h"
#include "mongo/db/server_options.h"
-#include "mongo/s/catalog/dist_lock_manager_mock.h"
#include "mongo/s/client/shard_registry.h"
#include "mongo/s/config_server_catalog_cache_loader.h"
diff --git a/src/mongo/db/s/sharding_mongod_test_fixture.cpp b/src/mongo/db/s/sharding_mongod_test_fixture.cpp
index c1e65f887ce..d0881708665 100644
--- a/src/mongo/db/s/sharding_mongod_test_fixture.cpp
+++ b/src/mongo/db/s/sharding_mongod_test_fixture.cpp
@@ -54,6 +54,7 @@
#include "mongo/db/repl/storage_interface_mock.h"
#include "mongo/db/s/collection_sharding_state_factory_shard.h"
#include "mongo/db/s/config_server_op_observer.h"
+#include "mongo/db/s/dist_lock_manager_mock.h"
#include "mongo/db/s/op_observer_sharding_impl.h"
#include "mongo/db/s/shard_local.h"
#include "mongo/db/s/shard_server_op_observer.h"
@@ -61,8 +62,6 @@
#include "mongo/executor/thread_pool_task_executor_test_fixture.h"
#include "mongo/rpc/metadata/repl_set_metadata.h"
#include "mongo/s/balancer_configuration.h"
-#include "mongo/s/catalog/dist_lock_catalog.h"
-#include "mongo/s/catalog/dist_lock_manager.h"
#include "mongo/s/catalog/sharding_catalog_client.h"
#include "mongo/s/catalog/type_collection.h"
#include "mongo/s/catalog/type_shard.h"
@@ -219,13 +218,8 @@ std::unique_ptr<ShardRegistry> ShardingMongodTestFixture::makeShardRegistry(
return std::make_unique<ShardRegistry>(std::move(shardFactory), configConnStr);
}
-std::unique_ptr<DistLockCatalog> ShardingMongodTestFixture::makeDistLockCatalog() {
- return nullptr;
-}
-
-std::unique_ptr<DistLockManager> ShardingMongodTestFixture::makeDistLockManager(
- std::unique_ptr<DistLockCatalog> distLockCatalog) {
- return nullptr;
+std::unique_ptr<DistLockManager> ShardingMongodTestFixture::makeDistLockManager() {
+ return std::make_unique<DistLockManagerMock>();
}
std::unique_ptr<ClusterCursorManager> ShardingMongodTestFixture::makeClusterCursorManager() {
@@ -249,17 +243,11 @@ Status ShardingMongodTestFixture::initializeGlobalShardingStateForMongodForTest(
executorPoolPtr->startup();
}
- auto distLockCatalogPtr = makeDistLockCatalog();
- _distLockCatalog = distLockCatalogPtr.get();
-
- auto distLockManagerPtr = makeDistLockManager(std::move(distLockCatalogPtr));
- _distLockManager = distLockManagerPtr.get();
-
auto catalogCache = std::make_unique<CatalogCache>(
getServiceContext(), CatalogCacheLoader::get(getServiceContext()));
auto const grid = Grid::get(operationContext());
- grid->init(makeShardingCatalogClient(std::move(distLockManagerPtr)),
+ grid->init(makeShardingCatalogClient(),
std::move(catalogCache),
makeShardRegistry(configConnStr),
makeClusterCursorManager(),
@@ -267,26 +255,28 @@ Status ShardingMongodTestFixture::initializeGlobalShardingStateForMongodForTest(
std::move(executorPoolPtr),
_mockNetwork);
- if (grid->catalogClient()) {
- grid->catalogClient()->startup();
+ DistLockManager::create(getServiceContext(), makeDistLockManager());
+ if (DistLockManager::get(operationContext())) {
+ DistLockManager::get(operationContext())->startUp();
}
return Status::OK();
}
void ShardingMongodTestFixture::setUp() {
+ ServiceContextMongoDTest::setUp();
ShardingTestFixtureCommon::setUp();
}
void ShardingMongodTestFixture::tearDown() {
ReplicaSetMonitor::cleanup();
- if (Grid::get(operationContext())->getExecutorPool() && !_executorPoolShutDown) {
- Grid::get(operationContext())->getExecutorPool()->shutdownAndJoin();
+ if (DistLockManager::get(operationContext())) {
+ DistLockManager::get(operationContext())->shutDown(operationContext());
}
- if (Grid::get(operationContext())->catalogClient()) {
- Grid::get(operationContext())->catalogClient()->shutDown(operationContext());
+ if (Grid::get(operationContext())->getExecutorPool() && !_executorPoolShutDown) {
+ Grid::get(operationContext())->getExecutorPool()->shutdownAndJoin();
}
if (Grid::get(operationContext())->shardRegistry()) {
diff --git a/src/mongo/db/s/sharding_mongod_test_fixture.h b/src/mongo/db/s/sharding_mongod_test_fixture.h
index 3940b46245d..7c6d955976e 100644
--- a/src/mongo/db/s/sharding_mongod_test_fixture.h
+++ b/src/mongo/db/s/sharding_mongod_test_fixture.h
@@ -30,17 +30,12 @@
#pragma once
#include "mongo/db/repl/replication_coordinator_mock.h"
+#include "mongo/db/s/dist_lock_manager.h"
#include "mongo/db/service_context_d_test_fixture.h"
#include "mongo/s/sharding_test_fixture_common.h"
namespace mongo {
-class CatalogCacheLoader;
-
-namespace repl {
-class ReplSettings;
-} // namespace repl
-
/**
* Sets up this fixture as a mongod with a storage engine, OpObserver, and as a member of a replica
* set.
@@ -57,7 +52,6 @@ protected:
~ShardingMongodTestFixture();
void setUp() override;
-
void tearDown() override;
/**
@@ -110,18 +104,9 @@ protected:
virtual std::unique_ptr<ShardRegistry> makeShardRegistry(ConnectionString configConnStr);
/**
- * Base class returns nullptr.
- */
- virtual std::unique_ptr<DistLockCatalog> makeDistLockCatalog();
-
- /**
- * Base class returns nullptr.
- *
- * Note: DistLockManager takes ownership of the DistLockCatalog, so if DistLockCatalog is not
- * nullptr, a real or mock DistLockManager must be supplied.
+ * Allows tests to conditionally construct a DistLockManager
*/
- virtual std::unique_ptr<DistLockManager> makeDistLockManager(
- std::unique_ptr<DistLockCatalog> distLockCatalog);
+ virtual std::unique_ptr<DistLockManager> makeDistLockManager();
/**
* Base class returns nullptr.
diff --git a/src/mongo/db/s/shardsvr_drop_collection_command.cpp b/src/mongo/db/s/shardsvr_drop_collection_command.cpp
index b838f3e8dcf..1fe52939abb 100644
--- a/src/mongo/db/s/shardsvr_drop_collection_command.cpp
+++ b/src/mongo/db/s/shardsvr_drop_collection_command.cpp
@@ -32,9 +32,7 @@
#include "mongo/db/auth/authorization_session.h"
#include "mongo/db/commands.h"
#include "mongo/db/drop_database_gen.h"
-#include "mongo/db/s/config/sharding_catalog_manager.h"
#include "mongo/db/s/sharding_logging.h"
-#include "mongo/s/catalog/dist_lock_manager.h"
#include "mongo/s/catalog/type_database.h"
#include "mongo/s/catalog_cache.h"
#include "mongo/s/grid.h"
diff --git a/src/mongo/db/s/shardsvr_drop_database_command.cpp b/src/mongo/db/s/shardsvr_drop_database_command.cpp
index 785ded10239..b2938bb3319 100644
--- a/src/mongo/db/s/shardsvr_drop_database_command.cpp
+++ b/src/mongo/db/s/shardsvr_drop_database_command.cpp
@@ -34,7 +34,6 @@
#include "mongo/db/drop_database_gen.h"
#include "mongo/db/s/config/sharding_catalog_manager.h"
#include "mongo/db/s/sharding_logging.h"
-#include "mongo/s/catalog/dist_lock_manager.h"
#include "mongo/s/catalog/type_database.h"
#include "mongo/s/catalog_cache.h"
#include "mongo/s/grid.h"
diff --git a/src/mongo/db/s/shardsvr_refine_collection_shard_key_command.cpp b/src/mongo/db/s/shardsvr_refine_collection_shard_key_command.cpp
index 8ecd531f64d..feed625c073 100644
--- a/src/mongo/db/s/shardsvr_refine_collection_shard_key_command.cpp
+++ b/src/mongo/db/s/shardsvr_refine_collection_shard_key_command.cpp
@@ -34,7 +34,6 @@
#include "mongo/db/drop_database_gen.h"
#include "mongo/db/s/config/sharding_catalog_manager.h"
#include "mongo/db/s/sharding_logging.h"
-#include "mongo/s/catalog/dist_lock_manager.h"
#include "mongo/s/catalog/type_database.h"
#include "mongo/s/catalog_cache.h"
#include "mongo/s/grid.h"
diff --git a/src/mongo/db/s/split_chunk.cpp b/src/mongo/db/s/split_chunk.cpp
index 718aa93c20e..f2b883d423b 100644
--- a/src/mongo/db/s/split_chunk.cpp
+++ b/src/mongo/db/s/split_chunk.cpp
@@ -44,6 +44,7 @@
#include "mongo/db/namespace_string.h"
#include "mongo/db/query/internal_plans.h"
#include "mongo/db/s/collection_sharding_runtime.h"
+#include "mongo/db/s/dist_lock_manager.h"
#include "mongo/db/s/shard_filtering_metadata_refresh.h"
#include "mongo/db/s/sharding_state.h"
#include "mongo/logv2/log.h"
@@ -135,7 +136,7 @@ StatusWith<boost::optional<ChunkRange>> splitChunk(OperationContext* opCtx,
const std::string whyMessage(str::stream()
<< "splitting chunk " << redact(chunkRange.toString()) << " in "
<< nss.toString());
- auto scopedDistLock = Grid::get(opCtx)->catalogClient()->getDistLockManager()->lock(
+ auto scopedDistLock = DistLockManager::get(opCtx)->lock(
opCtx, nss.ns(), whyMessage, DistLockManager::kDefaultLockTimeout);
if (!scopedDistLock.isOK()) {
return scopedDistLock.getStatus().withContext(
diff --git a/src/mongo/db/s/split_chunk_test.cpp b/src/mongo/db/s/split_chunk_test.cpp
index 3d9147c4130..298059e85cc 100644
--- a/src/mongo/db/s/split_chunk_test.cpp
+++ b/src/mongo/db/s/split_chunk_test.cpp
@@ -34,6 +34,7 @@
#include <boost/optional.hpp>
#include "mongo/db/json.h"
+#include "mongo/db/s/dist_lock_manager_mock.h"
#include "mongo/db/s/shard_server_test_fixture.h"
#include "mongo/db/s/sharding_initialization_mongod.h"
#include "mongo/db/s/split_chunk.h"
@@ -42,7 +43,6 @@
#include "mongo/executor/remote_command_response.h"
#include "mongo/executor/task_executor.h"
#include "mongo/logv2/log.h"
-#include "mongo/s/catalog/dist_lock_manager_mock.h"
#include "mongo/s/catalog/type_chunk.h"
#include "mongo/s/catalog/type_collection.h"
#include "mongo/s/catalog/type_database.h"
diff --git a/src/mongo/db/s/transaction_coordinator_futures_util_test.cpp b/src/mongo/db/s/transaction_coordinator_futures_util_test.cpp
index ccb1d84d1c8..d3e7e2818e1 100644
--- a/src/mongo/db/s/transaction_coordinator_futures_util_test.cpp
+++ b/src/mongo/db/s/transaction_coordinator_futures_util_test.cpp
@@ -292,12 +292,11 @@ protected:
// expected shards. We cannot mock the network responses for the ShardRegistry reload, since the
// ShardRegistry reload is done over DBClient, not the NetworkInterface, and there is no
// DBClientMock analogous to the NetworkInterfaceMock.
- std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient(
- std::unique_ptr<DistLockManager> distLockManager) override {
+ std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient() override {
class StaticCatalogClient final : public ShardingCatalogClientMock {
public:
- StaticCatalogClient() : ShardingCatalogClientMock(nullptr) {}
+ StaticCatalogClient() = default;
StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards(
OperationContext* opCtx, repl::ReadConcernLevel readConcern) override {
diff --git a/src/mongo/db/s/transaction_coordinator_test_fixture.cpp b/src/mongo/db/s/transaction_coordinator_test_fixture.cpp
index 377a1993213..560dc51524b 100644
--- a/src/mongo/db/s/transaction_coordinator_test_fixture.cpp
+++ b/src/mongo/db/s/transaction_coordinator_test_fixture.cpp
@@ -74,13 +74,12 @@ void TransactionCoordinatorTestFixture::tearDown() {
ShardServerTestFixture::tearDown();
}
-std::unique_ptr<ShardingCatalogClient> TransactionCoordinatorTestFixture::makeShardingCatalogClient(
- std::unique_ptr<DistLockManager> distLockManager) {
+std::unique_ptr<ShardingCatalogClient>
+TransactionCoordinatorTestFixture::makeShardingCatalogClient() {
class StaticCatalogClient final : public ShardingCatalogClientMock {
public:
- StaticCatalogClient(std::vector<ShardId> shardIds)
- : ShardingCatalogClientMock(nullptr), _shardIds(std::move(shardIds)) {}
+ StaticCatalogClient(std::vector<ShardId> shardIds) : _shardIds(std::move(shardIds)) {}
StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards(
OperationContext* opCtx, repl::ReadConcernLevel readConcern) override {
diff --git a/src/mongo/db/s/transaction_coordinator_test_fixture.h b/src/mongo/db/s/transaction_coordinator_test_fixture.h
index 37232f0d340..6a60538a9fa 100644
--- a/src/mongo/db/s/transaction_coordinator_test_fixture.h
+++ b/src/mongo/db/s/transaction_coordinator_test_fixture.h
@@ -53,8 +53,7 @@ protected:
* ShardRegistry reload is done over DBClient, not the NetworkInterface, and there is no
* DBClientMock analogous to the NetworkInterfaceMock.
*/
- std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient(
- std::unique_ptr<DistLockManager> distLockManager) override;
+ std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient() override;
void assertCommandSentAndRespondWith(const StringData& commandName,
const StatusWith<BSONObj>& response,
diff --git a/src/mongo/db/s/vector_clock_config_server_test.cpp b/src/mongo/db/s/vector_clock_config_server_test.cpp
index 6bc2d618594..81e8138c097 100644
--- a/src/mongo/db/s/vector_clock_config_server_test.cpp
+++ b/src/mongo/db/s/vector_clock_config_server_test.cpp
@@ -33,8 +33,8 @@
#include "mongo/db/keys_collection_manager.h"
#include "mongo/db/logical_time_validator.h"
#include "mongo/db/s/config/config_server_test_fixture.h"
+#include "mongo/db/s/dist_lock_manager_mock.h"
#include "mongo/db/vector_clock_mutable.h"
-#include "mongo/s/catalog/dist_lock_manager_mock.h"
#include "mongo/unittest/death_test.h"
#include "mongo/util/clock_source_mock.h"
@@ -71,10 +71,8 @@ protected:
// The VectorClock tests assume nothing else ticks ClusterTime. However,
// ConfigServerTestFixture installs an actual DistLockManager, which does writes (thereby
// ticking ClusterTime). So for these tests, that is overridden to be a mock.
- std::unique_ptr<DistLockManager> makeDistLockManager(
- std::unique_ptr<DistLockCatalog> distLockCatalog) override {
- invariant(distLockCatalog);
- return std::make_unique<DistLockManagerMock>(std::move(distLockCatalog));
+ std::unique_ptr<DistLockManager> makeDistLockManager() override {
+ return std::make_unique<DistLockManagerMock>();
}
/**