diff options
author | Jack Mulrow <jack.mulrow@mongodb.com> | 2022-03-16 13:54:52 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-03-16 15:17:37 +0000 |
commit | fd26d99c7700fcb4d7a8bfc0153ff425a389e0c4 (patch) | |
tree | d974c66ba1586d695ba26389c02e9f1d6363abd0 /src | |
parent | d173689149305c83f6c5e45878e54698694f4106 (diff) | |
download | mongo-fd26d99c7700fcb4d7a8bfc0153ff425a389e0c4.tar.gz |
SERVER-63495 Link cluster find and strategy into mongod
Diffstat (limited to 'src')
-rw-r--r-- | src/mongo/db/s/SConscript | 2 | ||||
-rw-r--r-- | src/mongo/db/s/cluster_find_cmd_d.cpp | 58 | ||||
-rw-r--r-- | src/mongo/s/SConscript | 4 | ||||
-rw-r--r-- | src/mongo/s/commands/SConscript | 32 | ||||
-rw-r--r-- | src/mongo/s/commands/cluster_command_test_fixture.cpp | 2 | ||||
-rw-r--r-- | src/mongo/s/commands/cluster_find_cmd.h (renamed from src/mongo/s/commands/cluster_find_cmd.cpp) | 96 | ||||
-rw-r--r-- | src/mongo/s/commands/cluster_find_cmd_s.cpp | 55 | ||||
-rw-r--r-- | src/mongo/s/grid.cpp | 6 | ||||
-rw-r--r-- | src/mongo/s/grid.h | 5 | ||||
-rw-r--r-- | src/mongo/s/mongos_topology_coordinator.cpp | 22 | ||||
-rw-r--r-- | src/mongo/s/mongos_topology_coordinator_test.cpp | 4 |
11 files changed, 220 insertions, 66 deletions
diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript index 8f37f180c1b..c2d83bdaedc 100644 --- a/src/mongo/db/s/SConscript +++ b/src/mongo/db/s/SConscript @@ -308,6 +308,7 @@ env.Library( 'cleanup_orphaned_cmd.cpp', 'clone_catalog_data_command.cpp', 'clone_collection_options_from_primary_shard_cmd.cpp', + 'cluster_find_cmd_d.cpp', 'collmod_coordinator.cpp', 'collmod_coordinator_document.idl', 'config/set_cluster_parameter_coordinator.cpp', @@ -426,6 +427,7 @@ env.Library( '$BUILD_DIR/mongo/db/timeseries/timeseries_conversion_util', '$BUILD_DIR/mongo/db/timeseries/timeseries_options', '$BUILD_DIR/mongo/idl/cluster_server_parameter', + '$BUILD_DIR/mongo/s/commands/cluster_commands_common', '$BUILD_DIR/mongo/s/commands/sharded_cluster_sharding_commands', '$BUILD_DIR/mongo/s/sharding_initialization', '$BUILD_DIR/mongo/s/sharding_router_api', diff --git a/src/mongo/db/s/cluster_find_cmd_d.cpp b/src/mongo/db/s/cluster_find_cmd_d.cpp new file mode 100644 index 00000000000..7bc45e4cc1d --- /dev/null +++ b/src/mongo/db/s/cluster_find_cmd_d.cpp @@ -0,0 +1,58 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/s/commands/cluster_find_cmd.h" + +namespace mongo { +namespace { + +/** + * Implements the cluster find command on mongod. + */ +struct ClusterFindCmdD { + static constexpr StringData kName = "clusterFind"_sd; + + static const std::set<std::string>& getApiVersions() { + return kNoApiVersions; + } + + static void doCheckAuthorization(OperationContext* opCtx, + bool hasTerm, + const NamespaceString& nss) { + uassert(ErrorCodes::Unauthorized, + "Unauthorized", + AuthorizationSession::get(opCtx->getClient()) + ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(), + ActionType::internal)); + } +}; +ClusterFindCmdBase<ClusterFindCmdD> clusterFindCmdD; + +} // namespace +} // namespace mongo diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript index 867de0bdedf..497f62be4d3 100644 --- a/src/mongo/s/SConscript +++ b/src/mongo/s/SConscript @@ -438,7 +438,7 @@ env.Library( '$BUILD_DIR/mongo/db/not_primary_error_tracker', '$BUILD_DIR/mongo/db/read_write_concern_defaults', '$BUILD_DIR/mongo/db/session_catalog', - 'commands/cluster_commands', + 'commands/cluster_commands_common', 'load_balancer_support', 'sharding_router_api', ], @@ -492,6 +492,7 @@ env.Library( '$BUILD_DIR/mongo/util/testing_options', '$BUILD_DIR/mongo/util/version_impl', 'commands/cluster_commands', + 'commands/cluster_commands_common', 'commands/sharded_cluster_commands', 'commands/sharded_cluster_sharding_commands', 'committed_optime_metadata_hook', @@ -542,6 +543,7 @@ env.Library( '$BUILD_DIR/mongo/util/signal_handlers', 'client/sharding_client', 'commands/cluster_commands', + 'commands/cluster_commands_common', 'committed_optime_metadata_hook', 'load_balancer_support', 'mongos_initializers', diff --git a/src/mongo/s/commands/SConscript b/src/mongo/s/commands/SConscript index 0a5e45d12ba..b394655d563 100644 --- a/src/mongo/s/commands/SConscript +++ b/src/mongo/s/commands/SConscript @@ -47,10 +47,9 @@ env.Library( 'cluster_drop_indexes_cmd.cpp', 'cluster_enable_sharding_cmd.cpp', 'cluster_explain_cmd.cpp', - 'cluster_explain.cpp', 'cluster_filemd5_cmd.cpp', 'cluster_find_and_modify_cmd.cpp', - 'cluster_find_cmd.cpp', + 'cluster_find_cmd_s.cpp', 'cluster_fsync_cmd.cpp', 'cluster_ftdc_commands.cpp', 'cluster_get_last_error_cmd.cpp', @@ -89,11 +88,9 @@ env.Library( 'cluster_validate_db_metadata_cmd.cpp', 'cluster_whats_my_uri_cmd.cpp', 'cluster_write_cmd.cpp', - 'document_shard_key_update_util.cpp', 'internal_transactions_test_commands.cpp', 'kill_sessions_remote.cpp', 's_read_write_concern_defaults_server_status.cpp', - 'strategy.cpp', 'cluster_commands.idl', 'internal_transactions_test_commands.idl', 'shard_collection.idl', @@ -150,9 +147,34 @@ env.Library( '$BUILD_DIR/mongo/s/sharding_router_api', '$BUILD_DIR/mongo/transport/message_compressor', '$BUILD_DIR/mongo/transport/transport_layer_common', + 'cluster_commands_common', ] ) +env.Library( + target='cluster_commands_common', + source=[ + 'cluster_explain.cpp', + 'document_shard_key_update_util.cpp', + 'strategy.cpp', + ], + LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/initialize_api_parameters', + '$BUILD_DIR/mongo/db/read_write_concern_defaults', + '$BUILD_DIR/mongo/db/repl/repl_server_parameters', + '$BUILD_DIR/mongo/db/shared_request_handling', + '$BUILD_DIR/mongo/db/stats/api_version_metrics', + '$BUILD_DIR/mongo/db/stats/counters', + '$BUILD_DIR/mongo/rpc/rewrite_state_change_errors', + '$BUILD_DIR/mongo/s/load_balancer_support', + '$BUILD_DIR/mongo/s/mongos_topology_coordinator', + '$BUILD_DIR/mongo/s/sharding_api', + '$BUILD_DIR/mongo/s/sharding_router_api', + '$BUILD_DIR/mongo/transport/message_compressor', + '$BUILD_DIR/mongo/transport/transport_layer_common', + ] +) + # These commands are linked in MongoS only # This library is currently also linked into mongoqd env.Library( @@ -193,6 +215,7 @@ env.Library( '$BUILD_DIR/mongo/s/vector_clock_mongos', '$BUILD_DIR/mongo/transport/transport_layer_common', 'cluster_commands', + 'cluster_commands_common', ] ) @@ -219,5 +242,6 @@ env.CppUnitTest( '$BUILD_DIR/mongo/s/sharding_router_test_fixture', '$BUILD_DIR/mongo/s/vector_clock_mongos', 'cluster_commands', + 'cluster_commands_common', ], ) diff --git a/src/mongo/s/commands/cluster_command_test_fixture.cpp b/src/mongo/s/commands/cluster_command_test_fixture.cpp index 49b3c7771e1..f18307d2169 100644 --- a/src/mongo/s/commands/cluster_command_test_fixture.cpp +++ b/src/mongo/s/commands/cluster_command_test_fixture.cpp @@ -52,6 +52,8 @@ void ClusterCommandTestFixture::setUp() { CatalogCacheTestFixture::setUp(); CatalogCacheTestFixture::setupNShards(numShards); + Grid::get(getServiceContext())->setShardingInitialized(); + // Set the initial clusterTime. VectorClock::get(getServiceContext())->advanceClusterTime_forTest(kInMemoryLogicalTime); diff --git a/src/mongo/s/commands/cluster_find_cmd.cpp b/src/mongo/s/commands/cluster_find_cmd.h index 46374f2e505..7c716729e6c 100644 --- a/src/mongo/s/commands/cluster_find_cmd.cpp +++ b/src/mongo/s/commands/cluster_find_cmd.h @@ -1,5 +1,5 @@ /** - * Copyright (C) 2018-present MongoDB, Inc. + * Copyright (C) 2022-present MongoDB, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the Server Side Public License, version 1, @@ -27,7 +27,7 @@ * it in the license file. */ -#include "mongo/platform/basic.h" +#pragma once #include <boost/optional.hpp> @@ -44,55 +44,24 @@ #include "mongo/s/cluster_commands_helpers.h" #include "mongo/s/commands/cluster_explain.h" #include "mongo/s/grid.h" +#include "mongo/s/is_mongos.h" #include "mongo/s/query/cluster_aggregate.h" #include "mongo/s/query/cluster_find.h" namespace mongo { -namespace { - -using std::string; -using std::unique_ptr; -using std::vector; - -const char kTermField[] = "term"; - -// Parses the command object to a FindCommandRequest, validates that no runtime constants were -// supplied with the command, and sets the constant runtime values that will be forwarded to each -// shard. -std::unique_ptr<FindCommandRequest> parseCmdObjectToFindCommandRequest(OperationContext* opCtx, - NamespaceString nss, - BSONObj cmdObj) { - auto findCommand = query_request_helper::makeFromFindCommand( - std::move(cmdObj), - std::move(nss), - APIParameters::get(opCtx).getAPIStrict().value_or(false)); - if (!findCommand->getReadConcern()) { - if (opCtx->isStartingMultiDocumentTransaction() || !opCtx->inMultiDocumentTransaction()) { - // If there is no explicit readConcern in the cmdObj, and this is either the first - // operation in a transaction, or not running in a transaction, then use the readConcern - // from the opCtx (which may be a cluster-wide default). - const auto& readConcernArgs = repl::ReadConcernArgs::get(opCtx); - findCommand->setReadConcern(readConcernArgs.toBSONInner()); - } - } - uassert(51202, - "Cannot specify runtime constants option to a mongos", - !findCommand->getLegacyRuntimeConstants()); - uassert(5746101, - "Cannot specify ntoreturn in a find command against mongos", - findCommand->getNtoreturn() == boost::none); - return findCommand; -} /** * Implements the find command on mongos. */ -class ClusterFindCmd final : public Command { +template <typename Impl> +class ClusterFindCmdBase final : public Command { public: - ClusterFindCmd() : Command("find") {} + static constexpr StringData kTermField = "term"_sd; + + ClusterFindCmdBase() : Command(Impl::kName) {} const std::set<std::string>& apiVersions() const { - return kApiVersions1; + return Impl::getApiVersions(); } std::unique_ptr<CommandInvocation> parse(OperationContext* opCtx, @@ -123,7 +92,9 @@ public: class Invocation final : public CommandInvocation { public: - Invocation(const ClusterFindCmd* definition, const OpMsgRequest& request, StringData dbName) + Invocation(const ClusterFindCmdBase* definition, + const OpMsgRequest& request, + StringData dbName) : CommandInvocation(definition), _request(request), _dbName(dbName) {} private: @@ -148,15 +119,14 @@ public: */ void doCheckAuthorization(OperationContext* opCtx) const final { auto hasTerm = _request.body.hasField(kTermField); - uassertStatusOK(auth::checkAuthForFind( - AuthorizationSession::get(opCtx->getClient()), ns(), hasTerm)); + Impl::doCheckAuthorization(opCtx, hasTerm, ns()); } void explain(OperationContext* opCtx, ExplainOptions::Verbosity verbosity, rpc::ReplyBuilderInterface* result) override { // Parse the command BSON to a FindCommandRequest. - auto findCommand = parseCmdObjectToFindCommandRequest(opCtx, ns(), _request.body); + auto findCommand = _parseCmdObjectToFindCommandRequest(opCtx, ns(), _request.body); try { const auto explainCmd = @@ -224,12 +194,14 @@ public: // We count find command as a query op. globalOpCounters.gotQuery(); + Grid::get(opCtx)->assertShardingIsInitialized(); + ON_BLOCK_EXIT([opCtx] { Grid::get(opCtx)->catalogCache()->checkAndRecordOperationBlockedByRefresh( opCtx, mongo::LogicalOp::opQuery); }); - auto findCommand = parseCmdObjectToFindCommandRequest(opCtx, ns(), _request.body); + auto findCommand = _parseCmdObjectToFindCommandRequest(opCtx, ns(), _request.body); const boost::intrusive_ptr<ExpressionContext> expCtx; auto cq = uassertStatusOK( @@ -287,11 +259,39 @@ public: } private: + /** + * Parses the command object to a FindCommandRequest, validates that no runtime constants + * were supplied with the command, and sets the constant runtime values that will be + * forwarded to each shard. + */ + static std::unique_ptr<FindCommandRequest> _parseCmdObjectToFindCommandRequest( + OperationContext* opCtx, NamespaceString nss, BSONObj cmdObj) { + auto findCommand = query_request_helper::makeFromFindCommand( + std::move(cmdObj), + std::move(nss), + APIParameters::get(opCtx).getAPIStrict().value_or(false)); + if (!findCommand->getReadConcern()) { + if (opCtx->isStartingMultiDocumentTransaction() || + !opCtx->inMultiDocumentTransaction()) { + // If there is no explicit readConcern in the cmdObj, and this is either the + // first operation in a transaction, or not running in a transaction, then use + // the readConcern from the opCtx (which may be a cluster-wide default). + const auto& readConcernArgs = repl::ReadConcernArgs::get(opCtx); + findCommand->setReadConcern(readConcernArgs.toBSONInner()); + } + } + uassert(51202, + "Cannot specify runtime constants option to a mongos", + !findCommand->getLegacyRuntimeConstants()); + uassert(5746101, + "Cannot specify ntoreturn in a find command against mongos", + findCommand->getNtoreturn() == boost::none); + return findCommand; + } + const OpMsgRequest& _request; const StringData _dbName; }; +}; -} cmdFindCluster; - -} // namespace } // namespace mongo diff --git a/src/mongo/s/commands/cluster_find_cmd_s.cpp b/src/mongo/s/commands/cluster_find_cmd_s.cpp new file mode 100644 index 00000000000..03402dc3c30 --- /dev/null +++ b/src/mongo/s/commands/cluster_find_cmd_s.cpp @@ -0,0 +1,55 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/s/commands/cluster_find_cmd.h" + +namespace mongo { +namespace { + +/** + * Implements the cluster find command on mongos. + */ +struct ClusterFindCmdS { + static constexpr StringData kName = "find"_sd; + + static const std::set<std::string>& getApiVersions() { + return kApiVersions1; + } + + static void doCheckAuthorization(OperationContext* opCtx, + bool hasTerm, + const NamespaceString& nss) { + uassertStatusOK( + auth::checkAuthForFind(AuthorizationSession::get(opCtx->getClient()), nss, hasTerm)); + } +}; +ClusterFindCmdBase<ClusterFindCmdS> clusterFindCmdS; + +} // namespace +} // namespace mongo diff --git a/src/mongo/s/grid.cpp b/src/mongo/s/grid.cpp index 6a2efb9d8c4..497dc3b0228 100644 --- a/src/mongo/s/grid.cpp +++ b/src/mongo/s/grid.cpp @@ -82,6 +82,12 @@ bool Grid::isShardingInitialized() const { return _shardingInitialized.load(); } +void Grid::assertShardingIsInitialized() const { + uassert(ErrorCodes::ShardingStateNotInitialized, + "Sharding is not enabled", + isShardingInitialized()); +} + void Grid::setShardingInitialized() { invariant(!_shardingInitialized.load()); _shardingInitialized.store(true); diff --git a/src/mongo/s/grid.h b/src/mongo/s/grid.h index 079ca621a6b..53abf66e483 100644 --- a/src/mongo/s/grid.h +++ b/src/mongo/s/grid.h @@ -85,6 +85,11 @@ public: bool isShardingInitialized() const; /** + * Throws if sharding is not initialized. + */ + void assertShardingIsInitialized() const; + + /** * Used to indicate the sharding initialization process is complete. Should only be called once * in the lifetime of a server. Protected by an atomic access guard. */ diff --git a/src/mongo/s/mongos_topology_coordinator.cpp b/src/mongo/s/mongos_topology_coordinator.cpp index a106d7019f0..d78aa14c764 100644 --- a/src/mongo/s/mongos_topology_coordinator.cpp +++ b/src/mongo/s/mongos_topology_coordinator.cpp @@ -54,13 +54,13 @@ MONGO_INITIALIZER(GenerateMongosInstanceId)(InitializerContext*) { } // Signals that a hello request has started waiting. -MONGO_FAIL_POINT_DEFINE(waitForHelloResponse); +MONGO_FAIL_POINT_DEFINE(waitForHelloResponseMongos); // Awaitable hello requests with the proper topologyVersions are expected to wait for // maxAwaitTimeMS on mongos. When set, this failpoint will hang right before waiting on a // topology change. -MONGO_FAIL_POINT_DEFINE(hangWhileWaitingForHelloResponse); +MONGO_FAIL_POINT_DEFINE(hangWhileWaitingForHelloResponseMongos); // Failpoint for hanging during quiesce mode on mongos. -MONGO_FAIL_POINT_DEFINE(hangDuringQuiesceMode); +MONGO_FAIL_POINT_DEFINE(hangDuringQuiesceModeMongos); // Simulates returning a specified error in the hello response. MONGO_FAIL_POINT_DEFINE(setCustomErrorInHelloResponseMongoS); @@ -154,15 +154,15 @@ std::shared_ptr<const MongosHelloResponse> MongosTopologyCoordinator::awaitHello HelloMetrics::get(opCtx)->incrementNumAwaitingTopologyChanges(); lk.unlock(); - if (MONGO_unlikely(waitForHelloResponse.shouldFail())) { + if (MONGO_unlikely(waitForHelloResponseMongos.shouldFail())) { // Used in tests that wait for this failpoint to be entered before shutting down mongos, // which is the only action that triggers a topology change. - LOGV2(4695704, "waitForHelloResponse failpoint enabled"); + LOGV2(4695704, "waitForHelloResponseMongos failpoint enabled"); } - if (MONGO_unlikely(hangWhileWaitingForHelloResponse.shouldFail())) { - LOGV2(4695501, "hangWhileWaitingForHelloResponse failpoint enabled"); - hangWhileWaitingForHelloResponse.pauseWhileSet(opCtx); + if (MONGO_unlikely(hangWhileWaitingForHelloResponseMongos.shouldFail())) { + LOGV2(4695501, "hangWhileWaitingForHelloResponseMongos failpoint enabled"); + hangWhileWaitingForHelloResponseMongos.pauseWhileSet(opCtx); } // Wait for a mongos topology change with timeout set to deadline. @@ -232,9 +232,9 @@ void MongosTopologyCoordinator::enterQuiesceModeAndWait(OperationContext* opCtx, HelloMetrics::get(getGlobalServiceContext())->resetNumAwaitingTopologyChanges(); } - if (MONGO_unlikely(hangDuringQuiesceMode.shouldFail())) { - LOGV2(4695700, "hangDuringQuiesceMode failpoint enabled"); - hangDuringQuiesceMode.pauseWhileSet(opCtx); + if (MONGO_unlikely(hangDuringQuiesceModeMongos.shouldFail())) { + LOGV2(4695700, "hangDuringQuiesceModeMongos failpoint enabled"); + hangDuringQuiesceModeMongos.pauseWhileSet(opCtx); } LOGV2(4695701, "Entering quiesce mode for mongos shutdown", "quiesceTime"_attr = quiesceTime); diff --git a/src/mongo/s/mongos_topology_coordinator_test.cpp b/src/mongo/s/mongos_topology_coordinator_test.cpp index 2572b0af613..b9fc86d6fd5 100644 --- a/src/mongo/s/mongos_topology_coordinator_test.cpp +++ b/src/mongo/s/mongos_topology_coordinator_test.cpp @@ -272,7 +272,7 @@ TEST_F(MongosTopoCoordTest, HelloReturnsErrorOnEnteringQuiesceMode) { auto quiesceTime = Milliseconds(0); // This will cause the hello request to hang. - auto waitForHelloFailPoint = globalFailPointRegistry().find("waitForHelloResponse"); + auto waitForHelloFailPoint = globalFailPointRegistry().find("waitForHelloResponseMongos"); auto timesEnteredFailPoint = waitForHelloFailPoint->setMode(FailPoint::alwaysOn); ON_BLOCK_EXIT([&] { waitForHelloFailPoint->setMode(FailPoint::off, 0); }); stdx::thread getHelloThread([&] { @@ -299,7 +299,7 @@ TEST_F(MongosTopoCoordTest, AlwaysDecrementNumAwaitingTopologyChangesOnErrorMong auto opCtx = makeOperationContext(); ASSERT_EQUALS(0, HelloMetrics::get(opCtx.get())->getNumAwaitingTopologyChanges()); - auto hangFP = globalFailPointRegistry().find("hangWhileWaitingForHelloResponse"); + auto hangFP = globalFailPointRegistry().find("hangWhileWaitingForHelloResponseMongos"); auto timesEnteredHangFP = hangFP->setMode(FailPoint::alwaysOn); // Use a novel error code to test this functionality. |