diff options
author | huayu-ouyang <huayu.ouyang@mongodb.com> | 2020-09-17 22:28:41 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-09-24 18:13:58 +0000 |
commit | f44ef61bcfe5a58fefd3617040fd1cbf0534594d (patch) | |
tree | b57ca1cbf078bd2b0cff8e36dfb13821abf687ab | |
parent | 68fe070e7f60879ccbe45e1f6ec953bb99d95075 (diff) | |
download | mongo-f44ef61bcfe5a58fefd3617040fd1cbf0534594d.tar.gz |
SERVER-50418 Add exhaustHello to serverStatus
-rw-r--r-- | src/mongo/db/repl/replication_info.cpp | 7 | ||||
-rw-r--r-- | src/mongo/db/service_entry_point_common.cpp | 3 | ||||
-rw-r--r-- | src/mongo/rpc/op_msg_integration_test.cpp | 317 | ||||
-rw-r--r-- | src/mongo/s/commands/cluster_is_master_cmd.cpp | 7 | ||||
-rw-r--r-- | src/mongo/s/commands/strategy.cpp | 2 | ||||
-rw-r--r-- | src/mongo/transport/ismaster_metrics.cpp | 48 | ||||
-rw-r--r-- | src/mongo/transport/ismaster_metrics.h | 10 | ||||
-rw-r--r-- | src/mongo/transport/service_entry_point_impl.cpp | 2 |
8 files changed, 305 insertions, 91 deletions
diff --git a/src/mongo/db/repl/replication_info.cpp b/src/mongo/db/repl/replication_info.cpp index e4ad5980808..37082c64b62 100644 --- a/src/mongo/db/repl/replication_info.cpp +++ b/src/mongo/db/repl/replication_info.cpp @@ -464,15 +464,16 @@ public: saslMechanismRegistry.advertiseMechanismNamesForUser(opCtx, cmdObj, &result); if (opCtx->isExhaust()) { - LOGV2_DEBUG(23905, 3, "Using exhaust for isMaster protocol"); + LOGV2_DEBUG(23905, 3, "Using exhaust for isMaster or hello protocol"); uassert(51756, - "An isMaster request with exhaust must specify 'maxAwaitTimeMS'", + "An isMaster or hello request with exhaust must specify 'maxAwaitTimeMS'", maxAwaitTimeMSField); invariant(clientTopologyVersion); InExhaustIsMaster::get(opCtx->getClient()->session().get()) - ->setInExhaustIsMaster(true /* inExhaustIsMaster */); + ->setInExhaustIsMaster(true /* inExhaust */, + cmdObj.firstElementFieldNameStringData()); if (clientTopologyVersion->getProcessId() == currentTopologyVersion.getProcessId() && clientTopologyVersion->getCounter() == currentTopologyVersion.getCounter()) { diff --git a/src/mongo/db/service_entry_point_common.cpp b/src/mongo/db/service_entry_point_common.cpp index 442c65be972..bea822092d3 100644 --- a/src/mongo/db/service_entry_point_common.cpp +++ b/src/mongo/db/service_entry_point_common.cpp @@ -1452,7 +1452,8 @@ DbResponse receivedCommands(OperationContext* opCtx, if (session) { if (!opCtx->isExhaust() || (c->getName() != "hello"_sd && c->getName() != "isMaster"_sd)) { - InExhaustIsMaster::get(session.get())->setInExhaustIsMaster(false); + InExhaustIsMaster::get(session.get()) + ->setInExhaustIsMaster(false, request.getCommandName()); } } diff --git a/src/mongo/rpc/op_msg_integration_test.cpp b/src/mongo/rpc/op_msg_integration_test.cpp index a2a380c4f9a..15cb888462c 100644 --- a/src/mongo/rpc/op_msg_integration_test.cpp +++ b/src/mongo/rpc/op_msg_integration_test.cpp @@ -709,29 +709,34 @@ TEST(OpMsg, ServerRejectsExhaustIsMasterWithoutMaxAwaitTimeMS) { ASSERT_NOT_OK(getStatusFromCommandResult(res)); } -TEST(OpMsg, ServerStatusCorrectlyShowsExhaustIsMasterMetrics) { +void serverStatusCorrectlyShowsExhaustMetrics(std::string commandName) { std::string errMsg; auto conn = std::unique_ptr<DBClientBase>( unittest::getFixtureConnectionString().connect("integration_test", errMsg)); uassert(ErrorCodes::SocketException, errMsg, conn); if (conn->isReplicaSetMember()) { - // Don't run on replica sets as the RSM will use the streamable isMaster protocol by - // default. This can cause inconsistencies in our metrics tests. + // Don't run on replica sets as the RSM will use the streamable hello or isMaster protocol + // by default. This can cause inconsistencies in our metrics tests. return; } - // Wait for stale exhuast streams to finish closing before testing the exhaust isMaster metrics. + bool useLegacyCommandName = (commandName != "hello"); + // Wait for stale exhuast streams to finish closing before testing the exhaust hello or metrics. ASSERT(waitForCondition([&] { auto serverStatusCmd = BSON("serverStatus" << 1); BSONObj serverStatusReply; ASSERT(conn->runCommand("admin", serverStatusCmd, serverStatusReply)); - return serverStatusReply["connections"]["exhaustIsMaster"].numberInt() == 0; + if (useLegacyCommandName) { + return serverStatusReply["connections"]["exhaustIsMaster"].numberInt() == 0; + } else { + return serverStatusReply["connections"]["exhaustHello"].numberInt() == 0; + } })); - // Issue an isMaster command without a topology version. - auto isMasterCmd = BSON("isMaster" << 1); - auto opMsgRequest = OpMsgRequest::fromDBAndBody("admin", isMasterCmd); + // Issue a hello or isMaster command without a topology version. + auto cmd = BSON(commandName << 1); + auto opMsgRequest = OpMsgRequest::fromDBAndBody("admin", cmd); auto request = opMsgRequest.serialize(); Message reply; @@ -741,13 +746,12 @@ TEST(OpMsg, ServerStatusCorrectlyShowsExhaustIsMasterMetrics) { auto topologyVersion = res["topologyVersion"].Obj().getOwned(); ASSERT(!OpMsg::isFlagSet(reply, OpMsg::kMoreToCome)); - isMasterCmd = - BSON("isMaster" << 1 << "topologyVersion" << topologyVersion << "maxAwaitTimeMS" << 100); - opMsgRequest = OpMsgRequest::fromDBAndBody("admin", isMasterCmd); + cmd = BSON(commandName << 1 << "topologyVersion" << topologyVersion << "maxAwaitTimeMS" << 100); + opMsgRequest = OpMsgRequest::fromDBAndBody("admin", cmd); request = opMsgRequest.serialize(); OpMsg::setFlag(&request, OpMsg::kExhaustSupported); - // Run isMaster command to initiate the exhaust stream. + // Run hello or isMaster command to initiate the exhaust stream. ASSERT(conn->call(request, reply)); ASSERT(OpMsg::isFlagSet(reply, OpMsg::kMoreToCome)); res = OpMsg::parse(reply).body; @@ -762,52 +766,78 @@ TEST(OpMsg, ServerStatusCorrectlyShowsExhaustIsMasterMetrics) { auto serverStatusCmd = BSON("serverStatus" << 1); BSONObj serverStatusReply; ASSERT(conn2->runCommand("admin", serverStatusCmd, serverStatusReply)); - ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); - + if (useLegacyCommandName) { + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } else { + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } // The exhaust stream would continue indefinitely. } +TEST(OpMsg, ServerStatusCorrectlyShowsExhaustIsMasterMetrics) { + return serverStatusCorrectlyShowsExhaustMetrics("isMaster"); +} + +TEST(OpMsg, ServerStatusCorrectlyShowsExhaustHelloMetrics) { + return serverStatusCorrectlyShowsExhaustMetrics("hello"); +} + TEST(OpMsg, ServerStatusCorrectlyShowsExhaustIsMasterMetricsWithIsMasterAlias) { + return serverStatusCorrectlyShowsExhaustMetrics("ismaster"); +} + +void exhaustMetricSwitchingCommandNames(bool useLegacyCommandNameAtStart) { std::string errMsg; - auto conn = std::unique_ptr<DBClientBase>( - unittest::getFixtureConnectionString().connect("integration_test", errMsg)); - uassert(ErrorCodes::SocketException, errMsg, conn); + const auto conn1AppName = "integration_test"; + auto conn1 = std::unique_ptr<DBClientBase>( + unittest::getFixtureConnectionString().connect(conn1AppName, errMsg)); + uassert(ErrorCodes::SocketException, errMsg, conn1); - if (conn->isReplicaSetMember()) { - // Don't run on replica sets as the RSM will use the streamable isMaster protocol by - // default. This can cause inconsistencies in our metrics tests. + if (conn1->isReplicaSetMember()) { + // Don't run on replica sets as the RSM will use the streamable hello or isMaster protocol + // by default. This can cause inconsistencies in our metrics tests. return; } - // Wait for stale exhuast streams to finish closing before testing the exhaust isMaster metrics. + // Wait for stale exhuast streams to finish closing before testing the exhaust metrics. ASSERT(waitForCondition([&] { auto serverStatusCmd = BSON("serverStatus" << 1); BSONObj serverStatusReply; - ASSERT(conn->runCommand("admin", serverStatusCmd, serverStatusReply)); - return serverStatusReply["connections"]["exhaustIsMaster"].numberInt() == 0; + ASSERT(conn1->runCommand("admin", serverStatusCmd, serverStatusReply)); + if (useLegacyCommandNameAtStart) { + return serverStatusReply["connections"]["exhaustIsMaster"].numberInt() == 0; + } else { + return serverStatusReply["connections"]["exhaustHello"].numberInt() == 0; + } })); - // Issue an isMaster command with the "ismaster" alias without a topology version. - auto lowerCaseIsMasterCmd = BSON("ismaster" << 1); - auto opMsgRequest = OpMsgRequest::fromDBAndBody("admin", lowerCaseIsMasterCmd); + // Issue a hello or isMaster command without a topology version. + std::string cmdName = "hello"; + if (useLegacyCommandNameAtStart) { + cmdName = "isMaster"; + } + // Issue a hello or isMaster command without a topology version. + auto cmd = BSON(cmdName << 1); + auto opMsgRequest = OpMsgRequest::fromDBAndBody("admin", cmd); auto request = opMsgRequest.serialize(); Message reply; - ASSERT(conn->call(request, reply)); + ASSERT(conn1->call(request, reply)); auto res = OpMsg::parse(reply).body; ASSERT_OK(getStatusFromCommandResult(res)); auto topologyVersion = res["topologyVersion"].Obj().getOwned(); ASSERT(!OpMsg::isFlagSet(reply, OpMsg::kMoreToCome)); - lowerCaseIsMasterCmd = - BSON("ismaster" << 1 << "topologyVersion" << topologyVersion << "maxAwaitTimeMS" << 100); - opMsgRequest = OpMsgRequest::fromDBAndBody("admin", lowerCaseIsMasterCmd); + cmd = BSON(cmdName << 1 << "topologyVersion" << topologyVersion << "maxAwaitTimeMS" << 100); + opMsgRequest = OpMsgRequest::fromDBAndBody("admin", cmd); request = opMsgRequest.serialize(); OpMsg::setFlag(&request, OpMsg::kExhaustSupported); - // Run the isMaster command with the "ismaster" alias. The aliased command should work - // identically to the default "isMaster" command name and initiate the the exhaust stream. - ASSERT(conn->call(request, reply)); + // Run hello or isMaster command to initiate the exhaust stream. + ASSERT(conn1->call(request, reply)); + auto lastRequestId = reply.header().getId(); ASSERT(OpMsg::isFlagSet(reply, OpMsg::kMoreToCome)); res = OpMsg::parse(reply).body; ASSERT_OK(getStatusFromCommandResult(res)); @@ -815,18 +845,92 @@ TEST(OpMsg, ServerStatusCorrectlyShowsExhaustIsMasterMetricsWithIsMasterAlias) { // Start a new connection to the server to check the serverStatus metrics. std::string newErrMsg; auto conn2 = std::unique_ptr<DBClientBase>( - unittest::getFixtureConnectionString().connect("integration_test", newErrMsg)); + unittest::getFixtureConnectionString().connect("integration_test2", newErrMsg)); uassert(ErrorCodes::SocketException, newErrMsg, conn2); + std::string threadName; + ASSERT(waitForCondition([&] { + threadName = getThreadNameByAppName(conn2.get(), conn1AppName); + return !threadName.empty(); + })); + auto serverStatusCmd = BSON("serverStatus" << 1); BSONObj serverStatusReply; ASSERT(conn2->runCommand("admin", serverStatusCmd, serverStatusReply)); - ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + if (useLegacyCommandNameAtStart) { + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } else { + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } - // The exhaust stream would continue indefinitely. + const auto failPointObj = BSON("configureFailPoint" + << "failCommand" + << "mode" << BSON("times" << 1) << "data" + << BSON("threadName" << threadName << "errorCode" + << ErrorCodes::NotWritablePrimary + << "failCommands" << BSON_ARRAY(cmdName))); + auto response = conn2->runCommand(OpMsgRequest::fromDBAndBody("admin", failPointObj)); + ASSERT_OK(getStatusFromCommandResult(response->getCommandReply())); + + // Wait for the exhaust stream to close from the error returned by hello or isMaster. + ASSERT(waitForCondition([&] { + const auto status = conn1->recv(reply, lastRequestId); + lastRequestId = reply.header().getId(); + res = OpMsg::parse(reply).body; + return !getStatusFromCommandResult(res).isOK(); + })); + + // Terminating the exhaust stream should not decrement the number of exhaust connections. + ASSERT(conn2->runCommand("admin", serverStatusCmd, serverStatusReply)); + if (useLegacyCommandNameAtStart) { + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } else { + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } + + // running a different command on conn1 to initiate a new exhaust stream. + std::string newCmdName = "isMaster"; + if (useLegacyCommandNameAtStart) { + newCmdName = "hello"; + } + std::cout << newCmdName; + auto newCmd = + BSON(newCmdName << 1 << "topologyVersion" << topologyVersion << "maxAwaitTimeMS" << 100); + opMsgRequest = OpMsgRequest::fromDBAndBody("admin", newCmd); + request = opMsgRequest.serialize(); + OpMsg::setFlag(&request, OpMsg::kExhaustSupported); + + ASSERT(conn1->call(request, reply)); + ASSERT(OpMsg::isFlagSet(reply, OpMsg::kMoreToCome)); + res = OpMsg::parse(reply).body; + ASSERT_OK(getStatusFromCommandResult(res)); + + // exhaust metric should decrease for the exhaust type that was closed, and increase for the + // exhaust type that was just opened. + ASSERT(conn2->runCommand("admin", serverStatusCmd, serverStatusReply)); + if (useLegacyCommandNameAtStart) { + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } else { + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } } -TEST(OpMsg, ExhaustIsMasterMetricDecrementsOnNewOpAfterTerminatingExhaustStream) { +TEST(OpMsg, ExhaustIsMasterMetricSwitchingCommandNames) { + return exhaustMetricSwitchingCommandNames(true); +} + +TEST(OpMsg, ExhaustHelloMetricSwitchingCommandNames) { + return exhaustMetricSwitchingCommandNames(false); +} + + +void exhaustMetricDecrementsOnNewOpAfterTerminatingExhaustStream(bool useLegacyCommandName) { std::string errMsg; const auto conn1AppName = "integration_test"; auto conn1 = std::unique_ptr<DBClientBase>( @@ -834,22 +938,30 @@ TEST(OpMsg, ExhaustIsMasterMetricDecrementsOnNewOpAfterTerminatingExhaustStream) uassert(ErrorCodes::SocketException, errMsg, conn1); if (conn1->isReplicaSetMember()) { - // Don't run on replica sets as the RSM will use the streamable isMaster protocol by - // default. This can cause inconsistencies in our metrics tests. + // Don't run on replica sets as the RSM will use the streamable hello or isMaster protocol + // by default. This can cause inconsistencies in our metrics tests. return; } - // Wait for stale exhuast streams to finish closing before testing the exhaust isMaster metrics. + // Wait for stale exhuast streams to finish closing before testing the exhaust metrics. ASSERT(waitForCondition([&] { auto serverStatusCmd = BSON("serverStatus" << 1); BSONObj serverStatusReply; ASSERT(conn1->runCommand("admin", serverStatusCmd, serverStatusReply)); - return serverStatusReply["connections"]["exhaustIsMaster"].numberInt() == 0; + if (useLegacyCommandName) { + return serverStatusReply["connections"]["exhaustIsMaster"].numberInt() == 0; + } else { + return serverStatusReply["connections"]["exhaustHello"].numberInt() == 0; + } })); - // Issue an isMaster command without a topology version. - auto isMasterCmd = BSON("isMaster" << 1); - auto opMsgRequest = OpMsgRequest::fromDBAndBody("admin", isMasterCmd); + // Issue a hello or isMaster command without a topology version. + std::string cmdName = "hello"; + if (useLegacyCommandName) { + cmdName = "isMaster"; + } + auto cmd = BSON(cmdName << 1); + auto opMsgRequest = OpMsgRequest::fromDBAndBody("admin", cmd); auto request = opMsgRequest.serialize(); Message reply; @@ -859,13 +971,12 @@ TEST(OpMsg, ExhaustIsMasterMetricDecrementsOnNewOpAfterTerminatingExhaustStream) auto topologyVersion = res["topologyVersion"].Obj().getOwned(); ASSERT(!OpMsg::isFlagSet(reply, OpMsg::kMoreToCome)); - isMasterCmd = - BSON("isMaster" << 1 << "topologyVersion" << topologyVersion << "maxAwaitTimeMS" << 100); - opMsgRequest = OpMsgRequest::fromDBAndBody("admin", isMasterCmd); + cmd = BSON(cmdName << 1 << "topologyVersion" << topologyVersion << "maxAwaitTimeMS" << 100); + opMsgRequest = OpMsgRequest::fromDBAndBody("admin", cmd); request = opMsgRequest.serialize(); OpMsg::setFlag(&request, OpMsg::kExhaustSupported); - // Run isMaster command to initiate the exhaust stream. + // Run hello or isMaster command to initiate the exhaust stream. ASSERT(conn1->call(request, reply)); auto lastRequestId = reply.header().getId(); ASSERT(OpMsg::isFlagSet(reply, OpMsg::kMoreToCome)); @@ -887,19 +998,24 @@ TEST(OpMsg, ExhaustIsMasterMetricDecrementsOnNewOpAfterTerminatingExhaustStream) auto serverStatusCmd = BSON("serverStatus" << 1); BSONObj serverStatusReply; ASSERT(conn2->runCommand("admin", serverStatusCmd, serverStatusReply)); - ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + if (useLegacyCommandName) { + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } else { + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } const auto failPointObj = BSON("configureFailPoint" << "failCommand" << "mode" << BSON("times" << 1) << "data" << BSON("threadName" << threadName << "errorCode" << ErrorCodes::NotWritablePrimary - << "failCommands" - << BSON_ARRAY("isMaster"))); + << "failCommands" << BSON_ARRAY(cmdName))); auto response = conn2->runCommand(OpMsgRequest::fromDBAndBody("admin", failPointObj)); ASSERT_OK(getStatusFromCommandResult(response->getCommandReply())); - // Wait for the exhaust stream to close from the error returned by isMaster. + // Wait for the exhaust stream to close from the error returned by hello or isMaster. ASSERT(waitForCondition([&] { const auto status = conn1->recv(reply, lastRequestId); lastRequestId = reply.header().getId(); @@ -907,17 +1023,32 @@ TEST(OpMsg, ExhaustIsMasterMetricDecrementsOnNewOpAfterTerminatingExhaustStream) return !getStatusFromCommandResult(res).isOK(); })); - // Terminating the exhaust stream should not decrement the number of 'exhaustIsMaster'. + // Terminating the exhaust stream should not decrement the number of exhaust connections. ASSERT(conn2->runCommand("admin", serverStatusCmd, serverStatusReply)); - ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + if (useLegacyCommandName) { + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } else { + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } - // 'exhaustIsMaster' should now decrement after calling serverStatus on the connection that used + // exhaust metric should now decrement after calling serverStatus on the connection that used // to have the exhaust stream. ASSERT(conn1->runCommand("admin", serverStatusCmd, serverStatusReply)); ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustHello"].numberInt()); } -TEST(OpMsg, ExhaustIsMasterMetricOnNewExhaustIsMasterAfterTerminatingExhaustStream) { +TEST(OpMsg, ExhaustIsMasterMetricDecrementsOnNewOpAfterTerminatingExhaustStream) { + return exhaustMetricDecrementsOnNewOpAfterTerminatingExhaustStream(true); +} + +TEST(OpMsg, ExhaustHelloMetricDecrementsOnNewOpAfterTerminatingExhaustStream) { + return exhaustMetricDecrementsOnNewOpAfterTerminatingExhaustStream(false); +} + +void exhaustMetricOnNewExhaustAfterTerminatingExhaustStream(bool useLegacyCommandName) { std::string errMsg; const auto conn1AppName = "integration_test"; auto conn1 = std::unique_ptr<DBClientBase>( @@ -925,22 +1056,30 @@ TEST(OpMsg, ExhaustIsMasterMetricOnNewExhaustIsMasterAfterTerminatingExhaustStre uassert(ErrorCodes::SocketException, errMsg, conn1); if (conn1->isReplicaSetMember()) { - // Don't run on replica sets as the RSM will use the streamable isMaster protocol by - // default. This can cause inconsistencies in our metrics tests. + // Don't run on replica sets as the RSM will use the streamable hello or isMaster protocol + // by default. This can cause inconsistencies in our metrics tests. return; } - // Wait for stale exhuast streams to finish closing before testing the exhaust isMaster metrics. + // Wait for stale exhuast streams to finish closing before testing the exhaust metrics. ASSERT(waitForCondition([&] { auto serverStatusCmd = BSON("serverStatus" << 1); BSONObj serverStatusReply; ASSERT(conn1->runCommand("admin", serverStatusCmd, serverStatusReply)); - return serverStatusReply["connections"]["exhaustIsMaster"].numberInt() == 0; + if (useLegacyCommandName) { + return serverStatusReply["connections"]["exhaustIsMaster"].numberInt() == 0; + } else { + return serverStatusReply["connections"]["exhaustHello"].numberInt() == 0; + } })); - // Issue an isMaster command without a topology version. - auto isMasterCmd = BSON("isMaster" << 1); - auto opMsgRequest = OpMsgRequest::fromDBAndBody("admin", isMasterCmd); + // Issue a hello or isMaster command without a topology version. + std::string cmdName = "hello"; + if (useLegacyCommandName) { + cmdName = "isMaster"; + } + auto cmd = BSON(cmdName << 1); + auto opMsgRequest = OpMsgRequest::fromDBAndBody("admin", cmd); auto request = opMsgRequest.serialize(); Message reply; @@ -950,13 +1089,12 @@ TEST(OpMsg, ExhaustIsMasterMetricOnNewExhaustIsMasterAfterTerminatingExhaustStre auto topologyVersion = res["topologyVersion"].Obj().getOwned(); ASSERT(!OpMsg::isFlagSet(reply, OpMsg::kMoreToCome)); - isMasterCmd = - BSON("isMaster" << 1 << "topologyVersion" << topologyVersion << "maxAwaitTimeMS" << 100); - opMsgRequest = OpMsgRequest::fromDBAndBody("admin", isMasterCmd); + cmd = BSON(cmdName << 1 << "topologyVersion" << topologyVersion << "maxAwaitTimeMS" << 100); + opMsgRequest = OpMsgRequest::fromDBAndBody("admin", cmd); request = opMsgRequest.serialize(); OpMsg::setFlag(&request, OpMsg::kExhaustSupported); - // Run isMaster command to initiate the exhaust stream. + // Run hello or isMaster command to initiate the exhaust stream. ASSERT(conn1->call(request, reply)); auto lastRequestId = reply.header().getId(); ASSERT(OpMsg::isFlagSet(reply, OpMsg::kMoreToCome)); @@ -978,19 +1116,24 @@ TEST(OpMsg, ExhaustIsMasterMetricOnNewExhaustIsMasterAfterTerminatingExhaustStre auto serverStatusCmd = BSON("serverStatus" << 1); BSONObj serverStatusReply; ASSERT(conn2->runCommand("admin", serverStatusCmd, serverStatusReply)); - ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + if (useLegacyCommandName) { + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } else { + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } const auto failPointObj = BSON("configureFailPoint" << "failCommand" << "mode" << BSON("times" << 1) << "data" << BSON("threadName" << threadName << "errorCode" << ErrorCodes::NotWritablePrimary - << "failCommands" - << BSON_ARRAY("isMaster"))); + << "failCommands" << BSON_ARRAY(cmdName))); auto response = conn2->runCommand(OpMsgRequest::fromDBAndBody("admin", failPointObj)); ASSERT_OK(getStatusFromCommandResult(response->getCommandReply())); - // Wait for the exhaust stream to close from the error returned by isMaster. + // Wait for the exhaust stream to close from the error returned by hello or isMaster. ASSERT(waitForCondition([&] { const auto status = conn1->recv(reply, lastRequestId); lastRequestId = reply.header().getId(); @@ -998,23 +1141,43 @@ TEST(OpMsg, ExhaustIsMasterMetricOnNewExhaustIsMasterAfterTerminatingExhaustStre return !getStatusFromCommandResult(res).isOK(); })); - // Terminating the exhaust stream should not decrement the number of 'exhaustIsMaster'. + // Terminating the exhaust stream should not decrement the number of exhaust connections. ASSERT(conn2->runCommand("admin", serverStatusCmd, serverStatusReply)); - ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + if (useLegacyCommandName) { + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } else { + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } - opMsgRequest = OpMsgRequest::fromDBAndBody("admin", isMasterCmd); + opMsgRequest = OpMsgRequest::fromDBAndBody("admin", cmd); request = opMsgRequest.serialize(); OpMsg::setFlag(&request, OpMsg::kExhaustSupported); - // Run isMaster command on conn1 to initiate a new exhaust stream. + // Run hello or isMaster command on conn1 to initiate a new exhaust stream. ASSERT(conn1->call(request, reply)); ASSERT(OpMsg::isFlagSet(reply, OpMsg::kMoreToCome)); res = OpMsg::parse(reply).body; ASSERT_OK(getStatusFromCommandResult(res)); - // 'exhaustIsMaster' should not increment or decrement after initiating a new exhaust stream. + // exhaust metric should not increment or decrement after initiating a new exhaust stream. ASSERT(conn2->runCommand("admin", serverStatusCmd, serverStatusReply)); - ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + if (useLegacyCommandName) { + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } else { + ASSERT_EQUALS(0, serverStatusReply["connections"]["exhaustIsMaster"].numberInt()); + ASSERT_EQUALS(1, serverStatusReply["connections"]["exhaustHello"].numberInt()); + } +} + +TEST(OpMsg, ExhaustIsMasterMetricOnNewExhaustIsMasterAfterTerminatingExhaustStream) { + return exhaustMetricOnNewExhaustAfterTerminatingExhaustStream(true); +} + +TEST(OpMsg, ExhaustHelloMetricOnNewExhaustHelloAfterTerminatingExhaustStream) { + return exhaustMetricOnNewExhaustAfterTerminatingExhaustStream(false); } TEST(OpMsg, ExhaustWithDBClientCursorBehavesCorrectly) { diff --git a/src/mongo/s/commands/cluster_is_master_cmd.cpp b/src/mongo/s/commands/cluster_is_master_cmd.cpp index f24cb3a298d..ced249eb614 100644 --- a/src/mongo/s/commands/cluster_is_master_cmd.cpp +++ b/src/mongo/s/commands/cluster_is_master_cmd.cpp @@ -198,15 +198,16 @@ public: saslMechanismRegistry.advertiseMechanismNamesForUser(opCtx, cmdObj, &result); if (opCtx->isExhaust()) { - LOGV2_DEBUG(23872, 3, "Using exhaust for isMaster protocol"); + LOGV2_DEBUG(23872, 3, "Using exhaust for isMaster or hello protocol"); uassert(51763, - "An isMaster request with exhaust must specify 'maxAwaitTimeMS'", + "An isMaster or hello request with exhaust must specify 'maxAwaitTimeMS'", maxAwaitTimeMSField); invariant(clientTopologyVersion); InExhaustIsMaster::get(opCtx->getClient()->session().get()) - ->setInExhaustIsMaster(true /* inExhaustIsMaster */); + ->setInExhaustIsMaster(true /* inExhaust */, + cmdObj.firstElementFieldNameStringData()); if (clientTopologyVersion->getProcessId() == currentMongosTopologyVersion.getProcessId() && diff --git a/src/mongo/s/commands/strategy.cpp b/src/mongo/s/commands/strategy.cpp index 3f45a230e97..3f4c0107662 100644 --- a/src/mongo/s/commands/strategy.cpp +++ b/src/mongo/s/commands/strategy.cpp @@ -317,7 +317,7 @@ void runCommand(OperationContext* opCtx, if (session) { if (!opCtx->isExhaust() || (command->getName() != "hello"_sd && command->getName() != "isMaster"_sd)) { - InExhaustIsMaster::get(session.get())->setInExhaustIsMaster(false); + InExhaustIsMaster::get(session.get())->setInExhaustIsMaster(false, commandName); } } diff --git a/src/mongo/transport/ismaster_metrics.cpp b/src/mongo/transport/ismaster_metrics.cpp index 0a68a46c4aa..c1a4d421b73 100644 --- a/src/mongo/transport/ismaster_metrics.cpp +++ b/src/mongo/transport/ismaster_metrics.cpp @@ -54,6 +54,18 @@ void IsMasterMetrics::decrementNumExhaustIsMaster() { _exhaustIsMasterConnections.fetchAndSubtract(1); } +size_t IsMasterMetrics::getNumExhaustHello() const { + return _exhaustHelloConnections.load(); +} + +void IsMasterMetrics::incrementNumExhaustHello() { + _exhaustHelloConnections.fetchAndAdd(1); +} + +void IsMasterMetrics::decrementNumExhaustHello() { + _exhaustHelloConnections.fetchAndSubtract(1); +} + size_t IsMasterMetrics::getNumAwaitingTopologyChanges() const { return _connectionsAwaitingTopologyChanges.load(); } @@ -78,19 +90,45 @@ InExhaustIsMaster::~InExhaustIsMaster() { if (_inExhaustIsMaster) { IsMasterMetrics::get(getGlobalServiceContext())->decrementNumExhaustIsMaster(); } + if (_inExhaustHello) { + IsMasterMetrics::get(getGlobalServiceContext())->decrementNumExhaustHello(); + } } bool InExhaustIsMaster::getInExhaustIsMaster() const { return _inExhaustIsMaster; } -void InExhaustIsMaster::setInExhaustIsMaster(bool inExhaustIsMaster) { - if (!_inExhaustIsMaster && inExhaustIsMaster) { - IsMasterMetrics::get(getGlobalServiceContext())->incrementNumExhaustIsMaster(); - } else if (_inExhaustIsMaster && !inExhaustIsMaster) { +bool InExhaustIsMaster::getInExhaustHello() const { + return _inExhaustHello; +} + +void InExhaustIsMaster::setInExhaustIsMaster(bool inExhaust, StringData commandName) { + bool isHello = (commandName == "hello"_sd); + + // Transition out of exhaust hello if setting inExhaust to false or if + // the isMaster command is used. + if (_inExhaustHello && (!inExhaust || !isHello)) { + IsMasterMetrics::get(getGlobalServiceContext())->decrementNumExhaustHello(); + _inExhaustHello = false; + } + + // Transition out of exhaust isMaster if setting inExhaust to false or if + // the hello command is used. + if (_inExhaustIsMaster && (!inExhaust || isHello)) { IsMasterMetrics::get(getGlobalServiceContext())->decrementNumExhaustIsMaster(); + _inExhaustIsMaster = false; + } + + if (inExhaust) { + if (isHello && !_inExhaustHello) { + IsMasterMetrics::get(getGlobalServiceContext())->incrementNumExhaustHello(); + _inExhaustHello = inExhaust; + } else if (!isHello && !_inExhaustIsMaster) { + IsMasterMetrics::get(getGlobalServiceContext())->incrementNumExhaustIsMaster(); + _inExhaustIsMaster = inExhaust; + } } - _inExhaustIsMaster = inExhaustIsMaster; } } // namespace mongo diff --git a/src/mongo/transport/ismaster_metrics.h b/src/mongo/transport/ismaster_metrics.h index 514b2088551..ed4d1a369bd 100644 --- a/src/mongo/transport/ismaster_metrics.h +++ b/src/mongo/transport/ismaster_metrics.h @@ -48,12 +48,14 @@ public: InExhaustIsMaster& operator=(InExhaustIsMaster&&) = delete; static InExhaustIsMaster* get(transport::Session* session); - void setInExhaustIsMaster(bool inExhaustIsMaster); + void setInExhaustIsMaster(bool inExhaust, StringData commandName); bool getInExhaustIsMaster() const; + bool getInExhaustHello() const; ~InExhaustIsMaster(); private: bool _inExhaustIsMaster = false; + bool _inExhaustHello = false; }; /** @@ -72,6 +74,7 @@ public: static IsMasterMetrics* get(OperationContext* opCtx); size_t getNumExhaustIsMaster() const; + size_t getNumExhaustHello() const; size_t getNumAwaitingTopologyChanges() const; void incrementNumAwaitingTopologyChanges(); @@ -84,10 +87,15 @@ private: void incrementNumExhaustIsMaster(); void decrementNumExhaustIsMaster(); + void incrementNumExhaustHello(); + void decrementNumExhaustHello(); + // The number of clients currently waiting in isMaster for a topology change. AtomicWord<size_t> _connectionsAwaitingTopologyChanges{0}; // The number of connections whose last request was an isMaster with exhaustAllowed. AtomicWord<size_t> _exhaustIsMasterConnections{0}; + // The number of connections whose last request was a hello with exhaustAllowed. + AtomicWord<size_t> _exhaustHelloConnections{0}; }; } // namespace mongo diff --git a/src/mongo/transport/service_entry_point_impl.cpp b/src/mongo/transport/service_entry_point_impl.cpp index 1824ccf1c94..75dc8f593dc 100644 --- a/src/mongo/transport/service_entry_point_impl.cpp +++ b/src/mongo/transport/service_entry_point_impl.cpp @@ -271,6 +271,8 @@ void ServiceEntryPointImpl::appendStats(BSONObjBuilder* bob) const { bob->append("active", static_cast<int>(sc->getActiveClientOperations())); bob->append("exhaustIsMaster", static_cast<int>(IsMasterMetrics::get(sc)->getNumExhaustIsMaster())); + bob->append("exhaustHello", + static_cast<int>(IsMasterMetrics::get(sc)->getNumExhaustHello())); bob->append("awaitingTopologyChanges", static_cast<int>(IsMasterMetrics::get(sc)->getNumAwaitingTopologyChanges())); } |