From 673c659b0ae6a66e6da1ce87dda1597df18067e7 Mon Sep 17 00:00:00 2001 From: Amirsaman Memaripour Date: Mon, 1 Jun 2020 16:48:20 -0400 Subject: SERVER-48519 Include mirrored read metrics in ftdc (cherry picked from commit 7423d1ad84b43b2b519d7ec4551eb127ef74764d) --- jstests/noPassthrough/ftdc_mirrored_reads.js | 88 ++++++++++++++++++++++++++++ src/mongo/db/ftdc/ftdc_server.cpp | 4 ++ src/mongo/db/mirror_maestro.cpp | 4 +- src/mongo/db/mirror_maestro.h | 2 + 4 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 jstests/noPassthrough/ftdc_mirrored_reads.js diff --git a/jstests/noPassthrough/ftdc_mirrored_reads.js b/jstests/noPassthrough/ftdc_mirrored_reads.js new file mode 100644 index 00000000000..705769cc6a7 --- /dev/null +++ b/jstests/noPassthrough/ftdc_mirrored_reads.js @@ -0,0 +1,88 @@ +/** + * Verify the FTDC metrics for mirrored reads. + * + * @tags: [requires_replication] + */ +load('jstests/libs/ftdc.js'); + +(function() { +'use strict'; + +const kDbName = "mirrored_reads_ftdc_test"; +const kCollName = "test"; +const kOperations = 100; + +function getMirroredReadsStats(rst) { + return rst.getPrimary().getDB(kDbName).serverStatus({mirroredReads: 1}).mirroredReads; +} + +function getDiagnosticData(rst) { + let db = rst.getPrimary().getDB('admin'); + const stats = verifyGetDiagnosticData(db).serverStatus; + assert(stats.hasOwnProperty('mirroredReads')); + return stats.mirroredReads; +} + +function sendAndCheckReads(rst) { + let seenBeforeReads = getMirroredReadsStats(rst).seen; + + jsTestLog(`Sending ${kOperations} reads to primary`); + for (var i = 0; i < kOperations; ++i) { + rst.getPrimary().getDB(kDbName).runCommand({find: kCollName, filter: {}}); + } + + jsTestLog("Verifying reads were seen by the maestro"); + let seenAfterReads = getMirroredReadsStats(rst).seen; + assert.lte(seenBeforeReads + kOperations, seenAfterReads); +} + +function activateFailPoint(rst) { + const db = rst.getPrimary().getDB(kDbName); + assert.commandWorked(db.adminCommand({ + configureFailPoint: "mirrorMaestroExpectsResponse", + mode: "alwaysOn", + })); +} + +const rst = new ReplSetTest({nodes: 3}); +rst.startSet(); +rst.initiateWithHighElectionTimeout(); + +// Mirror every mirror-able command. +assert.commandWorked( + rst.getPrimary().adminCommand({setParameter: 1, mirrorReads: {samplingRate: 1.0}})); + +jsTestLog("Verifying diagnostic collection for mirrored reads"); +{ + let statsBeforeReads = getDiagnosticData(rst); + // The following metrics are not included by default. + assert(!statsBeforeReads.hasOwnProperty('resolved')); + assert(!statsBeforeReads.hasOwnProperty('resolvedBreakdown')); + + let seenBeforeReads = statsBeforeReads.seen; + sendAndCheckReads(rst); + assert.soon(() => { + let seenAfterReads = getDiagnosticData(rst).seen; + jsTestLog(`Seen ${seenAfterReads} mirrored reads so far`); + return seenBeforeReads + kOperations <= seenAfterReads; + }, "Failed to update FTDC metrics within time limit", 5000); +} + +jsTestLog("Verifying diagnostic collection when mirrorMaestroExpectsResponse"); +{ + activateFailPoint(rst); + assert.soon(() => { + return getDiagnosticData(rst).hasOwnProperty('resolved'); + }, "Failed to find 'resolved' in mirrored reads FTDC metrics within time limit", 5000); + let resolvedBeforeReads = getDiagnosticData(rst).resolved; + sendAndCheckReads(rst); + assert.soon(() => { + let resolvedAfterReads = getDiagnosticData(rst).resolved; + jsTestLog(`Mirrored ${resolvedAfterReads} reads so far`); + // There are two secondaries, so `kOperations * 2` reads must be resolved. + return resolvedBeforeReads + kOperations * 2 <= resolvedAfterReads; + }, "Failed to update extended FTDC metrics within time limit", 10000); +} + +rst.stopSet(); +})(); diff --git a/src/mongo/db/ftdc/ftdc_server.cpp b/src/mongo/db/ftdc/ftdc_server.cpp index fa215af1dce..f4fcf1e0ec2 100644 --- a/src/mongo/db/ftdc/ftdc_server.cpp +++ b/src/mongo/db/ftdc/ftdc_server.cpp @@ -44,6 +44,7 @@ #include "mongo/db/ftdc/ftdc_server_gen.h" #include "mongo/db/ftdc/ftdc_system_stats.h" #include "mongo/db/jsobj.h" +#include "mongo/db/mirror_maestro.h" #include "mongo/db/service_context.h" #include "mongo/util/synchronized_value.h" @@ -207,6 +208,8 @@ public: // of active migrations. // "timing" is filtered out because it triggers frequent schema changes. // "defaultRWConcern" is excluded because it changes rarely and instead included in rotation + // "mirroredReads" is included to append the number of mirror-able operations observed and + // mirrored by this process in FTDC collections. BSONObjBuilder commandBuilder; commandBuilder.append(kCommand, 1); @@ -214,6 +217,7 @@ public: commandBuilder.append("sharding", false); commandBuilder.append("timing", false); commandBuilder.append("defaultRWConcern", false); + commandBuilder.append(MirrorMaestro::kServerStatusSectionName, true); if (gDiagnosticDataCollectionEnableLatencyHistograms.load()) { BSONObjBuilder subObjBuilder(commandBuilder.subobjStart("opLatencies")); diff --git a/src/mongo/db/mirror_maestro.cpp b/src/mongo/db/mirror_maestro.cpp index ebaecf32b28..b54183fdc7b 100644 --- a/src/mongo/db/mirror_maestro.cpp +++ b/src/mongo/db/mirror_maestro.cpp @@ -68,7 +68,6 @@ constexpr auto kMirrorMaestroConnPoolMaxSize = 4ull; // Never use more tha constexpr auto kMirroredReadsParamName = "mirrorReads"_sd; -constexpr auto kMirroredReadsName = "mirroredReads"_sd; constexpr auto kMirroredReadsSeenKey = "seen"_sd; constexpr auto kMirroredReadsSentKey = "sent"_sd; constexpr auto kMirroredReadsResolvedKey = "resolved"_sd; @@ -170,7 +169,8 @@ class MirroredReadsSection final : public ServerStatusSection { public: using CounterT = long long; - MirroredReadsSection() : ServerStatusSection(kMirroredReadsName.toString()) {} + MirroredReadsSection() + : ServerStatusSection(MirrorMaestro::kServerStatusSectionName.toString()) {} bool includeByDefault() const override { return false; diff --git a/src/mongo/db/mirror_maestro.h b/src/mongo/db/mirror_maestro.h index f595fd0bcc9..8d22586bfdd 100644 --- a/src/mongo/db/mirror_maestro.h +++ b/src/mongo/db/mirror_maestro.h @@ -71,6 +71,8 @@ public: * This function will noop if the MirrorMaestro is currently being initialized or shutdown. */ static void tryMirrorRequest(OperationContext* opCtx) noexcept; + + static constexpr auto kServerStatusSectionName = "mirroredReads"_sd; }; } // namespace mongo -- cgit v1.2.1