diff options
| -rw-r--r-- | jstests/noPassthrough/mirror_reads.js | 56 | ||||
| -rw-r--r-- | src/mongo/db/mirror_maestro.cpp | 19 |
2 files changed, 59 insertions, 16 deletions
diff --git a/jstests/noPassthrough/mirror_reads.js b/jstests/noPassthrough/mirror_reads.js index b97f303c974..5ecf7c7eb83 100644 --- a/jstests/noPassthrough/mirror_reads.js +++ b/jstests/noPassthrough/mirror_reads.js @@ -42,22 +42,41 @@ function sendAndCheckReads({rst, cmd, minRate, maxRate}) { assert.soon(() => { let currentMirroredReads = getMirroredReadsStats(rst); + let readsSent = currentMirroredReads.sent - startMirroredReads.sent; + let readsResolved = currentMirroredReads.resolved - startMirroredReads.resolved; + // The number of reads the primary has decided to mirror to secondaries, but hasn't yet + // sent. + let readsPending = currentMirroredReads.pending; let readsSeen = currentMirroredReads.seen - startMirroredReads.seen; - let readsMirrored = currentMirroredReads.resolved - startMirroredReads.resolved; - let numNodes = rst.getSecondaries().length; - jsTestLog(`Seen ${readsSeen} requests; ` + - `verified ${readsMirrored / numNodes} requests ` + - `x ${numNodes} nodes`); + jsTestLog( + "Verifying that all mirrored reads sent from primary have been recieved by secondaries: " + + tojson({ + sent: readsSent, + resolved: readsResolved, + pending: readsPending, + seen: readsSeen + })); + + return ((readsPending == 0) && (readsSent == readsResolved)); + }, "Did not resolve all requests within time limit", 10000); - let rate = readsMirrored / readsSeen / numNodes; - return (rate >= minRate) && (readsSeen >= kBurstCount); - }, "Did not verify all requests within time limit", 10000); let currentMirroredReads = getMirroredReadsStats(rst); - const resolvedRate = (currentMirroredReads.resolved - startMirroredReads.resolved) / - (currentMirroredReads.seen - startMirroredReads.seen) / rst.getSecondaries().length; - jsTestLog(`Comparing resolvedRate: ${resolvedRate} versus maxRate: ${maxRate}`); - assert(resolvedRate <= maxRate); + + jsTestLog("Verifying statistics: " + + tojson({current: currentMirroredReads, start: startMirroredReads})); + + let readsSeen = currentMirroredReads.seen - startMirroredReads.seen; + let readsSent = currentMirroredReads.sent - startMirroredReads.sent; + let readsMirrored = currentMirroredReads.resolved - startMirroredReads.resolved; + let numNodes = rst.getSecondaries().length; + let rate = readsMirrored / readsSeen / numNodes; + + // Check that the primary has seen all the mirrored-read supporting operations we've sent it + assert.gte(readsSeen, kBurstCount); + // Check that the rate of mirroring meets the provided criteria + assert.gte(rate, minRate); + assert.lte(rate, maxRate); jsTestLog(`Verified ${tojson(cmd)} was mirrored`); } @@ -94,8 +113,12 @@ function verifyMirrorReads(rst, cmd) { { const rst = new ReplSetTest({ nodes: 3, - nodeOptions: - {setParameter: {"failpoint.mirrorMaestroExpectsResponse": tojson({mode: "alwaysOn"})}} + nodeOptions: { + setParameter: { + "failpoint.mirrorMaestroExpectsResponse": tojson({mode: "alwaysOn"}), + "failpoint.mirrorMaestroTracksPending": tojson({mode: "alwaysOn"}) + } + } }); rst.startSet(); rst.initiateWithHighElectionTimeout(); @@ -198,7 +221,10 @@ function verifyMirroringDistribution(rst) { const rst = new ReplSetTest({ nodes: secondaries + 1, nodeOptions: { - setParameter: {"failpoint.mirrorMaestroExpectsResponse": tojson({mode: "alwaysOn"})} + setParameter: { + "failpoint.mirrorMaestroExpectsResponse": tojson({mode: "alwaysOn"}), + "failpoint.mirrorMaestroTracksPending": tojson({mode: "alwaysOn"}) + } } }); rst.startSet(); diff --git a/src/mongo/db/mirror_maestro.cpp b/src/mongo/db/mirror_maestro.cpp index 46d645e3c10..14eb37427a5 100644 --- a/src/mongo/db/mirror_maestro.cpp +++ b/src/mongo/db/mirror_maestro.cpp @@ -72,8 +72,10 @@ constexpr auto kMirroredReadsSeenKey = "seen"_sd; constexpr auto kMirroredReadsSentKey = "sent"_sd; constexpr auto kMirroredReadsResolvedKey = "resolved"_sd; constexpr auto kMirroredReadsResolvedBreakdownKey = "resolvedBreakdown"_sd; +constexpr auto kMirroredReadsPendingKey = "pending"_sd; MONGO_FAIL_POINT_DEFINE(mirrorMaestroExpectsResponse); +MONGO_FAIL_POINT_DEFINE(mirrorMaestroTracksPending); class MirrorMaestroImpl { public: @@ -186,7 +188,9 @@ public: section.append(kMirroredReadsResolvedKey, resolved.loadRelaxed()); section.append(kMirroredReadsResolvedBreakdownKey, resolvedBreakdown.toBSON()); } - + if (MONGO_unlikely(mirrorMaestroTracksPending.shouldFail())) { + section.append(kMirroredReadsPendingKey, pending.loadRelaxed()); + } return section.obj(); }; @@ -227,6 +231,8 @@ public: AtomicWord<CounterT> seen; AtomicWord<CounterT> sent; AtomicWord<CounterT> resolved; + // Counts the number of operations that are scheduled to be mirrored, but haven't yet been sent. + AtomicWord<CounterT> pending; } gMirroredReadsSection; auto parseMirroredReadsParameters(const BSONObj& obj) { @@ -330,9 +336,20 @@ void MirrorMaestroImpl::tryMirror(std::shared_ptr<CommandInvocation> invocation) // building new bsons and evaluating randomness in a less important context. auto requestState = std::make_unique<MirroredRequestState>( this, std::move(hosts), std::move(invocation), std::move(params)); + if (MONGO_unlikely(mirrorMaestroTracksPending.shouldFail())) { + // We've scheduled the operation to be mirrored; it is now "pending" until it has actually + // been sent to a secondary. + gMirroredReadsSection.pending.fetchAndAdd(1); + } ExecutorFuture(_executor) // .getAsync([clientExecutorHandle, requestState = std::move(requestState)](const auto& status) mutable { + ON_BLOCK_EXIT([&] { + if (MONGO_unlikely(mirrorMaestroTracksPending.shouldFail())) { + // The read has been sent to at least one secondary, so it's no longer pending + gMirroredReadsSection.pending.fetchAndSubtract(1); + } + }); if (!ErrorCodes::isShutdownError(status)) { invariant(status.isOK()); requestState->mirror(); |