diff options
Diffstat (limited to 'jstests/noPassthrough/mirror_reads.js')
| -rw-r--r-- | jstests/noPassthrough/mirror_reads.js | 127 |
1 files changed, 100 insertions, 27 deletions
diff --git a/jstests/noPassthrough/mirror_reads.js b/jstests/noPassthrough/mirror_reads.js index 6315c4df9b1..4a27724aeef 100644 --- a/jstests/noPassthrough/mirror_reads.js +++ b/jstests/noPassthrough/mirror_reads.js @@ -3,7 +3,7 @@ * * @tags: [ * requires_replication, - * requires_fcv_61 + * requires_fcv_62 * ] */ @@ -17,24 +17,28 @@ function setParameter({rst, value}) { const kBurstCount = 1000; const kDbName = "mirrored_reads_test"; const kCollName = "test"; +// We use an arbitrarily large maxTimeMS to avoid timing out when processing the mirrored read +// on slower builds. Otherwise, on slower builds, the primary.mirroredReads.sent metric could +// be incremented but not the secondary.mirroredReads.processedAsSecondary metric. +const kLargeMaxTimeMS = 100000000; function getMirroredReadsStats(node) { return node.getDB(kDbName).serverStatus({mirroredReads: 1}).mirroredReads; } -function sendAndCheckReads({rst, cmd, minRate, maxRate}) { +function sendAndCheckReads({rst, cmd, minRate, maxRate, burstCount}) { const primary = rst.getPrimary(); const secondaries = rst.getSecondaries(); let initialPrimaryStats = getMirroredReadsStats(primary); - let initialSecondariesReceived = []; + let initialProcessedAsSecondary = []; for (const secondary of rst.getSecondaries()) { - let secondaryMirroredReadsReceived = getMirroredReadsStats(secondary).received; - initialSecondariesReceived.push(secondaryMirroredReadsReceived); + let secondaryMirroredReadsProcessed = getMirroredReadsStats(secondary).processedAsSecondary; + initialProcessedAsSecondary.push(secondaryMirroredReadsProcessed); } - jsTestLog(`Sending ${kBurstCount} request burst of ${tojson(cmd)} to primary`); + jsTestLog(`Sending ${burstCount} request burst of ${tojson(cmd)} to primary`); - for (var i = 0; i < kBurstCount; ++i) { + for (var i = 0; i < burstCount; ++i) { rst.getPrimary().getDB(kDbName).runCommand(cmd); } @@ -43,18 +47,20 @@ function sendAndCheckReads({rst, cmd, minRate, maxRate}) { // Verify that the commands have been observed on the primary { const currentPrimaryStats = getMirroredReadsStats(primary); - assert.lte(initialPrimaryStats.seen + kBurstCount, currentPrimaryStats.seen); + assert.lte(initialPrimaryStats.seen + burstCount, currentPrimaryStats.seen); } // Verify that the reads mirrored to the secondaries have responded and secondaries receive the // same amount of mirrored reads that were sent by the primary. let currentPrimaryMirroredReadsStats; let readsSent; + let readsSucceeded; assert.soon(() => { currentPrimaryMirroredReadsStats = getMirroredReadsStats(primary); readsSent = currentPrimaryMirroredReadsStats.sent - initialPrimaryStats.sent; let readsResolved = currentPrimaryMirroredReadsStats.resolved - initialPrimaryStats.resolved; + readsSucceeded = currentPrimaryMirroredReadsStats.succeeded - initialPrimaryStats.succeeded; // The number of reads the primary has decided to mirror to secondaries, but hasn't yet // sent. let readsPending = currentPrimaryMirroredReadsStats.pending; @@ -64,23 +70,25 @@ function sendAndCheckReads({rst, cmd, minRate, maxRate}) { tojson({ sent: readsSent, resolved: readsResolved, + succeeded: readsSucceeded, pending: readsPending, seen: readsSeen })); return ((readsPending == 0) && (readsSent === readsResolved)); }, "Did not resolve all requests within time limit", 10000); - // The number of mirrored reads received across all secondaries. - let readsReceived = 0; + // The number of mirrored reads processed across all secondaries. + let readsProcessedAsSecondaryTotal = 0; for (let i = 0; i < secondaries.length; i++) { const currentSecondaryMirroredReadsStats = getMirroredReadsStats(secondaries[i]); - const received = - currentSecondaryMirroredReadsStats.received - initialSecondariesReceived[i]; - jsTestLog("Verifying number of reads received by secondary " + secondaries[i] + ": " + - tojson({received: received})); - readsReceived += received; + const processedAsSecondary = currentSecondaryMirroredReadsStats.processedAsSecondary - + initialProcessedAsSecondary[i]; + jsTestLog("Verifying number of reads processed by secondary " + secondaries[i] + ": " + + tojson({processedAsSecondary: processedAsSecondary})); + readsProcessedAsSecondaryTotal += processedAsSecondary; } - assert.eq(readsReceived, readsSent); + assert.eq(readsProcessedAsSecondaryTotal, readsSucceeded); + assert.eq(readsProcessedAsSecondaryTotal, readsSent); jsTestLog("Verifying primary statistics: " + tojson({current: currentPrimaryMirroredReadsStats, start: initialPrimaryStats})); @@ -91,7 +99,7 @@ function sendAndCheckReads({rst, cmd, minRate, maxRate}) { let rate = readsMirrored / readsSeen / numNodes; // Check that the primary has seen all the mirrored-read supporting operations we've sent it - assert.gte(readsSeen, kBurstCount); + assert.gte(readsSeen, burstCount); // Check that the rate of mirroring meets the provided criteria assert.gte(rate, minRate); assert.lte(rate, maxRate); @@ -105,7 +113,13 @@ function verifyMirrorReads(rst, cmd) { let samplingRate = 0.0; assert.commandWorked(setParameter({rst: rst, value: {samplingRate: samplingRate}})); - sendAndCheckReads({rst: rst, cmd: cmd, minRate: samplingRate, maxRate: samplingRate}); + sendAndCheckReads({ + rst: rst, + cmd: cmd, + minRate: samplingRate, + maxRate: samplingRate, + burstCount: kBurstCount + }); } { @@ -113,7 +127,13 @@ function verifyMirrorReads(rst, cmd) { let samplingRate = 1.0; assert.commandWorked(setParameter({rst: rst, value: {samplingRate: samplingRate}})); - sendAndCheckReads({rst: rst, cmd: cmd, minRate: samplingRate, maxRate: samplingRate}); + sendAndCheckReads({ + rst: rst, + cmd: cmd, + minRate: samplingRate, + maxRate: samplingRate, + burstCount: kBurstCount + }); } { @@ -124,7 +144,40 @@ function verifyMirrorReads(rst, cmd) { let min = samplingRate - gaussDeviation; assert.commandWorked(setParameter({rst: rst, value: {samplingRate: samplingRate}})); - sendAndCheckReads({rst: rst, cmd: cmd, minRate: min, maxRate: max}); + sendAndCheckReads( + {rst: rst, cmd: cmd, minRate: min, maxRate: max, burstCount: kBurstCount}); + } +} + +function verifyProcessedAsSecondary(rst) { + // Mirror every mirror-able command. + const samplingRate = 1.0; + assert.commandWorked(setParameter({rst: rst, value: {samplingRate: samplingRate}})); + + for (const secondary of rst.getSecondaries()) { + assert.commandWorked(secondary.getDB(kDbName).adminCommand({ + configureFailPoint: "failCommand", + mode: "alwaysOn", + data: { + errorCode: ErrorCodes.MaxTimeMSExpired, + failCommands: ["find"], + } + })); + } + + // With enabled fail point, check that no commands succeed or are processed, but all are + // resolved. + sendAndCheckReads({ + rst: rst, + cmd: {find: kCollName, filter: {}}, + minRate: samplingRate, + maxRate: samplingRate, + burstCount: kBurstCount + }); + + for (const secondary of rst.getSecondaries()) { + assert.commandWorked(secondary.getDB(kDbName).adminCommand( + {configureFailPoint: "failCommand", mode: "off"})); } } @@ -156,20 +209,32 @@ function verifyMirrorReads(rst, cmd) { } jsTestLog("Verifying mirrored reads for 'find' commands"); - verifyMirrorReads(rst, {find: kCollName, filter: {}}); + verifyMirrorReads(rst, {find: kCollName, filter: {}, maxTimeMS: kLargeMaxTimeMS}); jsTestLog("Verifying mirrored reads for 'count' commands"); - verifyMirrorReads(rst, {count: kCollName, query: {}}); + verifyMirrorReads(rst, {count: kCollName, query: {}, maxTimeMS: kLargeMaxTimeMS}); jsTestLog("Verifying mirrored reads for 'distinct' commands"); - verifyMirrorReads(rst, {distinct: kCollName, key: "x"}); + verifyMirrorReads(rst, {distinct: kCollName, key: "x", maxTimeMS: kLargeMaxTimeMS}); jsTestLog("Verifying mirrored reads for 'findAndModify' commands"); - verifyMirrorReads(rst, {findAndModify: kCollName, query: {}, update: {'$inc': {x: 1}}}); + verifyMirrorReads(rst, { + findAndModify: kCollName, + query: {}, + update: {'$inc': {x: 1}}, + maxTimeMS: kLargeMaxTimeMS + }); jsTestLog("Verifying mirrored reads for 'update' commands"); - verifyMirrorReads( - rst, {update: kCollName, updates: [{q: {_id: 1}, u: {'$inc': {x: 1}}}], ordered: false}); + verifyMirrorReads(rst, { + update: kCollName, + updates: [{q: {_id: 1}, u: {'$inc': {x: 1}}}], + ordered: false, + maxTimeMS: kLargeMaxTimeMS + }); + + jsTestLog("Verifying processedAsSecondary field for 'find' commands"); + verifyProcessedAsSecondary(rst); rst.stopSet(); } @@ -221,7 +286,15 @@ function verifyMirroringDistribution(rst) { let before = getMirroredReadsStats(rst.getPrimary()); - sendAndCheckReads({rst: rst, cmd: {find: kCollName, filter: {}}, minRate: min, maxRate: max}); + sendAndCheckReads({ + rst: rst, + cmd: {find: kCollName, filter: {}}, + minRate: min, + maxRate: max, + burstCount: kBurstCount, + checkExpectedReadsProcessed: false, + expectedReadsProcessed: 0 + }); let after = getMirroredReadsStats(rst.getPrimary()); |