1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
|
/**
* Verify the FTDC metrics for mirrored reads.
*
* @tags: [
* requires_replication,
* requires_fcv_62
* ]
*/
load('jstests/libs/ftdc.js');
(function() {
'use strict';
const kDbName = "mirrored_reads_ftdc_test";
const kCollName = "test";
const kOperations = 100;
const rst = new ReplSetTest({nodes: 3});
rst.startSet();
rst.initiateWithHighElectionTimeout();
const primary = rst.getPrimary();
const secondaries = rst.getSecondaries();
function getMirroredReadsStats(node) {
return node.getDB(kDbName).serverStatus({mirroredReads: 1}).mirroredReads;
}
function getDiagnosticData(node) {
let db = node.getDB('admin');
const stats = verifyGetDiagnosticData(db).serverStatus;
assert(stats.hasOwnProperty('mirroredReads'));
return stats.mirroredReads;
}
function getMirroredReadsProcessedAsSecondary() {
let readsProcessed = 0;
for (let i = 0; i < secondaries.length; i++) {
const stats = getMirroredReadsStats(secondaries[i]);
readsProcessed += stats.processedAsSecondary;
}
return readsProcessed;
}
function waitForPrimaryToSendMirroredReads(expectedReadsSeen, expectedReadsSent) {
assert.soon(() => {
jsTestLog("Verifying reads were seen and sent by the maestro");
jsTestLog("ExpectedReadsSent :" + expectedReadsSent +
", ExpectedReadsSeen:" + expectedReadsSeen);
const afterPrimaryReadStats = getMirroredReadsStats(primary);
const actualMirrorableReadsSeen = afterPrimaryReadStats.seen;
const actualMirroredReadsSent = afterPrimaryReadStats.sent;
jsTestLog("Primary metrics after reads: " + tojson(afterPrimaryReadStats));
return expectedReadsSeen <= actualMirrorableReadsSeen &&
expectedReadsSent == actualMirroredReadsSent;
});
}
function sendAndCheckReads(rst) {
const primary = rst.getPrimary();
// Initial metrics before sending kOperations number of finds.
const initialPrimaryReadStats = getMirroredReadsStats(primary);
const mirrorableReadsSeenBefore = initialPrimaryReadStats.seen;
const mirroredReadsSentBefore = initialPrimaryReadStats.sent;
jsTestLog(`Sending ${kOperations} reads to primary`);
for (var i = 0; i < kOperations; ++i) {
primary.getDB(kDbName).runCommand({find: kCollName, filter: {}});
}
const expectedReadsSeen = mirrorableReadsSeenBefore + kOperations;
const expectedReadsSent = mirroredReadsSentBefore + (2 * kOperations);
// Wait for primary to have sent out all mirrored reads.
waitForPrimaryToSendMirroredReads(expectedReadsSeen, expectedReadsSent);
}
function activateFailPoint(node) {
const db = node.getDB(kDbName);
assert.commandWorked(db.adminCommand({
configureFailPoint: "mirrorMaestroExpectsResponse",
mode: "alwaysOn",
}));
}
// Mirror every mirror-able command.
assert.commandWorked(primary.adminCommand({setParameter: 1, mirrorReads: {samplingRate: 1.0}}));
{
// Send and check reads with mirrorMaestroExpectsResponse failpoint disabled by default.
jsTestLog("Verifying diagnostic collection for mirrored reads on primary");
let primaryStatsBeforeReads = getDiagnosticData(primary);
let initialMirrorableReadsSeen = primaryStatsBeforeReads.seen;
// The following metrics are not included by default.
assert(!primaryStatsBeforeReads.hasOwnProperty('resolved'));
assert(!primaryStatsBeforeReads.hasOwnProperty('resolvedBreakdown'));
sendAndCheckReads(rst);
assert.soon(() => {
let mirrorableReadsSeen = getDiagnosticData(primary).seen;
jsTestLog(`Seen ${mirrorableReadsSeen} mirrored reads so far`);
return initialMirrorableReadsSeen + kOperations <= mirrorableReadsSeen;
}, "Failed to update FTDC metrics within time limit", 30000);
}
{
// Send and check reads after activating mirrorMaestroExpectsResponse fail point.
jsTestLog("Verifying diagnostic collection when mirrorMaestroExpectsResponse");
activateFailPoint(primary);
assert.soon(() => {
return getDiagnosticData(primary).hasOwnProperty('resolved');
}, "Failed to find 'resolved' in mirrored reads FTDC metrics within time limit", 30000);
let primaryStatsBeforeReads = getDiagnosticData(primary);
let mirroredReadsProcessedBefore = getMirroredReadsProcessedAsSecondary();
let primaryResolvedBeforeReads = primaryStatsBeforeReads.resolved;
let primarySentBeforeReads = primaryStatsBeforeReads.sent;
sendAndCheckReads(rst);
assert.soon(() => {
let primaryResolvedAfterReads = getDiagnosticData(primary).resolved;
jsTestLog(`Mirrored ${primaryResolvedAfterReads} reads so far`);
for (let i = 0; i < secondaries.length; i++) {
jsTestLog("Secondary " + secondaries[i] +
" metrics: " + tojson(getMirroredReadsStats(secondaries[i])));
}
// There are two secondaries, so `kOperations * 2` reads must be resolved.
return primaryResolvedBeforeReads + kOperations * 2 <= primaryResolvedAfterReads;
}, "Failed to update extended FTDC metrics within time limit", 10000);
const primaryDataAfterReads = getDiagnosticData(primary);
const primarySentAfterReads = primaryDataAfterReads.sent;
const primaryResolvedAfterReads = primaryDataAfterReads.resolved;
assert.eq(primaryResolvedAfterReads - primaryResolvedBeforeReads,
primarySentAfterReads - primarySentBeforeReads,
primaryDataAfterReads);
jsTestLog("Verifying diagnostic collection for mirrored reads on secondaries");
let mirroredReadsSucceeded = getDiagnosticData(primary).succeeded;
let mirroredReadsProcessedAfter = getMirroredReadsProcessedAsSecondary();
assert.eq(mirroredReadsSucceeded, mirroredReadsProcessedAfter - mirroredReadsProcessedBefore);
}
rst.stopSet();
})();
|
