1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
|
/**
* Tests hedging metrics in the serverStatus output.
*/
(function() {
"use strict";
load("jstests/libs/fail_point_util.js");
/*
* Verifies that the server status response has the hegingMetrics fields that we expect.
*/
function verifyServerStatusFields(serverStatusResponse) {
assert(serverStatusResponse.hasOwnProperty("hedgingMetrics"),
"Expected the serverStatus response to have a 'hedgingMetrics' field\n" +
tojson(serverStatusResponse));
assert(
serverStatusResponse.hedgingMetrics.hasOwnProperty("numTotalOperations"),
"The 'hedgingMetrics' field in serverStatus did not have the 'numTotalOperations' field\n" +
tojson(serverStatusResponse.hedgingMetrics));
assert(
serverStatusResponse.hedgingMetrics.hasOwnProperty("numTotalHedgedOperations"),
"The 'hedgingMetrics' field in serverStatus did not have the 'numTotalHedgedOperations' field\n" +
tojson(serverStatusResponse.hedgingMetrics));
assert(
serverStatusResponse.hedgingMetrics.hasOwnProperty("numAdvantageouslyHedgedOperations"),
"The 'hedgingMetrics' field in serverStatus did not have the 'numAdvantageouslyHedgedOperations' field\n" +
tojson(serverStatusResponse.hedgingMetrics));
}
/*
* Verifies that eventually the hedgingMetrics in the server status response is equal to
* the expected hedgingMetrics.
*/
function checkServerStatusHedgingMetrics(mongosConn, expectedHedgingMetrics) {
assert.soon(
() => {
const serverStatus = assert.commandWorked(mongosConn.adminCommand({serverStatus: 1}));
verifyServerStatusFields(serverStatus);
return bsonWoCompare(serverStatus.hedgingMetrics, expectedHedgingMetrics) === 0;
},
`expect the hedgingMetrics to eventually be equal to ${tojson(expectedHedgingMetrics)}`,
serverStatusCheckTimeoutMS);
}
function setCommandDelay(nodeConn, command, delay, ns) {
assert.commandWorked(nodeConn.adminCommand({
configureFailPoint: "failCommand",
mode: {times: 1},
data: {
failInternalCommands: true,
blockConnection: true,
blockTimeMS: delay,
failCommands: [command],
namespace: ns,
}
}));
}
function clearCommandDelay(nodeConn) {
assert.commandWorked(nodeConn.adminCommand({
configureFailPoint: "failCommand",
mode: "off",
}));
}
const st = new ShardingTest({
mongos: [{
setParameter: {
logComponentVerbosity: tojson({network: {verbosity: 2}}),
// Force the mongos's replica set monitors to always include all the eligible nodes.
"failpoint.scanningServerSelectorIgnoreLatencyWindow": tojson({mode: "alwaysOn"}),
"failpoint.sdamServerSelectorIgnoreLatencyWindow": tojson({mode: "alwaysOn"}),
// Force the mongos to send requests to hosts in alphabetical order of host names.
"failpoint.networkInterfaceSendRequestsToTargetHostsInAlphabeticalOrder":
tojson({mode: "alwaysOn"})
}
}],
shards: 1,
rs: {nodes: 2, setParameter: {logComponentVerbosity: tojson({command: {verbosity: 1}})}}
});
const dbName = "foo";
const collName = "bar";
const ns = dbName + "." + collName;
const testDB = st.s.getDB(dbName);
const serverStatusCheckTimeoutMS = 5000;
assert.commandWorked(st.s.adminCommand({enableSharding: dbName}));
st.ensurePrimaryShard(dbName, st.shard0.shardName);
// Force the mongos's replica set monitors to always include all the eligible nodes.
const replicaSetMonitorProtocol =
assert.commandWorked(st.s.adminCommand({getParameter: 1, replicaSetMonitorProtocol: 1}))
.replicaSetMonitorProtocol;
let serverSelectorFailPoint = configureFailPoint(st.s,
replicaSetMonitorProtocol === "scanning"
? "scanningServerSelectorIgnoreLatencyWindow"
: "sdamServerSelectorIgnoreLatencyWindow");
// Force the mongos to send requests to hosts in alphabetical order of host names.
let sendRequestsFailPoint =
configureFailPoint(st.s, "networkInterfaceSendRequestsToTargetHostsInAlphabeticalOrder");
let sortedNodes = [...st.rs0.nodes].sort((node1, node2) => node1.host.localeCompare(node2.host));
let expectedHedgingMetrics = {
numTotalOperations: 0,
numTotalHedgedOperations: 0,
numAdvantageouslyHedgedOperations: 0
};
jsTestLog("Run a command with hedging disabled, and verify the metrics does not change");
assert.commandWorked(testDB.runCommand(
{count: collName, query: {x: {$gte: 0}}, $readPreference: {mode: "primaryPreferred"}}));
checkServerStatusHedgingMetrics(testDB, expectedHedgingMetrics);
jsTestLog("Run commands with hedging enabled, and verify the metrics are as expected");
// Make the command slower on the first target host, and verify there is an advantageous
// hedged read.
try {
setCommandDelay(sortedNodes[0], "count", 1000, ns);
assert.commandWorked(testDB.runCommand(
{count: collName, query: {x: {$gte: 0}}, $readPreference: {mode: "nearest"}}));
} finally {
clearCommandDelay(sortedNodes[0]);
}
expectedHedgingMetrics.numTotalOperations += 1;
expectedHedgingMetrics.numTotalHedgedOperations += 1;
expectedHedgingMetrics.numAdvantageouslyHedgedOperations += 1;
checkServerStatusHedgingMetrics(testDB, expectedHedgingMetrics);
// Make the command slower on the second target host, and verify there is no advantageous
// hedged read. Block the command on the first target host for some time to allow the hedged
// request to get sent to the second target host.
try {
setCommandDelay(sortedNodes[0], "count", 100, ns);
setCommandDelay(sortedNodes[1], "count", 1000, ns);
assert.commandWorked(testDB.runCommand(
{count: collName, query: {x: {$gte: 0}}, $readPreference: {mode: "nearest"}}));
} finally {
clearCommandDelay(sortedNodes[0]);
clearCommandDelay(sortedNodes[1]);
}
expectedHedgingMetrics.numTotalOperations += 1;
expectedHedgingMetrics.numTotalHedgedOperations += 1;
checkServerStatusHedgingMetrics(testDB, expectedHedgingMetrics);
st.stop();
}());
|
