1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
/**
* This test makes sure 'find' and 'getMore' commands fail correctly during rollback.
*
* @tags: [
* requires_majority_read_concern,
* ]
*/
(function() {
"use strict";
load("jstests/replsets/libs/rollback_test.js");
load("jstests/replsets/rslib.js");
const dbName = "test";
const collName = "coll";
// Set up Rollback Test.
let rollbackTest = new RollbackTest();
// Insert documents to be read later.
assert.commandWorked(rollbackTest.getPrimary().getDB(dbName)[collName].insert([{}, {}, {}]));
let rollbackNode = rollbackTest.transitionToRollbackOperations();
// Open a cursor on 'rollbackNode' which returns partial results, but will remain open and idle
// during the rollback process.
const findCmdRes =
assert.commandWorked(rollbackNode.getDB(dbName).runCommand({"find": collName, batchSize: 2}));
assert.eq(2, findCmdRes.cursor.firstBatch.length, findCmdRes);
const idleCursorId = findCmdRes.cursor.id;
assert.neq(0, idleCursorId, findCmdRes);
const failPointAfterTransition =
configureFailPoint(rollbackNode, "rollbackHangAfterTransitionToRollback");
const failPointAfterPinCursor = configureFailPoint(rollbackNode, "getMoreHangAfterPinCursor");
const joinGetMoreThread = startParallelShell(() => {
db.getMongo().setSecondaryOk();
const cursorID = assert.commandWorked(db.runCommand({"find": "coll", batchSize: 0})).cursor.id;
// Make sure an outstanding read operation gets killed during rollback even though the read
// was started before rollback. Outstanding read operations are killed during rollback and
// their connections are closed shortly after. So we would get either an error
// (InterruptedDueToReplStateChange) if the error message is sent out and received before
// the connection is closed or a network error exception.
try {
assert.commandFailedWithCode(db.runCommand({"getMore": cursorID, collection: "coll"}),
ErrorCodes.InterruptedDueToReplStateChange);
} catch (e) {
assert.includes(e.toString(), "network error while attempting to run command");
}
}, rollbackNode.port);
const cursorIdToBeReadDuringRollback =
assert.commandWorked(rollbackNode.getDB(dbName).runCommand({"find": collName, batchSize: 0}))
.cursor.id;
// Wait for 'getMore' to hang on the test collection.
assert.soonNoExcept(() => {
const filter = {"command.getMore": {$exists: true}, "command.collection": collName};
return rollbackNode.getDB(dbName).adminCommand("currentOp", filter).inprog.length === 1;
});
// Start rollback.
rollbackTest.transitionToSyncSourceOperationsBeforeRollback();
rollbackTest.transitionToSyncSourceOperationsDuringRollback();
jsTestLog("Reconnecting to " + rollbackNode.host + " after rollback");
reconnect(rollbackNode.getDB(dbName));
// Wait for rollback to hang. We continuously retry the wait command since the rollback node
// might reject new connections initially, causing the command to fail.
assert.soon(() => {
try {
failPointAfterTransition.wait();
return true;
} catch (e) {
return false;
}
});
failPointAfterPinCursor.off();
jsTestLog("Wait for 'getMore' thread to join.");
joinGetMoreThread();
jsTestLog("Reading during rollback.");
// Make sure that read operations fail during rollback.
assert.commandFailedWithCode(rollbackNode.getDB(dbName).runCommand({"find": collName}),
ErrorCodes.NotPrimaryOrSecondary);
assert.commandFailedWithCode(rollbackNode.getDB(dbName).runCommand(
{"getMore": cursorIdToBeReadDuringRollback, collection: collName}),
ErrorCodes.NotPrimaryOrSecondary);
// Disable the best-effort check for primary-ness in the service entry point, so that we
// exercise the real check for primary-ness in 'find' and 'getMore' commands.
configureFailPoint(rollbackNode, "skipCheckingForNotPrimaryInCommandDispatch");
jsTestLog("Reading during rollback (again with command dispatch checks disabled).");
assert.commandFailedWithCode(rollbackNode.getDB(dbName).runCommand({"find": collName}),
ErrorCodes.NotPrimaryOrSecondary);
assert.commandFailedWithCode(rollbackNode.getDB(dbName).runCommand(
{"getMore": cursorIdToBeReadDuringRollback, collection: collName}),
ErrorCodes.NotPrimaryOrSecondary);
failPointAfterTransition.off();
rollbackTest.transitionToSteadyStateOperations();
const replMetrics = assert.commandWorked(rollbackNode.adminCommand({serverStatus: 1})).metrics.repl;
assert.eq(replMetrics.stateTransition.lastStateTransition, "rollback");
assert(replMetrics.stateTransition.userOperationsRunning,
() => "Response should have a 'stateTransition.userOperationsRunning' field: " +
tojson(replMetrics));
assert(replMetrics.stateTransition.userOperationsKilled,
() => "Response should have a 'stateTransition.userOperationsKilled' field: " +
tojson(replMetrics));
// Run a getMore against the idle cursor that remained open throughout the rollback. The getMore
// should fail since the cursor has been invalidated by the rollback.
assert.commandFailedWithCode(
rollbackNode.getDB(dbName).runCommand({"getMore": idleCursorId, collection: collName}),
ErrorCodes.QueryPlanKilled);
// Check the replica set.
rollbackTest.stop();
}());
|
