1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
|
load("jstests/noPassthrough/libs/index_build.js");
load('jstests/replsets/libs/rollback_test.js');
const RollbackResumableIndexBuildTest = class {
/**
* Runs a resumable index build rollback test. The phase that the index build will be in when
* rollback starts is specified by rollbackStartFailPointName. The phase that the index build
* will resume from after rollback completes is specified by rollbackEndFailPointName. If
* either of these points is in the drain writes phase, documents to insert into the side
* writes table must be specified by sideWrites. locksYieldedFailPointName specifies a point
* during the index build between rollbackEndFailPointName and rollbackStartFailPointName at
* which its locks are yielded. Documents specified by insertsToBeRolledBack are inserted after
* transitioning to rollback operations and will be rolled back.
*/
static run(rollbackTest,
dbName,
collName,
indexSpec,
rollbackStartFailPointName,
rollbackStartFailPointData,
rollbackEndFailPointName,
rollbackEndFailPointData,
locksYieldedFailPointName,
insertsToBeRolledBack,
sideWrites = []) {
const originalPrimary = rollbackTest.getPrimary();
if (!ResumableIndexBuildTest.resumableIndexBuildsEnabled(originalPrimary)) {
jsTestLog("Skipping test because resumable index builds are not enabled");
return;
}
rollbackTest.awaitLastOpCommitted();
// Set internalQueryExecYieldIterations to 0 and maxIndexBuildDrainBatchSize to 1 so that
// the index build is guaranteed to yield its locks between the rollback end and start
// failpoints.
assert.commandWorked(
originalPrimary.adminCommand({setParameter: 1, internalQueryExecYieldIterations: 0}));
assert.commandWorked(
originalPrimary.adminCommand({setParameter: 1, maxIndexBuildDrainBatchSize: 1}));
const coll = originalPrimary.getDB(dbName).getCollection(collName);
const indexName = "rollback_resumable_index_build";
const rollbackEndFp =
configureFailPoint(originalPrimary, rollbackEndFailPointName, rollbackEndFailPointData);
const rollbackStartFp = configureFailPoint(
originalPrimary, rollbackStartFailPointName, rollbackStartFailPointData);
const awaitCreateIndex = ResumableIndexBuildTest.createIndexWithSideWrites(
rollbackTest, function(collName, indexSpec, indexName) {
assert.commandFailedWithCode(
db.getCollection(collName).createIndex(indexSpec, {name: indexName}),
ErrorCodes.InterruptedDueToReplStateChange);
}, coll, indexSpec, indexName, sideWrites);
// Wait until we've completed the last operation that won't be rolled back so that we can
// begin the operations that will be rolled back.
rollbackEndFp.wait();
const buildUUID = extractUUIDFromObject(
IndexBuildTest
.assertIndexes(coll, 2, ["_id_"], [indexName], {includeBuildUUIDs: true})[indexName]
.buildUUID);
rollbackTest.transitionToRollbackOperations();
assert.commandWorked(coll.insert(insertsToBeRolledBack));
// Move the index build forward to a point at which its locks are yielded. This allows the
// primary to step down during the call to transitionToSyncSourceOperationsBeforeRollback()
// below.
const locksYieldedFp = configureFailPoint(
originalPrimary, locksYieldedFailPointName, {namespace: coll.getFullName()});
rollbackEndFp.off();
locksYieldedFp.wait();
rollbackTest.transitionToSyncSourceOperationsBeforeRollback();
// The index creation will report as having failed due to InterruptedDueToReplStateChange,
// but it is still building in the background.
awaitCreateIndex();
// Wait until the index build reaches the desired starting point so that we can start the
// rollback.
locksYieldedFp.off();
rollbackStartFp.wait();
// We ignore the return value here because the node will go into rollback immediately upon
// the failpoint being disabled, causing the configureFailPoint command to appear as if it
// failed to run due to a network error despite successfully disabling the failpoint.
startParallelShell(
funWithArgs(function(rollbackStartFailPointName) {
// Wait until we reach rollback state and then disable the failpoint
// so that the index build can be interrupted for rollback.
checkLog.containsJson(db.getMongo(), 21593);
db.adminCommand({configureFailPoint: rollbackStartFailPointName, mode: "off"});
}, rollbackStartFailPointName), originalPrimary.port);
rollbackTest.transitionToSyncSourceOperationsDuringRollback();
// Ensure that the index build was interrupted for rollback.
checkLog.containsJson(originalPrimary, 20347, {
buildUUID: function(uuid) {
return uuid["uuid"]["$uuid"] === buildUUID;
}
});
rollbackTest.transitionToSteadyStateOperations();
// Ensure that the index build completed after rollback.
ResumableIndexBuildTest.assertCompleted(
originalPrimary, coll, buildUUID, 2, ["_id_", indexName]);
assert.commandWorked(
rollbackTest.getPrimary().getDB(dbName).getCollection(collName).dropIndex(indexName));
}
/**
* Runs the resumable index build rollback test in the case where rollback begins after the
* index build has already completed. The point during the index build to roll back to is
* specified by rollbackEndFailPointName. If this point is in the drain writes phase, documents
* to insert into the side writes table must be specified by sideWrites. Documents specified by
* insertsToBeRolledBack are inserted after transitioning to rollback operations and will be
* rolled back.
*/
static runIndexBuildComplete(rollbackTest,
dbName,
collName,
indexSpec,
rollbackEndFailPointName,
rollbackEndFailPointData,
insertsToBeRolledBack,
sideWrites = []) {
const originalPrimary = rollbackTest.getPrimary();
if (!ResumableIndexBuildTest.resumableIndexBuildsEnabled(originalPrimary)) {
jsTestLog("Skipping test because resumable index builds are not enabled");
return;
}
rollbackTest.awaitLastOpCommitted();
const coll = originalPrimary.getDB(dbName).getCollection(collName);
const indexName = "rollback_resumable_index_build";
const rollbackEndFp =
configureFailPoint(originalPrimary, rollbackEndFailPointName, rollbackEndFailPointData);
const awaitCreateIndex = ResumableIndexBuildTest.createIndexWithSideWrites(
rollbackTest, function(collName, indexSpec, indexName) {
assert.commandWorked(db.runCommand({
createIndexes: collName,
indexes: [{key: indexSpec, name: indexName}],
// Commit quorum is disabled so that the index build can
// complete while the primary is isolated and will roll back.
commitQuorum: 0
}));
}, coll, indexSpec, indexName, sideWrites);
// Wait until we reach the desired ending point so that we can begin the operations that
// will be rolled back.
rollbackEndFp.wait();
const buildUUID = extractUUIDFromObject(
IndexBuildTest
.assertIndexes(coll, 2, ["_id_"], [indexName], {includeBuildUUIDs: true})[indexName]
.buildUUID);
rollbackTest.transitionToRollbackOperations();
assert.commandWorked(coll.insert(insertsToBeRolledBack));
// Disable the rollback end failpoint so that the index build can continue and wait for the
// index build to complete.
rollbackEndFp.off();
awaitCreateIndex();
rollbackTest.transitionToSyncSourceOperationsBeforeRollback();
rollbackTest.transitionToSyncSourceOperationsDuringRollback();
rollbackTest.transitionToSteadyStateOperations();
// Ensure that the index build restarted after rollback.
checkLog.containsJson(originalPrimary, 20660, {
buildUUID: function(uuid) {
return uuid["uuid"]["$uuid"] === buildUUID;
}
});
// Ensure that the index build completed after rollback.
ResumableIndexBuildTest.assertCompleted(
originalPrimary, coll, buildUUID, 2, ["_id_", indexName]);
assert.commandWorked(
rollbackTest.getPrimary().getDB(dbName).getCollection(collName).dropIndex(indexName));
}
};
|
