1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
|
/**
* Confirms that aborting a background index build on a primary node during step down does not leave
* the node in an inconsistent state.
*
* @tags: [
* requires_replication,
* ]
*/
(function() {
"use strict";
load("jstests/libs/fail_point_util.js");
load("jstests/libs/feature_flag_util.js");
load('jstests/noPassthrough/libs/index_build.js');
const rst = new ReplSetTest({nodes: 3});
rst.startSet();
rst.initiate();
let primary = rst.getPrimary();
let testDB = primary.getDB('test');
let coll = testDB.getCollection('test');
assert.commandWorked(coll.insert({a: 1}));
IndexBuildTest.pauseIndexBuilds(primary);
const awaitIndexBuild = IndexBuildTest.startIndexBuild(
primary, coll.getFullName(), {a: 1}, {}, [ErrorCodes.InterruptedDueToReplStateChange]);
// When the index build starts, find its op id.
const opId = IndexBuildTest.waitForIndexBuildToScanCollection(testDB, coll.getName(), 'a_1');
IndexBuildTest.assertIndexBuildCurrentOpContents(testDB, opId, (op) => {
jsTestLog('Inspecting db.currentOp() entry for index build: ' + tojson(op));
assert.eq(
undefined,
op.connectionId,
'Was expecting IndexBuildsCoordinator op; found db.currentOp() for connection thread instead: ' +
tojson(op));
assert.eq(coll.getFullName(),
op.ns,
'Unexpected ns field value in db.currentOp() result for index build: ' + tojson(op));
});
// Index build should be present in the config.system.indexBuilds collection.
const indexMap =
IndexBuildTest.assertIndexes(coll, 2, ["_id_"], ["a_1"], {includeBuildUUIDs: true});
const indexBuildUUID = indexMap['a_1'].buildUUID;
assert(primary.getCollection('config.system.indexBuilds').findOne({_id: indexBuildUUID}));
assert.commandWorked(primary.adminCommand(
{configureFailPoint: "hangIndexBuildBeforeAbortCleanUp", mode: "alwaysOn"}));
// Signal the index builder thread to exit.
assert.commandWorked(testDB.killOp(opId));
// Wait for the index build to hang before cleaning up.
IndexBuildTest.resumeIndexBuilds(primary);
assert.commandWorked(primary.adminCommand({
waitForFailPoint: "hangIndexBuildBeforeAbortCleanUp",
timesEntered: 1,
maxTimeMS: kDefaultWaitForFailPointTimeout
}));
// Step down the primary.
assert.commandWorked(testDB.adminCommand({replSetStepDown: 10, secondaryCatchUpPeriodSecs: 10}));
rst.waitForState(primary, ReplSetTest.State.SECONDARY);
// Resume the abort.
assert.commandWorked(
primary.adminCommand({configureFailPoint: "hangIndexBuildBeforeAbortCleanUp", mode: "off"}));
awaitIndexBuild();
const gracefulIndexBuildFlag = FeatureFlagUtil.isEnabled(testDB, "IndexBuildGracefulErrorHandling");
if (!gracefulIndexBuildFlag) {
// We expect the node to crash without this feature enabled.
assert.soon(function() {
return rawMongoProgramOutput().search(/Fatal assertion.*51101/) >= 0;
});
// After restarting the old primary, we expect that the index build completes successfully.
rst.stop(
primary.nodeId, undefined, {forRestart: true, allowedExitCode: MongoRunner.EXIT_ABORT});
rst.start(primary.nodeId, undefined, true /* restart */);
}
// Wait for primary and secondaries to reach goal state, and for the index build to complete.
primary = rst.waitForPrimary();
rst.awaitSecondaryNodes();
rst.awaitReplication();
if (gracefulIndexBuildFlag) {
// "Index build: joined after abort".
checkLog.containsJson(primary, 20655);
// Wait for the index build to complete.
rst.awaitReplication();
// Verify that the interrupted index build was aborted.
IndexBuildTest.assertIndexes(rst.getPrimary().getDB('test').getCollection('test'), 1, ['_id_']);
IndexBuildTest.assertIndexes(
rst.getSecondary().getDB('test').getCollection('test'), 1, ['_id_']);
} else {
// Verify that the stepped up node completed the index build.
IndexBuildTest.assertIndexes(
rst.getPrimary().getDB('test').getCollection('test'), 2, ['_id_', 'a_1']);
IndexBuildTest.assertIndexes(
rst.getSecondary().getDB('test').getCollection('test'), 2, ['_id_', 'a_1']);
// This test triggers an unclean shutdown (an fassert), which may cause inaccurate fast counts.
TestData.skipEnforceFastCountOnValidate = true;
}
rst.stopSet();
})();
|
