1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
|
/**
* Tests crash recovery with the history store. Runs a workload while repeatedly killing all the
* nodes of the replica set. Finally ensures that the db hashes match.
*
* @tags: [multiversion_incompatible, requires_persistence]
*/
(function() {
"use strict";
load("jstests/libs/parallel_shell_helpers.js"); // For funWithArgs
function workload(iteration) {
load("jstests/libs/parallelTester.js"); // For Thread().
const nthreads = 50;
let threads = [];
// The workload consists of 50 threads, each of which continuously inserts a document and bloats
// the inserted document.
for (let t = 0; t < nthreads; t++) {
const thread = new Thread(function(t, iteration) {
for (let i = 0;; i++) {
const id =
db.c.insertMany([{iteration: iteration, doc: i, t: t, count: 0, entries: []}])
.insertedIds[0];
for (let j = 0; j < 200; j++) {
const entry = {bigField: new Array(120).fill(j)};
db.c.update({_id: id}, {$inc: {count: 1}, $push: {entries: entry}});
}
}
}, t, iteration);
threads.push(thread);
thread.start();
}
// We expect each thread to throw, because at some point during its execution, all the nodes
// will be killed.
for (var t = 0; t < nthreads; t++) {
assert.throws(() => {
threads[t].join();
});
}
}
const rst = new ReplSetTest({
name: jsTestName() + "_rst",
// Only one node is allowed to step up and become primary. This is to prevent synchronization
// issues where a former primary but now secondary may enter rollback on finding that its
// oplog has diverged.
nodes: [{}, {rsConfig: {priority: 0}}, {rsConfig: {priority: 0}}],
// Set the wiredTigerCacheSizeGB to stress the WT cache and create conditions for the usage of
// the history store.
// Set slowms to avoid logging "Slow query" lines. We expect many of them, and so we don't want
// to clog the log file.
nodeOptions: {wiredTigerCacheSizeGB: 0.25, slowms: 30000}
});
rst.startSet();
rst.initiate();
for (let i = 0; i < 20; i++) {
jsTestLog("About to start workload, iteration: " + i);
const workloadThread = startParallelShell(funWithArgs(workload, i),
rst.getPrimary().port,
false, /* noConnect */
'--quiet' /* Make the shell less verbose */);
// Allow the workload to run for several seconds as we sleep.
const sleepSeconds = Math.floor(15 + Math.random() * 45);
jsTestLog("Sleeping for " + sleepSeconds + " seconds, before killing nodes.");
sleep(sleepSeconds * 1000);
jsTestLog("Killing all nodes.");
for (let i = 0; i < 3; i++) {
const pid = rst.getNodeId(rst.nodes[i]);
try {
rst.stop(
pid, 9 /* signal */, {skipValidation: true}, {forRestart: true, waitpid: true});
assert(false, "Expected error after killing node.");
} catch (e) {
assert.eq(e.returnCode, MongoRunner.EXIT_SIGKILL, e);
jsTestLog("Node killed as expected: " + e);
}
}
workloadThread();
jsTestLog("Restarting repl set.");
const result = rst.startSet({wiredTigerCacheSizeGB: 0.25}, true /* restart */);
assert.eq(result.length, 3, result);
// Wait until the primary and secondaries are up and running.
jsTestLog("Waiting for primary and secondaries.");
rst.getPrimary();
rst.awaitSecondaryNodes();
jsTestLog("Checking that the db hashes match.");
rst.checkReplicatedDataHashes();
}
rst.stopSet();
})();
|
