/**
 * Tests crash recovery with the history store. Runs a workload while repeatedly killing all the
 * nodes of the replica set. Finally ensures that the db hashes match.
 *
 * @tags: [multiversion_incompatible, requires_persistence, requires_replication]
 */
(function() {
"use strict";

load("jstests/libs/parallel_shell_helpers.js");  // For funWithArgs

function workload(iteration) {
    load("jstests/libs/parallelTester.js");  // For Thread().

    const nthreads = 50;
    let threads = [];

    // The workload consists of 50 threads, each of which continuously inserts a document and bloats
    // the inserted document.
    for (let t = 0; t < nthreads; t++) {
        const thread = new Thread(function(t, iteration) {
            for (let i = 0;; i++) {
                const id =
                    db.c.insertMany([{iteration: iteration, doc: i, t: t, count: 0, entries: []}])
                        .insertedIds[0];
                for (let j = 0; j < 200; j++) {
                    const entry = {bigField: new Array(120).fill(j)};
                    db.c.update({_id: id}, {$inc: {count: 1}, $push: {entries: entry}});
                }
            }
        }, t, iteration);
        threads.push(thread);

        thread.start();
    }

    // We expect each thread to throw, because at some point during its execution, all the nodes
    // will be killed.
    for (var t = 0; t < nthreads; t++) {
        assert.throws(() => {
            threads[t].join();
        });
    }
}

const rst = new ReplSetTest({
    name: jsTestName() + "_rst",
    // Only one node is allowed to step up and become primary. This is to prevent synchronization
    // issues where a former primary but now secondary may enter rollback on finding that its
    // oplog has diverged.
    nodes: [{}, {rsConfig: {priority: 0}}, {rsConfig: {priority: 0}}],
    // Set the wiredTigerCacheSizeGB to stress the WT cache and create conditions for the usage of
    // the history store.
    // Set slowms to avoid logging "Slow query" lines. We expect many of them, and so we don't want
    // to clog the log file.
    nodeOptions: {wiredTigerCacheSizeGB: 0.25, slowms: 30000}
});

rst.startSet();
rst.initiate();

for (let i = 0; i < 10; i++) {
    jsTestLog("About to start workload, iteration: " + i);
    const workloadThread = startParallelShell(funWithArgs(workload, i),
                                              rst.getPrimary().port,
                                              false, /* noConnect */
                                              '--quiet' /* Make the shell less verbose */);

    // Allow the workload to run for several seconds as we sleep.
    const sleepSeconds = Math.floor(15 + Math.random() * 45);
    jsTestLog("Sleeping for " + sleepSeconds + " seconds, before killing nodes.");
    sleep(sleepSeconds * 1000);

    jsTestLog("Killing all nodes.");
    for (let i = 0; i < 3; i++) {
        const pid = rst.getNodeId(rst.nodes[i]);
        try {
            rst.stop(
                pid, 9 /* signal */, {skipValidation: true}, {forRestart: true, waitpid: true});
            assert(false, "Expected error after killing node.");
        } catch (e) {
            assert.eq(e.returnCode, MongoRunner.EXIT_SIGKILL, e);
            jsTestLog("Node killed as expected: " + e);
        }
    }

    workloadThread();

    jsTestLog("Restarting repl set.");
    const result = rst.startSet({wiredTigerCacheSizeGB: 0.25}, true /* restart */);
    assert.eq(result.length, 3, result);

    // Wait until the primary and secondaries are up and running.
    jsTestLog("Waiting for primary and secondaries.");
    rst.getPrimary();
    rst.awaitSecondaryNodes();

    jsTestLog("Checking that the db hashes match.");
    rst.checkReplicatedDataHashes();
}

rst.stopSet();
})();