SERVER-43867 Move check_log.js inside src/mongo/shell/ SERVER-43867 Work around unrecoverability of rollback via refetch in tests

(cherry picked from commit 7bb6711e9df7f135af19cd4f371ab21c60c026ed)
(cherry picked from commit 7c42573b96ecfa43fbf76aa01cd659ee00fc6103)

4 files changed, 187 insertions, 2 deletions

diff --git a/src/mongo/scripting/engine.cpp b/src/mongo/scripting/engine.cpp
index 676b20c6f18..2e5a0e2abd0 100644
--- a/src/mongo/scripting/engine.cpp
+++ b/src/mongo/scripting/engine.cpp
@@ -296,6 +296,7 @@ ScriptingFunction Scope::createFunction(const char* code) {
 
 namespace JSFiles {
 extern const JSFile collection;
+extern const JSFile check_log;
 extern const JSFile crud_api;
 extern const JSFile db;
 extern const JSFile explain_query;
@@ -322,6 +323,7 @@ void Scope::execCoreFiles() {
     execSetup(JSFiles::query);
     execSetup(JSFiles::bulk_api);
     execSetup(JSFiles::error_codes);
+    execSetup(JSFiles::check_log);
     execSetup(JSFiles::collection);
     execSetup(JSFiles::crud_api);
     execSetup(JSFiles::explain_query);
diff --git a/src/mongo/shell/SConscript b/src/mongo/shell/SConscript
index 78a88004b2f..f83d615678f 100644
--- a/src/mongo/shell/SConscript
+++ b/src/mongo/shell/SConscript
@@ -36,6 +36,7 @@ js_header = env.JSHeader(
     source=[
         "assert.js",
         "bulk_api.js",
+        "check_log.js",
         "collection.js",
         "crud_api.js",
         "db.js",
diff --git a/src/mongo/shell/check_log.js b/src/mongo/shell/check_log.js
new file mode 100644
index 00000000000..c21a885c2db
--- /dev/null
+++ b/src/mongo/shell/check_log.js
@@ -0,0 +1,137 @@
+/*
+ * Helper functions which connect to a server, and check its logs for particular strings.
+ */
+var checkLog;
+
+(function() {
+"use strict";
+
+if (checkLog) {
+    return;  // Protect against this file being double-loaded.
+}
+
+checkLog = (function() {
+    let getGlobalLog = function(conn) {
+        let cmdRes;
+        try {
+            cmdRes = conn.adminCommand({getLog: 'global'});
+        } catch (e) {
+            // Retry with network errors.
+            print("checkLog ignoring failure: " + e);
+            return null;
+        }
+
+        return assert.commandWorked(cmdRes).log;
+    };
+
+    /*
+     * Calls the 'getLog' function on the provided connection 'conn' to see if the provided msg
+     * is found in the logs. Note: this function does not throw an exception, so the return
+     * value should not be ignored.
+     */
+    const checkContainsOnce = function(conn, msg) {
+        const logMessages = getGlobalLog(conn);
+        if (logMessages === null) {
+            return false;
+        }
+        for (let logMsg of logMessages) {
+            if (logMsg.includes(msg)) {
+                return true;
+            }
+        }
+        return false;
+    };
+
+    /*
+     * Calls the 'getLog' function at regular intervals on the provided connection 'conn' until
+     * the provided 'msg' is found in the logs, or it times out. Throws an exception on timeout.
+     */
+    let contains = function(conn, msg, timeout = 5 * 60 * 1000) {
+        assert.soon(function() {
+            return checkContainsOnce(conn, msg);
+        }, 'Could not find log entries containing the following message: ' + msg, timeout, 300);
+    };
+
+    /*
+     * Calls the 'getLog' function at regular intervals on the provided connection 'conn' until
+     * the provided 'msg' is found in the logs 'expectedCount' times, or it times out.
+     * Throws an exception on timeout. If 'exact' is true, checks whether the count is exactly
+     * equal to 'expectedCount'. Otherwise, checks whether the count is at least equal to
+     * 'expectedCount'. Early returns when at least 'expectedCount' entries are found.
+     */
+    let containsWithCount = function(
+        conn, msg, expectedCount, timeout = 5 * 60 * 1000, exact = true) {
+        let expectedStr = exact ? 'exactly ' : 'at least ';
+        assert.soon(
+            function() {
+                let count = 0;
+                let logMessages = getGlobalLog(conn);
+                if (logMessages === null) {
+                    return false;
+                }
+                for (let i = 0; i < logMessages.length; i++) {
+                    if (logMessages[i].indexOf(msg) != -1) {
+                        count++;
+                    }
+                    if (!exact && count >= expectedCount) {
+                        print("checkLog found at least " + expectedCount +
+                              " log entries containing the following message: " + msg);
+                        return true;
+                    }
+                }
+
+                return exact ? expectedCount === count : expectedCount <= count;
+            },
+            'Did not find ' + expectedStr + expectedCount + ' log entries containing the ' +
+                'following message: ' + msg,
+            timeout,
+            300);
+    };
+
+    /*
+     * Similar to containsWithCount, but checks whether there are at least 'expectedCount'
+     * instances of 'msg' in the logs.
+     */
+    let containsWithAtLeastCount = function(conn, msg, expectedCount, timeout = 5 * 60 * 1000) {
+        containsWithCount(conn, msg, expectedCount, timeout, /*exact*/ false);
+    };
+
+    /*
+     * Converts a scalar or object to a string format suitable for matching against log output.
+     * Field names are not quoted, and by default strings which are not within an enclosing
+     * object are not escaped. Similarly, integer values without an enclosing object are
+     * serialized as integers, while those within an object are serialized as floats to one
+     * decimal point. NumberLongs are unwrapped prior to serialization.
+     */
+    const formatAsLogLine = function(value, escapeStrings, toDecimal) {
+        if (typeof value === "string") {
+            return (escapeStrings ? `"${value}"` : value);
+        } else if (typeof value === "number") {
+            return (Number.isInteger(value) && toDecimal ? value.toFixed(1) : value);
+        } else if (value instanceof NumberLong) {
+            return `${value}`.match(/NumberLong..(.*)../m)[1];
+        } else if (typeof value !== "object") {
+            return value;
+        } else if (Object.keys(value).length === 0) {
+            return Array.isArray(value) ? "[]" : "{}";
+        }
+        let serialized = [];
+        escapeStrings = toDecimal = true;
+        for (let fieldName in value) {
+            const valueStr = formatAsLogLine(value[fieldName], escapeStrings, toDecimal);
+            serialized.push(Array.isArray(value) ? valueStr : `${fieldName}: ${valueStr}`);
+        }
+        return (Array.isArray(value) ? `[ ${serialized.join(', ')} ]`
+                                     : `{ ${serialized.join(', ')} }`);
+    };
+
+    return {
+        getGlobalLog: getGlobalLog,
+        checkContainsOnce: checkContainsOnce,
+        contains: contains,
+        containsWithCount: containsWithCount,
+        containsWithAtLeastCount: containsWithAtLeastCount,
+        formatAsLogLine: formatAsLogLine
+    };
+})();
+})();
diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js
index 76d2f23171a..8c2ac8f1673 100644
--- a/src/mongo/shell/replsettest.js
+++ b/src/mongo/shell/replsettest.js
@@ -601,8 +601,8 @@ var ReplSetTest = function(opts) {
             var ready = true;
 
             for (var i = 0; i < len; i++) {
-                var isMaster = slavesToCheck[i].getDB("admin").runCommand({ismaster: 1});
-                var arbiter = (isMaster.arbiterOnly == undefined ? false : isMaster.arbiterOnly);
+                var isMaster = slavesToCheck[i].adminCommand({ismaster: 1});
+                var arbiter = (isMaster.arbiterOnly === undefined ? false : isMaster.arbiterOnly);
                 ready = ready && (isMaster.secondary || arbiter);
             }
 
@@ -611,6 +611,51 @@ var ReplSetTest = function(opts) {
     };
 
     /**
+     * A special version of awaitSecondaryNodes() used exclusively by rollback_test.js.
+     * Wraps around awaitSecondaryNodes() itself and checks for an unrecoverable rollback
+     * if it throws.
+     */
+    this.awaitSecondaryNodesForRollbackTest = function(
+        timeout, slaves, connToCheckForUnrecoverableRollback) {
+        try {
+            this.awaitSecondaryNodes(timeout, slaves);
+        } catch (originalEx) {
+            // There is a special case where we expect the (rare) possibility of unrecoverable
+            // rollbacks with EMRC:false in rollback suites with unclean shutdowns.
+            jsTestLog("Exception in 'awaitSecondaryNodes', checking for unrecoverable rollback");
+            if (connToCheckForUnrecoverableRollback) {
+                const conn = connToCheckForUnrecoverableRollback;
+
+                const statusRes = assert.commandWorked(conn.adminCommand({replSetGetStatus: 1}));
+                const isRecovering = (statusRes.myState === ReplSetTest.State.RECOVERING);
+                const hasNoSyncSource = (statusRes.syncSourceId === -1);
+
+                const cmdLineOptsRes = assert.commandWorked(conn.adminCommand("getCmdLineOpts"));
+                const hasEMRCFalse =
+                    (cmdLineOptsRes.parsed.replication.enableMajorityReadConcern === false);
+
+                if (isRecovering && hasNoSyncSource && hasEMRCFalse) {
+                    try {
+                        const n = this.getNodeId(conn);
+                        const connToCheck = _useBridge ? _unbridgedNodes[n] : this.nodes[n];
+                        // Confirm that the node is unable to recover after rolling back.
+                        checkLog.contains(
+                            connToCheck,
+                            "remote oplog does not contain entry with optime matching our required optime ",
+                            60 * 1000);
+                    } catch (checkLogEx) {
+                        throw originalEx;
+                    }
+                    // Add this info to the original exception.
+                    originalEx.unrecoverableRollbackDetected = true;
+                }
+            }
+            // Re-throw the original exception in all cases.
+            throw originalEx;
+        }
+    };
+
+    /**
      * Blocks until the specified node says it's syncing from the given upstream node.
      */
     this.awaitSyncSource = function(node, upstreamNode, timeout) {