SERVER-61754 dbCheck does not hold strong locks during batches

The dbCheck command accepts an optional (default true) parameter,
'snapshotRead', that uses point-in-time reads to check each batch. When
true, dbCheck does not block writes. When set to false, dbCheck reverts to
blocking writes.

(cherry picked from commit da10567348cf7883e2ec40a441e38efc17c23385)

2 files changed, 105 insertions, 0 deletions

diff --git a/jstests/noPassthrough/dbcheck_no_history_on_secondary.js b/jstests/noPassthrough/dbcheck_no_history_on_secondary.js
new file mode 100644
index 00000000000..4ca67e82d31
--- /dev/null
+++ b/jstests/noPassthrough/dbcheck_no_history_on_secondary.js
@@ -0,0 +1,84 @@
+/**
+ * Confirms that a dbCheck batch operation logs an error in the health log of a secondary that does
+ * not have enough available history.
+ *
+ * @tags: [
+ *   multiversion_incompatible,
+ *   requires_replication,
+ * ]
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/fail_point_util.js");
+
+const replTest = new ReplSetTest({
+    name: "dbcheck_lag",
+    nodes: 2,
+    nodeOptions: {
+        setParameter: {
+            // Set the history window to ensure that dbCheck does not cause the server to crash
+            // even when no history is available.
+            minSnapshotHistoryWindowInSeconds: 0,
+        }
+    }
+});
+replTest.startSet();
+replTest.initiate();
+
+const primary = replTest.getPrimary();
+const secondary = replTest.getSecondary();
+
+const testDB = primary.getDB('test');
+for (let i = 0; i < 100; i++) {
+    assert.commandWorked(testDB.foo.insert({a: i}));
+}
+
+const sleepMs = 3000;
+const fp = configureFailPoint(primary, 'SleepDbCheckInBatch', {sleepMs: sleepMs});
+
+// Returns immediately and starts a background task.
+assert.commandWorked(testDB.getSiblingDB('test').runCommand({dbCheck: 1}));
+
+// Write some data to advance the durable timestamp while we're waiting for dbCheck to run.
+for (let i = 0; i < 100; i++) {
+    assert.commandWorked(testDB.foo.insert({_id: i}));
+}
+
+fp.off();
+
+// Wait for primary to complete the task and the secondaries to process. Note that we still have to
+// wait for the health log entries to appear because they are logged shortly after processing
+// batches.
+const dbCheckCompleted = (db) => {
+    return db.currentOp().inprog.filter(x => x["desc"] == "dbCheck")[0] === undefined;
+};
+assert.soon(() => dbCheckCompleted(testDB), "dbCheck timed out");
+replTest.awaitReplication();
+
+{
+    // Expect no errors on the primary. Health log write is logged after batch is replicated.
+    const healthlog = primary.getDB('local').system.healthlog;
+    assert.soon(() => healthlog.find().hasNext(), 'expected health log to not be empty');
+
+    const errors = healthlog.find({severity: 'error'});
+    assert(!errors.hasNext(), () => 'expected no errors, found: ' + tojson(errors.next()));
+}
+
+{
+    // Expect an error on the secondary.
+    const healthlog = secondary.getDB('local').system.healthlog;
+    assert.soon(() => healthlog.find().hasNext(), 'expected health log to not be empty');
+
+    const errors = healthlog.find({severity: 'error'});
+    assert(errors.hasNext(), 'expected error, found none');
+
+    const error = errors.next();
+    assert.eq(
+        false, error.data.success, "expected failure, found success. log entry: " + tojson(error));
+    assert(error.data.error.includes("SnapshotTooOld"),
+           "expected to find SnapshotTooOld error. log entry: " + tojson(error));
+}
+
+replTest.stopSet();
+})();
diff --git a/jstests/replsets/dbcheck.js b/jstests/replsets/dbcheck.js
index 147c6a736f6..e0c5a5d1610 100644
--- a/jstests/replsets/dbcheck.js
+++ b/jstests/replsets/dbcheck.js
@@ -5,6 +5,7 @@
  *   # We need persistence as we temporarily restart nodes as standalones.
  *   requires_persistence,
  *   assumes_against_mongod_not_mongos,
+ *   requires_fcv_50,
  * ]
  */
 
@@ -199,6 +200,25 @@ function simpleTestConsistent() {
     });
 }
 
+function simpleTestNonSnapshot() {
+    let primary = replSet.getPrimary();
+    clearLog();
+
+    assert.neq(primary, undefined);
+    let db = primary.getDB(dbName);
+    assert.commandWorked(db.runCommand({"dbCheck": multiBatchSimpleCollName, snapshotRead: false}));
+
+    awaitDbCheckCompletion(db, multiBatchSimpleCollName);
+
+    checkLogAllConsistent(primary);
+    checkTotalCounts(primary, db[multiBatchSimpleCollName]);
+
+    forEachSecondary(function(secondary) {
+        checkLogAllConsistent(secondary);
+        checkTotalCounts(secondary, secondary.getDB(dbName)[multiBatchSimpleCollName]);
+    });
+}
+
 // Same thing, but now with concurrent updates.
 function concurrentTestConsistent() {
     let primary = replSet.getPrimary();
@@ -227,6 +247,7 @@ function concurrentTestConsistent() {
 }
 
 simpleTestConsistent();
+simpleTestNonSnapshot();
 concurrentTestConsistent();
 
 // Test the various other parameters.