SERVER-45610 Reject commands that read data when node is in RECOVERING state

 create mode 100644 jstests/libs/all_commands_test.js
 create mode 100644 jstests/replsets/db_reads_while_recovering_all_commands.js

(cherry picked from commit 34dc015fcb40b8e4c2c99aadf1a78d7b64de6146)

21 files changed, 555 insertions, 14 deletions

diff --git a/jstests/hooks/validate_collections.js b/jstests/hooks/validate_collections.js
index 31e5d26affe..93fba795625 100644
--- a/jstests/hooks/validate_collections.js
+++ b/jstests/hooks/validate_collections.js
@@ -23,6 +23,12 @@ function validateCollections(db, obj) {
 
     var adminDB = db.getSiblingDB("admin");
 
+    // Skip validating collections for arbiters.
+    if (adminDB.isMaster('admin').arbiterOnly === true) {
+        print('Skipping collection validation on arbiter for db: ' + tojson(db));
+        return success;
+    }
+
     // Don't run validate on view namespaces.
     let filter = {type: "collection"};
     if (jsTest.options().skipValidationOnInvalidViewDefinitions) {
diff --git a/jstests/libs/all_commands_test.js b/jstests/libs/all_commands_test.js
new file mode 100644
index 00000000000..93f16ec868b
--- /dev/null
+++ b/jstests/libs/all_commands_test.js
@@ -0,0 +1,77 @@
+/**
+ * A library for testing behaviors over the set of all available commands.
+ * Users of this library must declare the expected results for all commands and pass
+ * them into a cmdMap object.
+ *
+ * Each entry in the map should have at least the following fields:
+ *      {
+ *          command: {<command object, e.g. 'find: test, filter: {a: 1}'>}
+ *          skip: <reason string> // optional field
+ *      }
+ *
+ * All the logic about how exactly a test should run is defined by the user.
+ * See the 'testAllCommands' function.
+ */
+const AllCommandsTest = (function() {
+    "use strict";
+    /**
+     * Verifies that the command map contains an entry for every command that exists on the server.
+     * This is already called in 'testAllCommands', so there is no need to call this directly.
+     *
+     * @param {Object} conn The shell connection to run the suite over.
+     * @param {map} cmdMap A map of all commands, with their invocations and expected behavior.
+     */
+    function checkCommandCoverage(conn, cmdMap) {
+        const res = assert.commandWorked(conn.adminCommand({listCommands: 1}));
+        const commandsInListCommands = Object.keys(res.commands);
+        const cmdListLen = commandsInListCommands.length;
+        let missingCommands = [];
+
+        // Make sure that all valid commands are covered in the cmdMap.
+        for (var i = 0; i < cmdListLen; i++) {
+            const command = commandsInListCommands[i];
+            if (!cmdMap[command]) {
+                missingCommands.push(command);
+            }
+        }
+        if (missingCommands.length !== 0) {
+            throw new Error("Command map is missing entries for " + missingCommands);
+        }
+
+        return commandsInListCommands;
+    }
+
+    /**
+     * The runner function for this library.
+     * Use the 'skip' option for tests that should not run.
+     *
+     * @param {Object} conn The shell connection to run the suite over.
+     * @param {map} cmdMap A map of all commands, with their invocations and expected behavior.
+     * @param {function} testFn A user-defined function to execute on every command.
+     */
+    function testAllCommands(conn, cmdMap, testFn) {
+        // First check that the map contains all available commands.
+        const commands = checkCommandCoverage(conn, cmdMap);
+
+        for (var i = 0; i < commands.length; i++) {
+            const command = commands[i];
+            const test = cmdMap[command];
+
+            // Coverage already guaranteed above, but check again just in case.
+            assert(test, "Coverage failure: must explicitly define a test for " + command);
+
+            jsTestLog("Running command: " + command);
+
+            if (test.skip !== undefined) {
+                jsTestLog("Skipping " + command + ": " + test.skip);
+                continue;
+            }
+
+            // Run logic specified by caller.
+            jsTestName("Testing " + command);
+            testFn(test);
+        }
+    }
+
+    return {testAllCommands: testAllCommands};
+})();
\ No newline at end of file
diff --git a/jstests/replsets/buildindexes_false_with_system_indexes.js b/jstests/replsets/buildindexes_false_with_system_indexes.js
index cfada55041b..78922e691b2 100644
--- a/jstests/replsets/buildindexes_false_with_system_indexes.js
+++ b/jstests/replsets/buildindexes_false_with_system_indexes.js
@@ -65,6 +65,7 @@
     assert.eq(["_id_"], hiddenAdminDb.system.roles.getIndexes().map(x => x.name).sort());
 
     secondary = rst.restart(secondary, {}, true /* wait for node to become healthy */);
+    rst.awaitSecondaryNodes();
     secondaryAdminDb = secondary.getDB("admin");
     assert.eq(["_id_"], secondaryAdminDb.system.users.getIndexes().map(x => x.name).sort());
     assert.eq(["_id_"], secondaryAdminDb.system.roles.getIndexes().map(x => x.name).sort());
diff --git a/jstests/replsets/db_reads_while_recovering_all_commands.js b/jstests/replsets/db_reads_while_recovering_all_commands.js
new file mode 100644
index 00000000000..dfee8382c55
--- /dev/null
+++ b/jstests/replsets/db_reads_while_recovering_all_commands.js
@@ -0,0 +1,387 @@
+/**
+ * This file defines tests for all existing commands and their expected behavior when run against a
+ * node that is in RECOVERING state.
+ *
+ * Tagged as multiversion-incompatible as the list of commands will vary depeding on version.
+ * @tags: [multiversion_incompatible]
+ */
+
+(function() {
+    "use strict";
+
+    // This will verify the completeness of our map and run all tests.
+    load("jstests/libs/all_commands_test.js");
+
+    const name = jsTestName();
+    const dbName = "alltestsdb";
+    const collName = "alltestscoll";
+    const fullNs = dbName + "." + collName;
+
+    // Pre-written reasons for skipping a test.
+    const isAnInternalCommand = "internal command";
+    const isNotAUserDataRead = "does not return user data";
+    const isPrimaryOnly = "primary only";
+
+    const allCommands = {
+        _addShard: {skip: isPrimaryOnly},
+        _cloneCollectionOptionsFromPrimaryShard: {skip: isPrimaryOnly},
+        _configsvrAddShard: {skip: isPrimaryOnly},
+        _configsvrAddShardToZone: {skip: isPrimaryOnly},
+        _configsvrBalancerCollectionStatus: {skip: isPrimaryOnly},
+        _configsvrBalancerStart: {skip: isPrimaryOnly},
+        _configsvrBalancerStatus: {skip: isPrimaryOnly},
+        _configsvrBalancerStop: {skip: isPrimaryOnly},
+        _configsvrClearJumboFlag: {skip: isPrimaryOnly},
+        _configsvrCommitChunkMerge: {skip: isPrimaryOnly},
+        _configsvrCommitChunkMigration: {skip: isPrimaryOnly},
+        _configsvrCommitChunkSplit: {skip: isPrimaryOnly},
+        _configsvrCommitMovePrimary: {skip: isPrimaryOnly},
+        _configsvrCreateCollection: {skip: isPrimaryOnly},
+        _configsvrCreateDatabase: {skip: isPrimaryOnly},
+        _configsvrDropCollection: {skip: isPrimaryOnly},
+        _configsvrDropDatabase: {skip: isPrimaryOnly},
+        _configsvrEnableSharding: {skip: isPrimaryOnly},
+        _configsvrEnsureChunkVersionIsGreaterThan: {skip: isPrimaryOnly},
+        _configsvrMoveChunk: {skip: isPrimaryOnly},
+        _configsvrMovePrimary: {skip: isPrimaryOnly},
+        _configsvrRefineCollectionShardKey: {skip: isPrimaryOnly},
+        _configsvrRemoveShard: {skip: isPrimaryOnly},
+        _configsvrRemoveShardFromZone: {skip: isPrimaryOnly},
+        _configsvrShardCollection: {skip: isPrimaryOnly},
+        _configsvrUpdateZoneKeyRange: {skip: isPrimaryOnly},
+        _flushDatabaseCacheUpdates: {skip: isPrimaryOnly},
+        _flushRoutingTableCacheUpdates: {skip: isPrimaryOnly},
+        _getNextSessionMods: {skip: isPrimaryOnly},
+        _getUserCacheGeneration: {skip: isNotAUserDataRead},
+        _hashBSONElement: {skip: isNotAUserDataRead},
+        _isSelf: {skip: isNotAUserDataRead},
+        _killOperations: {skip: isNotAUserDataRead},
+        _mergeAuthzCollections: {skip: isPrimaryOnly},
+        _migrateClone: {skip: isPrimaryOnly},
+        _recvChunkAbort: {skip: isPrimaryOnly},
+        _recvChunkCommit: {skip: isPrimaryOnly},
+        _recvChunkStart: {skip: isPrimaryOnly},
+        _recvChunkStatus: {skip: isPrimaryOnly},
+        _shardsvrCloneCatalogData: {skip: isPrimaryOnly},
+        _shardsvrMovePrimary: {skip: isPrimaryOnly},
+        _shardsvrShardCollection: {skip: isPrimaryOnly},
+        _transferMods: {skip: isPrimaryOnly},
+        abortTransaction: {skip: isPrimaryOnly},
+        aggregate: {
+            command: {aggregate: collName, pipeline: [{$match: {}}], cursor: {}},
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NotMasterOrSecondary,
+        },
+        appendOplogNote: {skip: isPrimaryOnly},
+        applyOps: {skip: isPrimaryOnly},
+        authenticate: {skip: isNotAUserDataRead},
+        authSchemaUpgrade: {skip: isPrimaryOnly},
+        availableQueryOptions: {skip: isNotAUserDataRead},
+        buildInfo: {skip: isNotAUserDataRead},
+        captrunc: {skip: isPrimaryOnly},
+        checkShardingIndex: {skip: isPrimaryOnly},
+        cleanupOrphaned: {skip: isPrimaryOnly},
+        clearLog: {skip: isNotAUserDataRead},
+        clone: {skip: isPrimaryOnly},
+        cloneCollection: {skip: isPrimaryOnly},
+        cloneCollectionAsCapped: {skip: isPrimaryOnly},
+        collMod: {skip: isPrimaryOnly},
+        collStats: {
+            command: {aggregate: collName, pipeline: [{$collStats: {count: {}}}], cursor: {}},
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NotMasterOrSecondary,
+        },
+        commitTransaction: {skip: isPrimaryOnly},
+        compact: {skip: isNotAUserDataRead},
+        configureFailPoint: {skip: isNotAUserDataRead},
+        connPoolStats: {skip: isNotAUserDataRead},
+        connPoolSync: {skip: isNotAUserDataRead},
+        connectionStatus: {skip: isNotAUserDataRead},
+        convertToCapped: {skip: isPrimaryOnly},
+        coordinateCommitTransaction: {skip: isNotAUserDataRead},
+        copydb: {skip: isPrimaryOnly},
+        copydbgetnonce: {skip: isPrimaryOnly},
+        copydbsaslstart: {skip: isPrimaryOnly},
+        count: {
+            command: {count: collName},
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NotMasterOrSecondary,
+        },
+        cpuload: {skip: isNotAUserDataRead},
+        create: {skip: isPrimaryOnly},
+        createIndexes: {skip: isPrimaryOnly},
+        createRole: {skip: isPrimaryOnly},
+        createUser: {skip: isPrimaryOnly},
+        currentOp: {skip: isNotAUserDataRead},
+        dataSize: {
+            command: {dataSize: fullNs},
+        },
+        dbCheck: {skip: isPrimaryOnly},
+        dbHash: {
+            command: {dbHash: 1},
+        },
+        dbStats: {
+            command: {dbStats: 1},
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NotMasterOrSecondary,
+        },
+        delete: {skip: isPrimaryOnly},
+        distinct: {
+            command: {distinct: collName, key: "a"},
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NotMasterOrSecondary,
+        },
+        driverOIDTest: {skip: isNotAUserDataRead},
+        drop: {skip: isPrimaryOnly},
+        dropAllRolesFromDatabase: {skip: isPrimaryOnly},
+        dropAllUsersFromDatabase: {skip: isPrimaryOnly},
+        dropConnections: {skip: isNotAUserDataRead},
+        dropDatabase: {skip: isPrimaryOnly},
+        dropIndexes: {skip: isPrimaryOnly},
+        dropRole: {skip: isPrimaryOnly},
+        dropUser: {skip: isPrimaryOnly},
+        echo: {skip: isNotAUserDataRead},
+        emptycapped: {skip: isPrimaryOnly},
+        endSessions: {skip: isNotAUserDataRead},
+        eval: {skip: isPrimaryOnly},
+        explain: {
+            command: {count: collName},
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NotMasterOrSecondary,
+        },
+        features: {skip: isNotAUserDataRead},
+        filemd5: {skip: isNotAUserDataRead},
+        find: {
+            command: {find: collName, filter: {a: 1}},
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NotMasterOrSecondary,
+        },
+        findAndModify: {skip: isPrimaryOnly},
+        flushRouterConfig: {skip: isNotAUserDataRead},
+        forceerror: {skip: isNotAUserDataRead},
+        fsync: {skip: isNotAUserDataRead},
+        fsyncUnlock: {skip: isNotAUserDataRead},
+        geoNear: {
+            command: {geoNear: collName, near: {type: "Point", coordinates: [-10, 10]}},
+            spherical: true,
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NotMasterOrSecondary,
+        },
+        geoSearch: {
+            command: {geoSearch: collName, search: {}, near: [-42, 42], maxDistance: 1},
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NotMasterOrSecondary
+        },
+        getCmdLineOpts: {skip: isNotAUserDataRead},
+        getDatabaseVersion: {skip: isNotAUserDataRead},
+        getDefaultRWConcern: {skip: isNotAUserDataRead},
+        getDiagnosticData: {skip: isNotAUserDataRead},
+        getFreeMonitoringStatus: {skip: isNotAUserDataRead},
+        getLastError: {skip: isPrimaryOnly},
+        getLog: {skip: isNotAUserDataRead},
+        getMore: {
+            command: {getMore: NumberLong(123), collection: collName},
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NotMasterOrSecondary
+        },
+        getParameter: {skip: isNotAUserDataRead},
+        getPrevError: {skip: isNotAUserDataRead},
+        getShardMap: {skip: isNotAUserDataRead},
+        getShardVersion: {skip: isPrimaryOnly},
+        getnonce: {skip: isNotAUserDataRead},
+        godinsert: {skip: isAnInternalCommand},
+        grantPrivilegesToRole: {skip: isPrimaryOnly},
+        grantRolesToRole: {skip: isPrimaryOnly},
+        grantRolesToUser: {skip: isPrimaryOnly},
+        group: {
+            command: {group: {ns: collName, key: {a: 1}}},
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NotMasterOrSecondary
+        },
+        handshake: {skip: isNotAUserDataRead},
+        hostInfo: {skip: isNotAUserDataRead},
+        httpClientRequest: {skip: isNotAUserDataRead},
+        insert: {skip: isPrimaryOnly},
+        internalRenameIfOptionsAndIndexesMatch: {skip: isAnInternalCommand},
+        invalidateUserCache: {skip: isNotAUserDataRead},
+        isMaster: {skip: isNotAUserDataRead},
+        journalLatencyTest: {skip: isNotAUserDataRead},
+        killAllSessions: {skip: isNotAUserDataRead},
+        killAllSessionsByPattern: {skip: isNotAUserDataRead},
+        killCursors: {skip: isNotAUserDataRead},
+        killOp: {skip: isNotAUserDataRead},
+        killSessions: {skip: isNotAUserDataRead},
+        listCollections: {
+            command: {listCollections: 1},
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NotMasterOrSecondary
+        },
+        listCommands: {command: {listCommands: 1}},
+        listDatabases: {
+            command: {listDatabases: 1},
+            isAdminCommand: true,
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NotMasterOrSecondary
+        },
+        listIndexes: {
+            command: {listIndexes: collName},
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NotMasterOrSecondary
+        },
+        lockInfo: {skip: isPrimaryOnly},
+        logApplicationMessage: {skip: isNotAUserDataRead},
+        logRotate: {skip: isNotAUserDataRead},
+        logout: {skip: isNotAUserDataRead},
+        makeSnapshot: {skip: isNotAUserDataRead},
+        mapReduce: {
+            command: {
+                mapReduce: collName,
+                map: function() {},
+                reduce: function(key, vals) {},
+                out: {inline: 1}
+            },
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NotMasterOrSecondary,
+        },
+        "mapreduce.shardedfinish": {skip: isAnInternalCommand},
+        mergeChunks: {skip: isPrimaryOnly},
+        moveChunk: {skip: isPrimaryOnly},
+        parallelCollectionScan: {
+            command: {parallelCollectionScan: collName, numCursors: 1},
+            expectFailure: true,
+            expectedErrorCode: ErrorCodes.NotMasterOrSecondary
+        },
+        ping: {skip: isNotAUserDataRead},
+        planCacheClear: {skip: isNotAUserDataRead},
+        planCacheClearFilters: {skip: isNotAUserDataRead},
+        planCacheListFilters: {skip: isNotAUserDataRead},
+        planCacheListPlans: {skip: isNotAUserDataRead},
+        planCacheListQueryShapes: {skip: isNotAUserDataRead},
+        planCacheSetFilter: {skip: isNotAUserDataRead},
+        prepareTransaction: {skip: isPrimaryOnly},
+        profile: {skip: isPrimaryOnly},
+        reapLogicalSessionCacheNow: {skip: isNotAUserDataRead},
+        refreshLogicalSessionCacheNow: {skip: isNotAUserDataRead},
+        refreshSessions: {skip: isNotAUserDataRead},
+        refreshSessionsInternal: {skip: isNotAUserDataRead},
+        reIndex: {skip: isNotAUserDataRead},
+        renameCollection: {skip: isPrimaryOnly},
+        repairCursor: {skip: isNotAUserDataRead},
+        repairDatabase: {skip: isNotAUserDataRead},
+        replSetAbortPrimaryCatchUp: {skip: isNotAUserDataRead},
+        replSetElect: {skip: isNotAUserDataRead},
+        replSetFreeze: {skip: isNotAUserDataRead},
+        replSetFresh: {skip: isNotAUserDataRead},
+        replSetGetConfig: {skip: isNotAUserDataRead},
+        replSetGetRBID: {skip: isNotAUserDataRead},
+        replSetGetStatus: {skip: isNotAUserDataRead},
+        replSetHeartbeat: {skip: isNotAUserDataRead},
+        replSetInitiate: {skip: isNotAUserDataRead},
+        replSetMaintenance: {skip: isNotAUserDataRead},
+        replSetReconfig: {skip: isNotAUserDataRead},
+        replSetRequestVotes: {skip: isNotAUserDataRead},
+        replSetStepDown: {skip: isNotAUserDataRead},
+        replSetStepUp: {skip: isNotAUserDataRead},
+        replSetSyncFrom: {skip: isNotAUserDataRead},
+        replSetTest: {skip: isNotAUserDataRead},
+        replSetUpdatePosition: {skip: isNotAUserDataRead},
+        replSetResizeOplog: {skip: isNotAUserDataRead},
+        resetError: {skip: isNotAUserDataRead},
+        resync: {skip: isNotAUserDataRead},
+        revokePrivilegesFromRole: {skip: isPrimaryOnly},
+        revokeRolesFromRole: {skip: isPrimaryOnly},
+        revokeRolesFromUser: {skip: isPrimaryOnly},
+        rolesInfo: {skip: isPrimaryOnly},
+        saslContinue: {skip: isPrimaryOnly},
+        saslStart: {skip: isPrimaryOnly},
+        serverStatus: {skip: isNotAUserDataRead},
+        setCommittedSnapshot: {skip: isNotAUserDataRead},
+        setDefaultRWConcern: {skip: isPrimaryOnly},
+        setIndexCommitQuorum: {skip: isPrimaryOnly},
+        setFeatureCompatibilityVersion: {skip: isPrimaryOnly},
+        setFreeMonitoring: {skip: isPrimaryOnly},
+        setParameter: {skip: isNotAUserDataRead},
+        setShardVersion: {skip: isNotAUserDataRead},
+        shardConnPoolStats: {skip: isNotAUserDataRead},
+        shardingState: {skip: isNotAUserDataRead},
+        shutdown: {skip: isNotAUserDataRead},
+        sleep: {skip: isNotAUserDataRead},
+        splitChunk: {skip: isPrimaryOnly},
+        splitVector: {skip: isPrimaryOnly},
+        stageDebug: {skip: isPrimaryOnly},
+        startRecordingTraffic: {skip: isNotAUserDataRead},
+        startSession: {skip: isNotAUserDataRead},
+        stopRecordingTraffic: {skip: isNotAUserDataRead},
+        top: {skip: isNotAUserDataRead},
+        touch: {skip: isNotAUserDataRead},
+        unsetSharding: {skip: isNotAUserDataRead},
+        update: {skip: isPrimaryOnly},
+        updateRole: {skip: isPrimaryOnly},
+        updateUser: {skip: isPrimaryOnly},
+        usersInfo: {skip: isPrimaryOnly},
+        validate: {skip: isNotAUserDataRead},
+        voteCommitIndexBuild: {skip: isNotAUserDataRead},
+        waitForFailPoint: {skip: isNotAUserDataRead},
+        waitForOngoingChunkSplits: {skip: isNotAUserDataRead},
+        whatsmysni: {skip: isNotAUserDataRead},
+        whatsmyuri: {skip: isNotAUserDataRead}
+    };
+
+    /**
+     * Helper function for failing commands or writes that checks the result 'res' of either.
+     * If 'code' is null we only check for failure, otherwise we confirm error code matches as
+     * well. On assert 'msg' is printed.
+     */
+    let assertCommandOrWriteFailed = function(res, code, msg) {
+        if (res.writeErrors !== undefined) {
+            assert.neq(0, res.writeErrors.length, msg);
+        } else if (res.code !== null) {
+            assert.commandFailedWithCode(res, code, msg);
+        } else {
+            assert.commandFailed(res, msg);
+        }
+    };
+
+    // Set up a two-node replica set and put the secondary into RECOVERING state.
+    const rst = new ReplSetTest({name: name, nodes: [{}, {rsConfig: {priority: 0}}]});
+    rst.startSet();
+    rst.initiate();
+
+    const primary = rst.getPrimary();
+    const primaryDb = primary.getDB(dbName);
+    assert.writeOK(primaryDb.getCollection(collName).insert(
+        {a: 42, loc: {type: "Point", coordinates: [1, 1]}}));
+    assert.writeOK(primaryDb.getCollection(collName).insert({b: 4242, pos: {long: 1, lat: 1}}));
+    assert.commandWorked(primaryDb.getCollection(collName).createIndex({loc: "2dsphere"}));
+    assert.commandWorked(primaryDb.getCollection(collName).createIndex(
+        {pos: "geoHaystack", other: 1}, {bucketSize: 1}));
+    rst.awaitReplication();
+
+    const secondary = rst.getSecondary();
+    const secondaryDb = secondary.getDB(dbName);
+
+    // This will lock the node into RECOVERING state until we turn it off.
+    assert.commandWorked(secondary.adminCommand({replSetMaintenance: 1}));
+
+    // Run all tests against the RECOVERING node.
+    AllCommandsTest.testAllCommands(secondary, allCommands, function(test) {
+        const testDb = secondaryDb.getSiblingDB(dbName);
+        let cmdDb = testDb;
+
+        if (test.isAdminCommand) {
+            cmdDb = testDb.getSiblingDB("admin");
+        }
+
+        if (test.expectFailure) {
+            const expectedErrorCode = test.expectedErrorCode;
+            assertCommandOrWriteFailed(
+                cmdDb.runCommand(test.command), expectedErrorCode, () => tojson(test.command));
+        } else {
+            assert.commandWorked(cmdDb.runCommand(test.command), () => tojson(test.command));
+        }
+    });
+
+    // Turn off maintenance mode and stop the test.
+    assert.commandWorked(secondary.adminCommand({replSetMaintenance: 0}));
+    rst.stopSet();
+})();
\ No newline at end of file
diff --git a/jstests/replsets/maintenance2.js b/jstests/replsets/maintenance2.js
index c5e6d9c07e6..0decf05bff3 100644
--- a/jstests/replsets/maintenance2.js
+++ b/jstests/replsets/maintenance2.js
@@ -34,7 +34,7 @@
 
     slaves.forEach(function(slave) {
         // put slave into maintenance (recovery) mode
-        slave.getDB("foo").adminCommand({replSetMaintenance: 1});
+        assert.commandWorked(slave.getDB("foo").adminCommand({replSetMaintenance: 1}));
 
         var stats = slave.getDB("foo").adminCommand({replSetGetStatus: 1});
         assert.eq(stats.myState, 3, "Slave should be in recovering state.");
@@ -54,6 +54,9 @@
         print("count should fail in recovering state...");
         slave.slaveOk = true;
         assert.commandFailed(slave.getDB("foo").runCommand({count: "foo"}));
+
+        // unset maintenance mode when done
+        assert.commandWorked(slave.getDB("foo").adminCommand({replSetMaintenance: 0}));
     });
 
     // Shut down the set and finish the test.
diff --git a/jstests/replsets/maintenance_non-blocking.js b/jstests/replsets/maintenance_non-blocking.js
index 5581ffe3546..aa57be0cf64 100644
--- a/jstests/replsets/maintenance_non-blocking.js
+++ b/jstests/replsets/maintenance_non-blocking.js
@@ -41,6 +41,9 @@ doTest = function() {
 
     print("******* fsyncUnlock'n secondary ************* ");
     sDB.fsyncUnlock();
+
+    print("******* unset replSetMaintenance on secondary ************* ");
+    assert.commandWorked(sDB.adminCommand({replSetMaintenance: 0}));
     replTest.stopSet();
 };
 
diff --git a/src/mongo/db/commands/dbcheck.cpp b/src/mongo/db/commands/dbcheck.cpp
index e462f89250b..219c82051d0 100644
--- a/src/mongo/db/commands/dbcheck.cpp
+++ b/src/mongo/db/commands/dbcheck.cpp
@@ -496,6 +496,10 @@ public:
         return false;
     }
 
+    bool maintenanceOk() const override {
+        return false;
+    }
+
     virtual bool adminOnly() const {
         return false;
     }
diff --git a/src/mongo/db/commands/dbcommands.cpp b/src/mongo/db/commands/dbcommands.cpp
index 261b332fc7d..fec001af137 100644
--- a/src/mongo/db/commands/dbcommands.cpp
+++ b/src/mongo/db/commands/dbcommands.cpp
@@ -911,6 +911,9 @@ public:
     virtual bool slaveOk() const {
         return true;
     }
+    bool maintenanceOk() const override {
+        return false;
+    }
     virtual bool supportsWriteConcern(const BSONObj& cmd) const override {
         return false;
     }
@@ -993,6 +996,9 @@ public:
     virtual bool slaveOk() const {
         return true;
     }
+    bool maintenanceOk() const override {
+        return false;
+    }
     virtual bool supportsWriteConcern(const BSONObj& cmd) const override {
         return false;
     }
diff --git a/src/mongo/db/commands/distinct.cpp b/src/mongo/db/commands/distinct.cpp
index 4ce5c7f78eb..483bb4bd037 100644
--- a/src/mongo/db/commands/distinct.cpp
+++ b/src/mongo/db/commands/distinct.cpp
@@ -67,9 +67,9 @@
 
 namespace mongo {
 
-using std::unique_ptr;
 using std::string;
 using std::stringstream;
+using std::unique_ptr;
 
 namespace dps = ::mongo::dotted_path_support;
 
@@ -85,6 +85,10 @@ public:
         return true;
     }
 
+    virtual bool maintenanceOk() const override {
+        return false;
+    }
+
     virtual bool supportsWriteConcern(const BSONObj& cmd) const override {
         return false;
     }
diff --git a/src/mongo/db/commands/geo_near_cmd.cpp b/src/mongo/db/commands/geo_near_cmd.cpp
index 5380968b199..97ee8014114 100644
--- a/src/mongo/db/commands/geo_near_cmd.cpp
+++ b/src/mongo/db/commands/geo_near_cmd.cpp
@@ -60,8 +60,8 @@
 
 namespace mongo {
 
-using std::unique_ptr;
 using std::stringstream;
+using std::unique_ptr;
 
 class Geo2dFindNearCmd : public ErrmsgCommandDeprecated {
 public:
@@ -76,6 +76,9 @@ public:
     bool slaveOverrideOk() const {
         return true;
     }
+    bool maintenanceOk() const override {
+        return false;
+    }
     bool supportsNonLocalReadConcern(const std::string& dbName, const BSONObj& cmdObj) const final {
         return true;
     }
diff --git a/src/mongo/db/commands/haystack.cpp b/src/mongo/db/commands/haystack.cpp
index 3b00077fe60..5f74b7279a5 100644
--- a/src/mongo/db/commands/haystack.cpp
+++ b/src/mongo/db/commands/haystack.cpp
@@ -78,6 +78,10 @@ public:
         return true;
     }
 
+    bool maintenanceOk() const override {
+        return false;
+    }
+
     bool supportsNonLocalReadConcern(const std::string& dbName, const BSONObj& cmdObj) const final {
         return true;
     }
diff --git a/src/mongo/db/commands/list_collections.cpp b/src/mongo/db/commands/list_collections.cpp
index 3509eb6f955..71cccabbdf6 100644
--- a/src/mongo/db/commands/list_collections.cpp
+++ b/src/mongo/db/commands/list_collections.cpp
@@ -205,6 +205,9 @@ public:
     virtual bool slaveOverrideOk() const {
         return true;
     }
+    virtual bool maintenanceOk() const override {
+        return false;
+    }
     virtual bool adminOnly() const {
         return false;
     }
diff --git a/src/mongo/db/commands/list_databases.cpp b/src/mongo/db/commands/list_databases.cpp
index c405b60ac39..0b7e63ddd02 100644
--- a/src/mongo/db/commands/list_databases.cpp
+++ b/src/mongo/db/commands/list_databases.cpp
@@ -65,6 +65,9 @@ public:
     virtual bool slaveOverrideOk() const {
         return true;
     }
+    virtual bool maintenanceOk() const override {
+        return false;
+    }
     virtual bool adminOnly() const {
         return true;
     }
@@ -165,4 +168,4 @@ public:
         return true;
     }
 } cmdListDatabases;
-}
+}  // namespace mongo
diff --git a/src/mongo/db/commands/list_indexes.cpp b/src/mongo/db/commands/list_indexes.cpp
index a6391248c0c..53665c8ef34 100644
--- a/src/mongo/db/commands/list_indexes.cpp
+++ b/src/mongo/db/commands/list_indexes.cpp
@@ -84,6 +84,9 @@ public:
     virtual bool slaveOverrideOk() const {
         return true;
     }
+    virtual bool maintenanceOk() const override {
+        return false;
+    }
     virtual bool adminOnly() const {
         return false;
     }
diff --git a/src/mongo/db/commands/mr.cpp b/src/mongo/db/commands/mr.cpp
index ced28109df0..6184ebd2835 100644
--- a/src/mongo/db/commands/mr.cpp
+++ b/src/mongo/db/commands/mr.cpp
@@ -1365,6 +1365,10 @@ public:
         return true;
     }
 
+    virtual bool maintenanceOk() const override {
+        return false;
+    }
+
     std::size_t reserveBytesForReply() const override {
         return FindCommon::kInitReplyBufferSize;
     }
@@ -1893,5 +1897,5 @@ public:
 
 } mapReduceFinishCommand;
 
-}  // namespace
+}  // namespace mr
 }  // namespace mongo
diff --git a/src/mongo/db/commands/parallel_collection_scan.cpp b/src/mongo/db/commands/parallel_collection_scan.cpp
index 617bddafd04..1ba05020acb 100644
--- a/src/mongo/db/commands/parallel_collection_scan.cpp
+++ b/src/mongo/db/commands/parallel_collection_scan.cpp
@@ -45,8 +45,8 @@
 
 namespace mongo {
 
-using std::unique_ptr;
 using std::string;
+using std::unique_ptr;
 using stdx::make_unique;
 
 namespace {
@@ -64,10 +64,15 @@ public:
     virtual bool supportsWriteConcern(const BSONObj& cmd) const override {
         return false;
     }
+
     virtual bool slaveOk() const {
         return true;
     }
 
+    virtual bool maintenanceOk() const override {
+        return false;
+    }
+
     bool supportsNonLocalReadConcern(const std::string& dbName, const BSONObj& cmdObj) const final {
         return true;
     }
diff --git a/src/mongo/db/commands/pipeline_command.cpp b/src/mongo/db/commands/pipeline_command.cpp
index bfc5eb99fe1..7eda4ba91a9 100644
--- a/src/mongo/db/commands/pipeline_command.cpp
+++ b/src/mongo/db/commands/pipeline_command.cpp
@@ -71,7 +71,9 @@ public:
                                      const BSONObj& cmdObj) const override {
         return !AggregationRequest::parseNs(dbName, cmdObj).isCollectionlessAggregateNS();
     }
-
+    bool maintenanceOk() const override {
+        return false;
+    }
     ReadWriteType getReadWriteType() const {
         return ReadWriteType::kRead;
     }
diff --git a/src/mongo/s/commands/cluster_db_stats_cmd.cpp b/src/mongo/s/commands/cluster_db_stats_cmd.cpp
index 9fb71f8aea6..2f787527bda 100644
--- a/src/mongo/s/commands/cluster_db_stats_cmd.cpp
+++ b/src/mongo/s/commands/cluster_db_stats_cmd.cpp
@@ -49,6 +49,9 @@ public:
     bool slaveOk() const override {
         return true;
     }
+    virtual bool maintenanceOk() const override {
+        return false;
+    }
     bool adminOnly() const override {
         return false;
     }
diff --git a/src/mongo/s/commands/cluster_list_databases_cmd.cpp b/src/mongo/s/commands/cluster_list_databases_cmd.cpp
index ffea8595f5a..f5f353ba921 100644
--- a/src/mongo/s/commands/cluster_list_databases_cmd.cpp
+++ b/src/mongo/s/commands/cluster_list_databases_cmd.cpp
@@ -46,9 +46,9 @@
 
 namespace mongo {
 
-using std::unique_ptr;
 using std::map;
 using std::string;
+using std::unique_ptr;
 using std::vector;
 
 namespace {
@@ -65,6 +65,10 @@ public:
         return true;
     }
 
+    virtual bool maintenanceOk() const override {
+        return false;
+    }
+
     virtual bool adminOnly() const {
         return true;
     }
diff --git a/src/mongo/s/commands/commands_public.cpp b/src/mongo/s/commands/commands_public.cpp
index a05f8521dc9..d37d8a33a8d 100644
--- a/src/mongo/s/commands/commands_public.cpp
+++ b/src/mongo/s/commands/commands_public.cpp
@@ -71,15 +71,15 @@
 
 namespace mongo {
 
-using std::unique_ptr;
-using std::shared_ptr;
 using std::list;
 using std::make_pair;
 using std::map;
 using std::multimap;
 using std::set;
+using std::shared_ptr;
 using std::string;
 using std::stringstream;
+using std::unique_ptr;
 using std::vector;
 
 namespace {
@@ -1090,6 +1090,10 @@ public:
         return false;
     }
 
+    bool maintenanceOk() const override {
+        return false;
+    }
+
     bool run(OperationContext* opCtx,
              const string& dbName,
              const BSONObj& cmdObj,
@@ -1469,7 +1473,7 @@ public:
 
     void help(stringstream& h) const override {
         h << "http://dochub.mongodb.org/core/geo#GeospatialIndexing-geoNearCommand";
-    }
+    }  // namespace
 
     void addRequiredPrivileges(const std::string& dbname,
                                const BSONObj& cmdObj,
@@ -1608,6 +1612,10 @@ public:
         return false;
     }
 
+    bool maintenanceOk() const override {
+        return false;
+    }
+
     bool run(OperationContext* opCtx,
              const string& dbName,
              const BSONObj& cmdObj,
@@ -1623,7 +1631,8 @@ public:
         return passthrough(opCtx, dbName, dbInfo.primaryId(), cmdObj, result);
     }
 
-} evalCmd;
+}  // namespace mongo
+evalCmd;
 
 class CmdListCollections final : public PublicGridCommand {
 public:
diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js
index 9b509244681..52573614e29 100644
--- a/src/mongo/shell/replsettest.js
+++ b/src/mongo/shell/replsettest.js
@@ -1558,8 +1558,15 @@ var ReplSetTest = function(opts) {
             const replSetConfig =
                 rst.getReplSetConfigFromNode ? rst.getReplSetConfigFromNode() : undefined;
 
-            rst.liveNodes.slaves.forEach(secondary => {
-                secondary.getDBNames().forEach(dbName => combinedDBs.add(dbName));
+            rst.liveNodes.slaves.forEach(node => {
+                // Arbiters have no replicated data.
+                if (node.getDB('admin').isMaster('admin').arbiterOnly) {
+                    print("checkDBHashesForReplSet skipping data of arbiter: " + node.host);
+                    return;
+                }
+                print("checkDBHashesForReplSet going to check data hashes on secondary: " +
+                      node.host);
+                node.getDBNames().forEach(dbName => combinedDBs.add(dbName));
             });
 
             for (var dbName of combinedDBs) {