SERVER-37074 Handle interrupted FCV downgrade in validate hook.

3 files changed, 236 insertions, 20 deletions

diff --git a/jstests/hooks/validate_collections.js b/jstests/hooks/validate_collections.js
index 0575bfdf532..5bfd118dcb5 100644
--- a/jstests/hooks/validate_collections.js
+++ b/jstests/hooks/validate_collections.js
@@ -2,6 +2,7 @@
 'use strict';
 
 function CollectionValidator() {
+    load('jstests/libs/feature_compatibility_version.js');
     load('jstests/libs/parallelTester.js');
 
     if (!(this instanceof CollectionValidator)) {
@@ -86,18 +87,20 @@ function CollectionValidator() {
     // Run a separate thread to validate collections on each server in parallel.
     const validateCollectionsThread = function(validatorFunc, host) {
         try {
+            load('jstests/libs/feature_compatibility_version.js');
+
             print('Running validate() on ' + host);
             const conn = new Mongo(host);
             conn.setSlaveOk();
             jsTest.authenticate(conn);
 
-            if (jsTest.options().forceValidationWithFeatureCompatibilityVersion) {
-                let adminDB = conn.getDB('admin');
-                // Make sure this node has the desired FCV.
-                assert.soon(() => {
-                    return adminDB.system.version.findOne({_id: 'featureCompatibilityVersion'})
-                               .version ===
-                        jsTest.options().forceValidationWithFeatureCompatibilityVersion;
+            const requiredFCV = jsTest.options().forceValidationWithFeatureCompatibilityVersion;
+            if (requiredFCV) {
+                // Make sure this node has the desired FCV as it may take time for the updates to
+                // replicate to the nodes that weren't part of the w=majority.
+                assert.soonNoExcept(() => {
+                    checkFCV(conn.getDB('admin'), requiredFCV);
+                    return true;
                 });
             }
 
@@ -123,16 +126,24 @@ function CollectionValidator() {
         let adminDB;
         let originalFCV;
 
-        if (jsTest.options().forceValidationWithFeatureCompatibilityVersion) {
+        const requiredFCV = jsTest.options().forceValidationWithFeatureCompatibilityVersion;
+        if (requiredFCV) {
             let conn = new Mongo(setFCVHost);
             adminDB = conn.getDB('admin');
-            originalFCV =
-                adminDB.system.version.findOne({_id: 'featureCompatibilityVersion'}).version;
-            if (originalFCV !== jsTest.options().forceValidationWithFeatureCompatibilityVersion) {
-                assert.commandWorked(adminDB.adminCommand({
-                    setFeatureCompatibilityVersion:
-                        jsTest.options().forceValidationWithFeatureCompatibilityVersion
-                }));
+            originalFCV = adminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
+
+            if (originalFCV.targetVersion) {
+                // If a previous FCV upgrade or downgrade was interrupted, then we run the
+                // setFeatureCompatibilityVersion command to complete it before attempting to set
+                // the feature compatibility version to 'requiredFCV'.
+                assert.commandWorked(adminDB.runCommand(
+                    {setFeatureCompatibilityVersion: originalFCV.targetVersion}));
+                checkFCV(adminDB, originalFCV.targetVersion);
+            }
+
+            if (originalFCV.version !== requiredFCV && originalFCV.targetVersion !== requiredFCV) {
+                assert.commandWorked(
+                    adminDB.runCommand({setFeatureCompatibilityVersion: requiredFCV}));
             }
         }
 
@@ -155,12 +166,13 @@ function CollectionValidator() {
             });
         }
 
-        if (jsTest.options().forceValidationWithFeatureCompatibilityVersion !== originalFCV) {
-            assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: originalFCV}));
+        if (originalFCV && originalFCV.version !== requiredFCV) {
+            assert.commandWorked(
+                adminDB.runCommand({setFeatureCompatibilityVersion: originalFCV.version}));
         }
     };
 }
 
-// Ensure compatability with existing callers. Cannot use `const` or `let` here since this file may
+// Ensure compatibility with existing callers. Cannot use `const` or `let` here since this file may
 // be loaded more than once.
 var validateCollections = new CollectionValidator().validateCollections;
diff --git a/jstests/libs/feature_compatibility_version.js b/jstests/libs/feature_compatibility_version.js
index 085666afa7b..b9bb718fe73 100644
--- a/jstests/libs/feature_compatibility_version.js
+++ b/jstests/libs/feature_compatibility_version.js
@@ -5,10 +5,13 @@
  * These constants represent the current "latest" and "last-stable" values for the
  * featureCompatibilityVersion parameter. They should only be used for testing of upgrade-downgrade
  * scenarios that are intended to be maintained between releases.
+ *
+ * We cannot use `const` when declaring them because it must be possible to load() this file
+ * multiple times.
  */
 
-const latestFCV = "4.2";
-const lastStableFCV = "4.0";
+var latestFCV = "4.2";
+var lastStableFCV = "4.0";
 
 /**
  * Checks the featureCompatibilityVersion document and server parameter. The
diff --git a/jstests/noPassthrough/validate_hook_resume_fcv_upgrade.js b/jstests/noPassthrough/validate_hook_resume_fcv_upgrade.js
new file mode 100644
index 00000000000..8a3c1d0a5ac
--- /dev/null
+++ b/jstests/noPassthrough/validate_hook_resume_fcv_upgrade.js
@@ -0,0 +1,201 @@
+/**
+ * Verifies that the validate hook is able to upgrade the feature compatibility version of the
+ * server regardless of what state any previous upgrades or downgrades have left it in.
+ */
+
+// The global 'db' variable is used by the data consistency hooks.
+var db;
+
+(function() {
+    "use strict";
+
+    load("jstests/libs/feature_compatibility_version.js");
+
+    // We skip doing the data consistency checks while terminating the cluster because they conflict
+    // with the counts of the number of times the "validate" command is run.
+    TestData.skipCollectionAndIndexValidation = true;
+
+    function makePatternForValidate(dbName, collName) {
+        return new RegExp(
+            "COMMAND.*command " + dbName +
+                "\\.\\$cmd appName: \"MongoDB Shell\" command: validate { validate: \"" + collName +
+                "\"",
+            "g");
+    }
+
+    function makePatternForSetFCV(targetVersion) {
+        return new RegExp(
+            "COMMAND.*command.*appName: \"MongoDB Shell\" command: setFeatureCompatibilityVersion" +
+                " { setFeatureCompatibilityVersion: \"" + targetVersion + "\"",
+            "g");
+    }
+
+    function countMatches(pattern, output) {
+        assert(pattern.global, "the 'g' flag must be used to find all matches");
+
+        let numMatches = 0;
+        while (pattern.exec(output) !== null) {
+            ++numMatches;
+        }
+        return numMatches;
+    }
+
+    function runValidateHook(testCase) {
+        db = testCase.conn.getDB("test");
+        TestData.forceValidationWithFeatureCompatibilityVersion = latestFCV;
+        try {
+            clearRawMongoProgramOutput();
+
+            load("jstests/hooks/run_validate_collections.js");
+
+            // We terminate the processes to ensure that the next call to rawMongoProgramOutput()
+            // will return all of their output.
+            testCase.teardown();
+            return rawMongoProgramOutput();
+        } finally {
+            db = undefined;
+            TestData.forceValidationWithFeatureCompatibilityVersion = undefined;
+        }
+    }
+
+    function testStandalone(additionalSetupFn, {
+        expectedAtTeardownFCV,
+        expectedSetLastStableFCV: expectedSetLastStableFCV = 0,
+        expectedSetLatestFCV: expectedSetLatestFCV = 0
+    } = {}) {
+        const conn =
+            MongoRunner.runMongod({setParameter: {logComponentVerbosity: tojson({command: 1})}});
+        assert.neq(conn, "mongod was unable to start up");
+
+        // Insert a document so the "validate" command has some actual work to do.
+        assert.commandWorked(conn.getDB("test").mycoll.insert({}));
+
+        // Run the additional setup function to put the server into the desired state.
+        additionalSetupFn(conn);
+
+        const output = runValidateHook({
+            conn: conn,
+            teardown: () => {
+                // The validate hook should leave the server with a feature compatibility version of
+                // 'expectedAtTeardownFCV' and no targetVersion.
+                checkFCV(conn.getDB("admin"), expectedAtTeardownFCV);
+                MongoRunner.stopMongod(conn);
+            }
+        });
+
+        const pattern = makePatternForValidate("test", "mycoll");
+        assert.eq(1,
+                  countMatches(pattern, output),
+                  "expected to find " + tojson(pattern) + " from mongod in the log output");
+
+        for (let [targetVersion, expectedCount] of[[lastStableFCV, expectedSetLastStableFCV],
+                                                   [latestFCV, expectedSetLatestFCV]]) {
+            // Since the additionalSetupFn() function may run the setFeatureCompatibilityVersion
+            // command and we don't have a guarantee those log messages were cleared when
+            // clearRawMongoProgramOutput() was called, we assert 'expectedSetLastStableFCV' and
+            // 'expectedSetLatestFCV' as lower bounds.
+            const pattern = makePatternForSetFCV(targetVersion);
+            assert.lte(expectedCount,
+                       countMatches(pattern, output),
+                       "expected to find " + tojson(pattern) + " from mongod in the log output");
+        }
+    }
+
+    function forceInterruptedUpgradeOrDowngrade(conn, targetVersion) {
+        // We create a separate connection to the server exclusively for running the
+        // setFeatureCompatibilityVersion command so only that operation is ever interrupted by
+        // the checkForInterruptFail failpoint.
+        const setFCVConn = new Mongo(conn.host);
+        const myUriRes = assert.commandWorked(setFCVConn.adminCommand({whatsmyuri: 1}));
+        const myUri = myUriRes.you;
+
+        const curOpRes =
+            assert.commandWorked(setFCVConn.adminCommand({currentOp: 1, client: myUri}));
+        const threadName = curOpRes.inprog[0].desc;
+
+        assert.commandWorked(conn.adminCommand({
+            configureFailPoint: "checkForInterruptFail",
+            mode: "alwaysOn",
+            data: {threadName, chance: 0.05},
+        }));
+
+        let attempts = 0;
+        assert.soon(
+            function() {
+                let res = setFCVConn.adminCommand({setFeatureCompatibilityVersion: targetVersion});
+
+                if (res.ok === 1) {
+                    assert.commandWorked(res);
+                } else {
+                    assert.commandFailedWithCode(res, ErrorCodes.Interrupted);
+                }
+
+                ++attempts;
+
+                res = assert.commandWorked(
+                    conn.adminCommand({getParameter: 1, featureCompatibilityVersion: 1}));
+
+                if (res.featureCompatibilityVersion.hasOwnProperty("targetVersion")) {
+                    checkFCV(conn.getDB("admin"), lastStableFCV, targetVersion);
+                    jsTest.log(`Reached partially downgraded state after ${attempts} attempts`);
+                    return true;
+                }
+
+                // Either upgrade the feature compatibility version so we can try downgrading again,
+                // or downgrade the feature compatibility version so we can try upgrading again.
+                // Note that we're using 'conn' rather than 'setFCVConn' to avoid the upgrade being
+                // interrupted.
+                assert.commandWorked(conn.adminCommand({
+                    setFeatureCompatibilityVersion: targetVersion === lastStableFCV ? latestFCV
+                                                                                    : lastStableFCV
+                }));
+            },
+            "failed to get featureCompatibilityVersion document into a partially downgraded" +
+                " state");
+
+        assert.commandWorked(conn.adminCommand({
+            configureFailPoint: "checkForInterruptFail",
+            mode: "off",
+        }));
+    }
+
+    (function testStandaloneInLatestFCV() {
+        testStandalone(conn => {
+            checkFCV(conn.getDB("admin"), latestFCV);
+        }, {expectedAtTeardownFCV: latestFCV});
+    })();
+
+    (function testStandaloneInLastStableFCV() {
+        testStandalone(conn => {
+            assert.commandWorked(
+                conn.adminCommand({setFeatureCompatibilityVersion: lastStableFCV}));
+            checkFCV(conn.getDB("admin"), lastStableFCV);
+        }, {
+            expectedAtTeardownFCV: lastStableFCV,
+            expectedSetLastStableFCV: 1,
+            expectedSetLatestFCV: 1
+        });
+    })();
+
+    (function testStandaloneWithInterruptedFCVDowngrade() {
+        testStandalone(conn => {
+            forceInterruptedUpgradeOrDowngrade(conn, lastStableFCV);
+        }, {
+            expectedAtTeardownFCV: lastStableFCV,
+            expectedSetLastStableFCV: 2,
+            expectedSetLatestFCV: 1
+        });
+    })();
+
+    (function testStandaloneWithInterruptedFCVUpgrade() {
+        testStandalone(conn => {
+            assert.commandWorked(
+                conn.adminCommand({setFeatureCompatibilityVersion: lastStableFCV}));
+            forceInterruptedUpgradeOrDowngrade(conn, latestFCV);
+        }, {
+            expectedAtTeardownFCV: lastStableFCV,
+            expectedSetLastStableFCV: 1,
+            expectedSetLatestFCV: 1
+        });
+    })();
+})();