SERVER-75532 Investigate the high variability of the runtime of analyze_shard_key.js in suites with chunk migration and/or stepdown/kill/terminate

(cherry picked from commit 212b1d6e6f8e9ef949952ac5628679a7a78e849e)

7 files changed, 160 insertions, 21 deletions

diff --git a/buildscripts/resmokeconfig/suites/analyze_shard_key_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/analyze_shard_key_jscore_passthrough.yml
index 48283fb067b..125a928720c 100644
--- a/buildscripts/resmokeconfig/suites/analyze_shard_key_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/analyze_shard_key_jscore_passthrough.yml
@@ -158,6 +158,8 @@ executor:
         queryAnalysisSamplerConfigurationRefreshSecs: 1
         queryAnalysisWriterIntervalSecs: 5
         analyzeShardKeyNumRanges: 10
+        analyzeShardKeySplitPointExpirationSecs: 10
+        ttlMonitorSleepSecs: 5
         logComponentVerbosity:
           verbosity: 0
           sharding: 2
diff --git a/jstests/concurrency/fsm_libs/cluster.js b/jstests/concurrency/fsm_libs/cluster.js
index 44fe572bff0..6c67fa79d05 100644
--- a/jstests/concurrency/fsm_libs/cluster.js
+++ b/jstests/concurrency/fsm_libs/cluster.js
@@ -459,6 +459,12 @@ var Cluster = function(options) {
         return cluster;
     };
 
+    this.getReplicaSets = function getReplicaSets() {
+        assert(initialized, 'cluster must be initialized first');
+        assert(this.isReplication() || this.isSharded());
+        return replSets;
+    };
+
     this.isBalancerEnabled = function isBalancerEnabled() {
         return this.isSharded() && options.sharded.enableBalancer;
     };
diff --git a/jstests/concurrency/fsm_workloads/analyze_shard_key.js b/jstests/concurrency/fsm_workloads/analyze_shard_key.js
index 6ee75f0124a..cf3f4665e63 100644
--- a/jstests/concurrency/fsm_workloads/analyze_shard_key.js
+++ b/jstests/concurrency/fsm_workloads/analyze_shard_key.js
@@ -21,6 +21,7 @@
 load("jstests/concurrency/fsm_libs/extend_workload.js");
 load("jstests/concurrency/fsm_workload_helpers/server_types.js");  // for isMongos
 load("jstests/libs/fail_point_util.js");
+load("jstests/libs/retryable_writes_util.js");
 load("jstests/libs/uuid_util.js");  // for 'extractUUIDFromObject'
 load("jstests/sharding/analyze_shard_key/libs/analyze_shard_key_util.js");
 
@@ -589,8 +590,11 @@ var $config = extendWorkload($config, function($config, $super) {
             // non-duplicate document using a random cursor. 4952606 is the error that the sampling
             // based split policy throws if it fails to find the specified number of split points.
             print(
-                `Failed to analyze the shard key due to duplicate keys returned by random cursor ${
-                    tojsononeline(err)}`);
+                `Failed to analyze the shard key due to duplicate keys returned by random ` +
+                `cursor. Skipping the next ${this.numAnalyzeShardKeySkipsAfterRandomCursorError} ` +
+                `analyzeShardKey states since the analyzeShardKey command is likely to fail with ` +
+                `this error again. ${tojsononeline(err)}`);
+            this.numAnalyzeShardKeySkips = this.numAnalyzeShardKeySkipsAfterRandomCursorError;
             return true;
         }
         if (this.expectedAggregateInterruptErrors.includes(err.code)) {
@@ -599,6 +603,11 @@ var $config = extendWorkload($config, function($config, $super) {
                     tojsononeline(err)}`);
             return true;
         }
+        if (err.code == 7559401) {
+            print(`Failed to analyze the shard key because one of the shards fetched the split ` +
+                  `point documents after the TTL deletions had started. ${tojsononeline(err)}`);
+            return true;
+        }
         return false;
     };
 
@@ -634,6 +643,94 @@ var $config = extendWorkload($config, function($config, $super) {
         return truncatedRes;
     };
 
+    // To avoid leaving a lot of config.analyzeShardKeySplitPoints documents around which could
+    // make restart recovery take a long time, overwrite the values of the
+    // 'analyzeShardKeySplitPointExpirationSecs' and 'ttlMonitorSleepSecs' server parameters to make
+    // the clean up occur as the workload runs, and then restore the original values during
+    // teardown().
+    $config.data.splitPointExpirationSecs = 10;
+    $config.data.ttlMonitorSleepSecs = 5;
+    $config.data.originalSplitPointExpirationSecs = {};
+    $config.data.originalTTLMonitorSleepSecs = {};
+
+    $config.data.overrideSplitPointExpiration = function overrideSplitPointExpiration(cluster) {
+        cluster.executeOnMongodNodes((db) => {
+            const res = assert.commandWorked(db.adminCommand({
+                setParameter: 1,
+                analyzeShardKeySplitPointExpirationSecs: this.splitPointExpirationSecs,
+            }));
+            this.originalSplitPointExpirationSecs[db.getMongo().host] = res.was;
+        });
+    };
+
+    $config.data.overrideTTLMonitorSleepSecs = function overrideTTLMonitorSleepSecs(cluster) {
+        cluster.executeOnMongodNodes((db) => {
+            const res = assert.commandWorked(
+                db.adminCommand({setParameter: 1, ttlMonitorSleepSecs: this.ttlMonitorSleepSecs}));
+            this.originalTTLMonitorSleepSecs[db.getMongo().host] = res.was;
+        });
+    };
+
+    $config.data.restoreSplitPointExpiration = function restoreSplitPointExpiration(cluster) {
+        cluster.executeOnMongodNodes((db) => {
+            assert.commandWorked(db.adminCommand({
+                setParameter: 1,
+                analyzeShardKeySplitPointExpirationSecs:
+                    this.originalSplitPointExpirationSecs[db.getMongo().host],
+            }));
+        });
+    };
+
+    $config.data.restoreTTLMonitorSleepSecs = function restoreTTLMonitorSleepSecs(cluster) {
+        cluster.executeOnMongodNodes((db) => {
+            assert.commandWorked(db.adminCommand({
+                setParameter: 1,
+                ttlMonitorSleepSecs: this.originalTTLMonitorSleepSecs[db.getMongo().host],
+            }));
+        });
+    };
+
+    $config.data.getNumDocuments = function getNumDocuments(db, collName) {
+        const firstBatch =
+            assert
+                .commandWorked(
+                    db.runCommand({aggregate: collName, pipeline: [{$count: "count"}], cursor: {}}))
+                .cursor.firstBatch;
+        return firstBatch.length == 0 ? 0 : firstBatch[0].count;
+    };
+
+    // To avoid leaving unnecessary documents in config database after this workload finishes,
+    // remove all the sampled query documents and split point documents during teardown().
+    $config.data.removeSampledQueryAndSplitPointDocuments =
+        function removeSampledQueryAndSplitPointDocuments(cluster) {
+        cluster.getReplicaSets().forEach(rst => {
+            while (true) {
+                try {
+                    const configDb = rst.getPrimary().getDB("config");
+                    jsTest.log("Removing sampled query documents and split points documents");
+                    jsTest.log(tojsononeline({
+                        sampledQueries: this.getNumDocuments(configDb, "sampledQueries"),
+                        sampledQueriesDiff: this.getNumDocuments(configDb, "sampledQueriesDiff"),
+                        analyzeShardKeySplitPoints:
+                            this.getNumDocuments(configDb, "analyzeShardKeySplitPoints"),
+
+                    }));
+
+                    assert.commandWorked(configDb.sampledQueries.remove({}));
+                    assert.commandWorked(configDb.sampledQueriesDiff.remove({}));
+                    assert.commandWorked(configDb.analyzeShardKeySplitPoints.remove({}));
+                    return;
+                } catch (e) {
+                    if (RetryableWritesUtil.isRetryableCode(e.code)) {
+                        print("Retry documents removal after error: " + tojson(e));
+                        continue;
+                    }
+                    throw e;
+                }
+            }
+        });
+    };
+
     ////
     // The body of the workload.
 
@@ -674,6 +771,9 @@ var $config = extendWorkload($config, function($config, $super) {
                                {comment: this.eligibleForSamplingComment});
         });
 
+        this.overrideSplitPointExpiration(cluster);
+        this.overrideTTLMonitorSleepSecs(cluster);
+
         // On a sharded cluster, running an aggregate command by default involves running getMore
         // commands since the cursor establisher in sharding is pessimistic about the router being
         // stale so it always makes a cursor with {batchSize: 0} on the shards and then run getMore
@@ -712,6 +812,10 @@ var $config = extendWorkload($config, function($config, $super) {
         print("Doing final validation of read and write distribution metrics " +
               tojson(this.truncateAnalyzeShardKeyResponseForLogging(metrics)));
         this.assertReadWriteDistributionMetrics(metrics, true /* isFinal */);
+
+        this.restoreSplitPointExpiration(cluster);
+        this.restoreTTLMonitorSleepSecs(cluster);
+        this.removeSampledQueryAndSplitPointDocuments(cluster);
     };
 
     $config.states.init = function init(db, collName) {
@@ -719,7 +823,18 @@ var $config = extendWorkload($config, function($config, $super) {
         this.metricsDocId = new UUID(this.metricsDocIdString);
     };
 
+    $config.data.numAnalyzeShardKeySkipsAfterRandomCursorError = 5;
+    // Set to a positive value when the analyzeShardKey command fails with an error that is likely
+    // to occur again upon the next try.
+    $config.data.numAnalyzeShardKeySkips = 0;
+
     $config.states.analyzeShardKey = function analyzeShardKey(db, collName) {
+        if (this.numAnalyzeShardKeySkips > 0) {
+            print("Skipping the analyzeShardKey state");
+            this.numAnalyzeShardKeySkips--;
+            return;
+        }
+
         print("Starting analyzeShardKey state");
         const ns = db.getName() + "." + collName;
         const res = db.adminCommand({analyzeShardKey: ns, key: this.shardKeyOptions.shardKey});
diff --git a/jstests/hooks/run_analyze_shard_key_background.js b/jstests/hooks/run_analyze_shard_key_background.js
index 625eb6bc59e..945d449acaa 100644
--- a/jstests/hooks/run_analyze_shard_key_background.js
+++ b/jstests/hooks/run_analyze_shard_key_background.js
@@ -204,6 +204,11 @@ function analyzeShardKey(ns, shardKey, indexKey) {
             tojsononeline(res)}`);
         return res;
     }
+    if (res.code == 7559401) {
+        print(`Failed to analyze the shard key because one of the shards fetched the split ` +
+              `point documents after the TTL deletions had started. ${tojsononeline(err)}`);
+        return res;
+    }
 
     assert.commandWorked(res);
     jsTest.log(`Finished analyzing the shard key: ${tojsononeline(res)}`);
diff --git a/jstests/libs/override_methods/network_error_and_txn_override.js b/jstests/libs/override_methods/network_error_and_txn_override.js
index cbb6ac98ab1..902277b2560 100644
--- a/jstests/libs/override_methods/network_error_and_txn_override.js
+++ b/jstests/libs/override_methods/network_error_and_txn_override.js
@@ -243,10 +243,22 @@ function isRetryableMoveChunkResponse(res) {
         res.code === ErrorCodes.CallbackCanceled;
 }
 
-function isFailedToSatisfyPrimaryReadPreferenceError(msg) {
-    const kReplicaSetMonitorError =
-        /^Could not find host matching read preference.*mode: "primary"/;
-    return msg.match(kReplicaSetMonitorError);
+function isFailedToSatisfyPrimaryReadPreferenceError(res) {
+    const kReplicaSetMonitorError = /Could not find host matching read preference.*mode:.*primary/;
+    if (res.hasOwnProperty("errmsg")) {
+        return res.errmsg.match(kReplicaSetMonitorError);
+    }
+    if (res.hasOwnProperty("message")) {
+        return res.message.match(kReplicaSetMonitorError);
+    }
+    if (res.hasOwnProperty("writeErrors")) {
+        for (let writeError of res.writeErrors) {
+            if (writeError.errmsg.match(kReplicaSetMonitorError)) {
+                return true;
+            }
+        }
+    }
+    return false;
 }
 
 function hasError(res) {
@@ -797,6 +809,17 @@ function shouldRetryWithNetworkErrorOverride(
     res, cmdName, startTime, logError, shouldOverrideAcceptableError = true) {
     assert(configuredForNetworkRetry());
 
+    if (isFailedToSatisfyPrimaryReadPreferenceError(res) &&
+        Date.now() - startTime < 5 * 60 * 1000) {
+        // ReplicaSetMonitor::getHostOrRefresh() waits up to 15 seconds to find the
+        // primary of the replica set. It is possible for the step up attempt of another
+        // node in the replica set to take longer than 15 seconds so we allow retrying
+        // for up to 5 minutes.
+        logError("Failed to find primary when attempting to run command," +
+                 " will retry for another 15 seconds");
+        return kContinue;
+    }
+
     if (RetryableWritesUtil.isRetryableWriteCmdName(cmdName)) {
         if ((cmdName === "findandmodify" || cmdName === "findAndModify") &&
             isRetryableExecutorCodeAndMessage(res.code, res.errmsg)) {
@@ -850,18 +873,6 @@ function shouldRetryWithNetworkErrorOverride(
             return kContinue;
         }
 
-        if (res.hasOwnProperty("errmsg") &&
-            isFailedToSatisfyPrimaryReadPreferenceError(res.errmsg) &&
-            Date.now() - startTime < 5 * 60 * 1000) {
-            // ReplicaSetMonitor::getHostOrRefresh() waits up to 15 seconds to find the
-            // primary of the replica set. It is possible for the step up attempt of another
-            // node in the replica set to take longer than 15 seconds so we allow retrying
-            // for up to 5 minutes.
-            logError("Failed to find primary when attempting to run command," +
-                     " will retry for another 15 seconds");
-            return kContinue;
-        }
-
         // Some sharding commands return raw responses from all contacted shards and there won't
         // be a top level code if shards returned more than one error code, in which case retry
         // if any error is retryable.
@@ -963,7 +974,7 @@ function shouldRetryWithNetworkExceptionOverride(
     if (numNetworkErrorRetries === 0) {
         logError("No retries, throwing");
         throw e;
-    } else if (isFailedToSatisfyPrimaryReadPreferenceError(e.message) &&
+    } else if (isFailedToSatisfyPrimaryReadPreferenceError(e) &&
                Date.now() - startTime < 5 * 60 * 1000) {
         // ReplicaSetMonitor::getHostOrRefresh() waits up to 15 seconds to find the
         // primary of the replica set. It is possible for the step up attempt of another
diff --git a/src/mongo/db/s/analyze_shard_key_read_write_distribution.h b/src/mongo/db/s/analyze_shard_key_read_write_distribution.h
index cacdc651999..6453dc29167 100644
--- a/src/mongo/db/s/analyze_shard_key_read_write_distribution.h
+++ b/src/mongo/db/s/analyze_shard_key_read_write_distribution.h
@@ -254,7 +254,7 @@ template <typename T>
 std::vector<T> addNumByRange(const std::vector<T>& l, const std::vector<T>& r) {
     invariant(!l.empty());
     invariant(!r.empty());
-    tassert(
+    uassert(
         7559401,
         str::stream()
             << "Failed to combine the 'numByRange' metrics from two shards since one has length "
diff --git a/src/mongo/s/analyze_shard_key_server_parameters.idl b/src/mongo/s/analyze_shard_key_server_parameters.idl
index fe441c20673..c7f8bf84f05 100644
--- a/src/mongo/s/analyze_shard_key_server_parameters.idl
+++ b/src/mongo/s/analyze_shard_key_server_parameters.idl
@@ -80,7 +80,7 @@ server_parameters:
         cpp_vartype: AtomicWord<int>
         cpp_varname: gAnalyzeShardKeySplitPointExpirationSecs
         default:
-            expr: 15 * 60
+            expr: 5 * 60
         validator:
             gt: 0