SERVER-32567 Replace queryoptimizer3.js with yield_with_drop.js FSM testr4.3.2

18 files changed, 150 insertions, 97 deletions

diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml
index 21d73d0933e..945a8bd954b 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml
@@ -141,6 +141,7 @@ selector:
   - jstests/concurrency/fsm_workloads/yield_sort.js
   - jstests/concurrency/fsm_workloads/yield_sort_merge.js
   - jstests/concurrency/fsm_workloads/yield_text.js
+  - jstests/concurrency/fsm_workloads/yield_with_drop.js
 
   # Uses non retryable writes.
   - jstests/concurrency/fsm_workloads/remove_and_bulk_insert.js
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml
index a8df6a100d7..80f4b700ae6 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml
@@ -141,6 +141,7 @@ selector:
   - jstests/concurrency/fsm_workloads/yield_sort.js
   - jstests/concurrency/fsm_workloads/yield_sort_merge.js
   - jstests/concurrency/fsm_workloads/yield_text.js
+  - jstests/concurrency/fsm_workloads/yield_with_drop.js
 
   # Uses non retryable writes.
   - jstests/concurrency/fsm_workloads/remove_and_bulk_insert.js
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml
index c83cf181b7f..6b2b73b0d41 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml
@@ -129,6 +129,7 @@ selector:
   - jstests/concurrency/fsm_workloads/yield_sort.js
   - jstests/concurrency/fsm_workloads/yield_sort_merge.js
   - jstests/concurrency/fsm_workloads/yield_text.js
+  - jstests/concurrency/fsm_workloads/yield_with_drop.js
 
   # Uses non retryable writes.
   - jstests/concurrency/fsm_workloads/remove_and_bulk_insert.js
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml
index 916c3911722..e7aa2ae5cc3 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml
@@ -134,6 +134,7 @@ selector:
   - jstests/concurrency/fsm_workloads/yield_sort.js
   - jstests/concurrency/fsm_workloads/yield_sort_merge.js
   - jstests/concurrency/fsm_workloads/yield_text.js
+  - jstests/concurrency/fsm_workloads/yield_with_drop.js
 
   # Uses non retryable writes.
   - jstests/concurrency/fsm_workloads/remove_and_bulk_insert.js
diff --git a/buildscripts/resmokeconfig/suites/multi_shard_local_read_write_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/multi_shard_local_read_write_multi_stmt_txn_jscore_passthrough.yml
index d77680f2395..d8b75981068 100644
--- a/buildscripts/resmokeconfig/suites/multi_shard_local_read_write_multi_stmt_txn_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/multi_shard_local_read_write_multi_stmt_txn_jscore_passthrough.yml
@@ -294,7 +294,6 @@ selector:
   - jstests/core/killop_drop_collection.js
   - jstests/core/loadserverscripts.js
   - jstests/core/mr_killop.js
-  - jstests/core/queryoptimizer3.js
   - jstests/core/remove9.js
   - jstests/core/removeb.js
   - jstests/core/removec.js
diff --git a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_jscore_passthrough.yml
index 18f468c702d..626e41756ac 100644
--- a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_jscore_passthrough.yml
@@ -321,7 +321,6 @@ selector:
   - jstests/core/killop_drop_collection.js
   - jstests/core/loadserverscripts.js
   - jstests/core/mr_killop.js
-  - jstests/core/queryoptimizer3.js
   - jstests/core/remove9.js
   - jstests/core/removeb.js
   - jstests/core/removec.js
diff --git a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough.yml
index 4ca32a5ce60..0841a15703a 100644
--- a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough.yml
@@ -306,7 +306,6 @@ selector:
   - jstests/core/killop_drop_collection.js
   - jstests/core/loadserverscripts.js
   - jstests/core/mr_killop.js
-  - jstests/core/queryoptimizer3.js
   - jstests/core/remove9.js
   - jstests/core/removeb.js
   - jstests/core/removec.js
diff --git a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough.yml
index bb2d81ad640..61cd190091f 100644
--- a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough.yml
@@ -313,7 +313,6 @@ selector:
   - jstests/core/killop_drop_collection.js
   - jstests/core/loadserverscripts.js
   - jstests/core/mr_killop.js
-  - jstests/core/queryoptimizer3.js
   - jstests/core/remove9.js
   - jstests/core/removeb.js
   - jstests/core/removec.js
diff --git a/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml b/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml
index c875e22fceb..1904d857277 100644
--- a/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml
+++ b/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml
@@ -338,7 +338,6 @@ selector:
   - jstests/core/killop_drop_collection.js
   - jstests/core/loadserverscripts.js
   - jstests/core/mr_killop.js
-  - jstests/core/queryoptimizer3.js
   - jstests/core/remove9.js
   - jstests/core/removeb.js
   - jstests/core/removec.js
diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml
index afd62b78a05..6d8f7474735 100644
--- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml
@@ -244,7 +244,6 @@ selector:
   - jstests/core/killop_drop_collection.js
   - jstests/core/loadserverscripts.js
   - jstests/core/mr_killop.js
-  - jstests/core/queryoptimizer3.js
   - jstests/core/remove9.js
   - jstests/core/removeb.js
   - jstests/core/removec.js
diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml
index 3ab6286262f..866869f99e0 100644
--- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml
@@ -238,7 +238,6 @@ selector:
   - jstests/core/killop_drop_collection.js
   - jstests/core/loadserverscripts.js
   - jstests/core/mr_killop.js
-  - jstests/core/queryoptimizer3.js
   - jstests/core/remove9.js
   - jstests/core/removeb.js
   - jstests/core/removec.js
diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_stepdown_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_stepdown_jscore_passthrough.yml
index 0d2e114ab28..be2ee02bdcd 100644
--- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_stepdown_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_stepdown_jscore_passthrough.yml
@@ -239,7 +239,6 @@ selector:
   - jstests/core/killop_drop_collection.js
   - jstests/core/loadserverscripts.js
   - jstests/core/mr_killop.js
-  - jstests/core/queryoptimizer3.js
   - jstests/core/remove9.js
   - jstests/core/removeb.js
   - jstests/core/removec.js
diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml
index fc2777ff204..7b08bb1ca5c 100644
--- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml
@@ -236,7 +236,6 @@ selector:
   - jstests/core/killop_drop_collection.js
   - jstests/core/loadserverscripts.js
   - jstests/core/mr_killop.js
-  - jstests/core/queryoptimizer3.js
   - jstests/core/remove9.js
   - jstests/core/removeb.js
   - jstests/core/removec.js
diff --git a/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml
index 275065eae23..6ae6ba36d3a 100644
--- a/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml
@@ -64,8 +64,6 @@ selector:
   - jstests/core/mr_correctness.js
   - jstests/core/mr_fail_invalid_js.js
   - jstests/core/mr_scope.js
-  # TODO: SERVER-30052
-  - jstests/core/queryoptimizer3.js
   # Tests that need triaging & remediation | blacklist decision
   # Comments list possible problem point under review.
   - jstests/core/stages_delete.js # Uses stageDebug command for deletes.
diff --git a/buildscripts/resmokeconfig/suites/sharded_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_multi_stmt_txn_jscore_passthrough.yml
index c8156fea5a3..2e35cee3109 100644
--- a/buildscripts/resmokeconfig/suites/sharded_multi_stmt_txn_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/sharded_multi_stmt_txn_jscore_passthrough.yml
@@ -266,7 +266,6 @@ selector:
   - jstests/core/killop_drop_collection.js
   - jstests/core/loadserverscripts.js
   - jstests/core/mr_killop.js
-  - jstests/core/queryoptimizer3.js
   - jstests/core/remove9.js
   - jstests/core/removeb.js
   - jstests/core/removec.js
diff --git a/jstests/concurrency/fsm_workloads/yield_with_drop.js b/jstests/concurrency/fsm_workloads/yield_with_drop.js
new file mode 100644
index 00000000000..f06f26beec8
--- /dev/null
+++ b/jstests/concurrency/fsm_workloads/yield_with_drop.js
@@ -0,0 +1,146 @@
+"use strict";
+
+/**
+ * Executes query operations that can yield while the source collection is dropped and recreated.
+ */
+
+var $config = (function() {
+    const data = {
+        kAllowedErrors: [
+            ErrorCodes.ConflictingOperationInProgress,
+            ErrorCodes.CursorNotFound,
+            ErrorCodes.DuplicateKey,
+            ErrorCodes.OperationFailed,
+            ErrorCodes.QueryPlanKilled,
+        ],
+        nDocs: 200,
+        genUpdateDoc: function genUpdateDoc() {
+            const newVal = Random.randInt(this.nDocs);
+            return {$set: {a: newVal}};
+        },
+        // TODO SERVER-44673: Replace this function with calls to
+        // commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors).
+        assertWriteWorkedOrFailedWithExpectedCode:
+            function assertWriteWorkedOrFailedWithExpectedCode(cmdRes) {
+                if (cmdRes.ok) {
+                    if (cmdRes.hasOwnProperty("writeErrors") && cmdRes.writeErrors.length > 0) {
+                        assertAlways(this.kAllowedErrors.includes(cmdRes.writeErrors[0].code),
+                                     cmdRes);
+                    }
+
+                    return;
+                }
+
+                assertAlways.commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors);
+            },
+        create: function create(db, collName) {
+            for (let i = 0; i < this.nDocs; i++) {
+                const cmdRes = db.runCommand({
+                    update: collName,
+                    updates: [{
+                        q: {_id: i},
+                        u: {$set: {a: i, b: this.nDocs - i, c: i, d: this.nDocs - i, e: "foo"}},
+                        upsert: true
+                    }]
+                });
+                this.assertWriteWorkedOrFailedWithExpectedCode(cmdRes);
+            }
+            assertAlways.commandWorkedOrFailedWithCode(db[collName].createIndex({a: 1}),
+                                                       this.kAllowedErrors);
+        }
+    };
+
+    var states = {
+        query: function query(db, collName) {
+            let cmdRes = db.runCommand(
+                {find: collName, filter: {c: {$lt: this.nDocs}}, batchSize: this.nDocs});
+            assertAlways.commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors);
+
+            if (cmdRes.hasOwnProperty("cursor") && cmdRes.cursor.id > 0) {
+                cmdRes = db.runCommand({getMore: cmdRes.cursor.id, collection: collName});
+                assertAlways.commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors);
+            }
+        },
+
+        update: function update(db, collName) {
+            const cmdRes = db.runCommand({
+                update: collName,
+                updates: [{q: {_id: Random.randInt(this.nDocs)}, u: this.genUpdateDoc()}]
+            });
+
+            this.assertWriteWorkedOrFailedWithExpectedCode(cmdRes);
+        },
+
+        remove: function remove(db, collName) {
+            const cmdRes = db.runCommand(
+                {delete: collName, deletes: [{q: {_id: Random.randInt(this.nDocs)}, limit: 1}]});
+
+            this.assertWriteWorkedOrFailedWithExpectedCode(cmdRes);
+        },
+
+        count: function count(db, collName) {
+            const cmdRes = db.runCommand({count: collName, query: {a: {$lt: this.nDocs}}});
+            assertAlways.commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors);
+        },
+
+        distinct: function distinct(db, collName) {
+            const cmdRes =
+                db.runCommand({distinct: collName, key: "a", query: {a: {$lt: this.nDocs}}});
+            assertAlways.commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors);
+        },
+
+        recreateColl: function recreateColl(db, collName) {
+            const cmdRes = db[collName].drop();
+            this.create(db, collName);
+        },
+    };
+
+    const kAllStatesEqual =
+        {update: 0.18, remove: 0.18, query: 0.18, count: 0.18, distinct: 0.18, recreateColl: 0.1};
+    const transitions = {
+        update: kAllStatesEqual,
+        remove: kAllStatesEqual,
+        query: kAllStatesEqual,
+        count: kAllStatesEqual,
+        distinct: kAllStatesEqual,
+        recreateColl: kAllStatesEqual,
+    };
+
+    /**
+     * Lowers yielding parameters to increase frequency and sets up collection.
+     */
+    function setup(db, collName, cluster) {
+        // Lower the following parameters to force even more yields.
+        cluster.executeOnMongodNodes(function lowerYieldParams(db) {
+            assertAlways.commandWorked(
+                db.adminCommand({setParameter: 1, internalQueryExecYieldIterations: 5}));
+            assertAlways.commandWorked(
+                db.adminCommand({setParameter: 1, internalQueryExecYieldPeriodMS: 1}));
+        });
+
+        data.create(db, collName);
+    }
+
+    /**
+     * Disables failpoints.
+     */
+    function teardown(db, collName, cluster) {
+        cluster.executeOnMongodNodes(function resetYieldParams(db) {
+            assertAlways.commandWorked(
+                db.adminCommand({setParameter: 1, internalQueryExecYieldIterations: 128}));
+            assertAlways.commandWorked(
+                db.adminCommand({setParameter: 1, internalQueryExecYieldPeriodMS: 10}));
+        });
+    }
+
+    return {
+        threadCount: 10,
+        iterations: 100,
+        startState: 'update',
+        states: states,
+        transitions: transitions,
+        setup: setup,
+        teardown: teardown,
+        data: data
+    };
+})();
diff --git a/jstests/core/queryoptimizer3.js b/jstests/core/queryoptimizer3.js
deleted file mode 100644
index 9fa0585991a..00000000000
--- a/jstests/core/queryoptimizer3.js
+++ /dev/null
@@ -1,69 +0,0 @@
-// Validates cases where index scans are aborted due to the collection being dropped (SERVER-4400)
-//
-// Drop and other sharding commands can conflict with LockBusy errors in a sharding passthrough
-// suite. This is because drop against a mongos takes distlocks, whereas drop against a mongod does
-// not. Due to the huge number of parallel drops in this test, the other thead is very likely to be
-// starved frequently.
-// Note: this tag can be safely removed once PM-697 is complete and replaces distlocks with a
-// LockManager that has a fairness policy, which distlocks lack.
-// @tags: [
-//   assumes_against_mongod_not_mongos,
-//   requires_non_retryable_writes,
-//   uses_multiple_connections,
-// ]
-
-(function() {
-'use strict';
-
-var coll = db.jstests_queryoptimizer3;
-
-var shellWaitHandle = startParallelShell(function() {
-    for (var i = 0; i < 400; ++i) {
-        sleep(50);
-        try {
-            db.jstests_queryoptimizer3.drop();
-        } catch (e) {
-            if (e.code === ErrorCodes.BackgroundOperationInProgressForNamespace) {
-                print("Background operation temporarily in progress while attempting to drop " +
-                      "collection.");
-                continue;
-            }
-            throw e;
-        }
-    }
-});
-
-for (var i = 0; i < 100; ++i) {
-    coll.drop();
-    assert.commandWorked(coll.ensureIndex({a: 1}));
-    assert.commandWorked(coll.ensureIndex({b: 1}));
-
-    var bulk = coll.initializeUnorderedBulkOp();
-    for (var j = 0; j < 100; ++j) {
-        bulk.insert({a: j, b: j});
-    }
-    assert.commandWorked(bulk.execute());
-
-    try {
-        var m = i % 5;
-        if (m == 0) {
-            coll.count({a: {$gte: 0}, b: {$gte: 0}});
-        } else if (m == 1) {
-            coll.find({a: {$gte: 0}, b: {$gte: 0}}).itcount();
-        } else if (m == 2) {
-            coll.remove({a: {$gte: 0}, b: {$gte: 0}});
-        } else if (m == 3) {
-            coll.update({a: {$gte: 0}, b: {$gte: 0}}, {});
-        } else if (m == 4) {
-            coll.distinct('x', {a: {$gte: 0}, b: {$gte: 0}});
-        }
-    } catch (e) {
-        print("Op killed during yield: " + e.message);
-    }
-}
-
-shellWaitHandle();
-
-// Ensure that the server is still responding
-assert.commandWorked(db.runCommand({isMaster: 1}));
-})();
diff --git a/jstests/core/queryoptimizer6.js b/jstests/core/queryoptimizer6.js
deleted file mode 100644
index 8e00772aa4e..00000000000
--- a/jstests/core/queryoptimizer6.js
+++ /dev/null
@@ -1,16 +0,0 @@
-// Test that $ne constraints are accounted for in QueryPattern. SERVER-4665
-
-t = db.jstests_queryoptimizer6;
-t.drop();
-
-t.save({a: 1});
-
-// There is a bug in the 2.4.x indexing where the first query below returns 0 results with this
-// index, but 1 result without it.
-//
-// t.ensureIndex( {b:1}, {sparse:true} );
-
-// The sparse index will be used, and recorded for this query pattern.
-assert.eq(1, t.find({a: 1, b: {$ne: 1}}).itcount());
-// The query pattern should be different, and the sparse index should not be used.
-assert.eq(1, t.find({a: 1}).itcount());