From d86e8a5bf40f1c6edbab37bef38cd0f1d5dc062d Mon Sep 17 00:00:00 2001 From: James Wahlin Date: Wed, 20 Nov 2019 20:31:43 +0000 Subject: SERVER-32567 Replace queryoptimizer3.js with yield_with_drop.js FSM test --- ...currency_sharded_kill_primary_with_balancer.yml | 1 + ...ncy_sharded_terminate_primary_with_balancer.yml | 1 + .../suites/concurrency_sharded_with_stepdowns.yml | 1 + ...urrency_sharded_with_stepdowns_and_balancer.yml | 1 + ...ead_write_multi_stmt_txn_jscore_passthrough.yml | 1 - ...lti_shard_multi_stmt_txn_jscore_passthrough.yml | 1 - ...ti_stmt_txn_kill_primary_jscore_passthrough.yml | 1 - ...tmt_txn_stepdown_primary_jscore_passthrough.yml | 1 - ..._stmt_txn_jscore_passthrough_with_migration.yml | 1 - ...lica_sets_multi_stmt_txn_jscore_passthrough.yml | 1 - ...ti_stmt_txn_kill_primary_jscore_passthrough.yml | 1 - ..._multi_stmt_txn_stepdown_jscore_passthrough.yml | 1 - ...mt_txn_terminate_primary_jscore_passthrough.yml | 1 - ...rded_causally_consistent_jscore_passthrough.yml | 2 - .../sharded_multi_stmt_txn_jscore_passthrough.yml | 1 - .../concurrency/fsm_workloads/yield_with_drop.js | 146 +++++++++++++++++++++ jstests/core/queryoptimizer3.js | 69 ---------- jstests/core/queryoptimizer6.js | 16 --- 18 files changed, 150 insertions(+), 97 deletions(-) create mode 100644 jstests/concurrency/fsm_workloads/yield_with_drop.js delete mode 100644 jstests/core/queryoptimizer3.js delete mode 100644 jstests/core/queryoptimizer6.js diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml index 21d73d0933e..945a8bd954b 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml @@ -141,6 +141,7 @@ selector: - jstests/concurrency/fsm_workloads/yield_sort.js - jstests/concurrency/fsm_workloads/yield_sort_merge.js - jstests/concurrency/fsm_workloads/yield_text.js + - jstests/concurrency/fsm_workloads/yield_with_drop.js # Uses non retryable writes. - jstests/concurrency/fsm_workloads/remove_and_bulk_insert.js diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml index a8df6a100d7..80f4b700ae6 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml @@ -141,6 +141,7 @@ selector: - jstests/concurrency/fsm_workloads/yield_sort.js - jstests/concurrency/fsm_workloads/yield_sort_merge.js - jstests/concurrency/fsm_workloads/yield_text.js + - jstests/concurrency/fsm_workloads/yield_with_drop.js # Uses non retryable writes. - jstests/concurrency/fsm_workloads/remove_and_bulk_insert.js diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml index c83cf181b7f..6b2b73b0d41 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml @@ -129,6 +129,7 @@ selector: - jstests/concurrency/fsm_workloads/yield_sort.js - jstests/concurrency/fsm_workloads/yield_sort_merge.js - jstests/concurrency/fsm_workloads/yield_text.js + - jstests/concurrency/fsm_workloads/yield_with_drop.js # Uses non retryable writes. - jstests/concurrency/fsm_workloads/remove_and_bulk_insert.js diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml index 916c3911722..e7aa2ae5cc3 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml @@ -134,6 +134,7 @@ selector: - jstests/concurrency/fsm_workloads/yield_sort.js - jstests/concurrency/fsm_workloads/yield_sort_merge.js - jstests/concurrency/fsm_workloads/yield_text.js + - jstests/concurrency/fsm_workloads/yield_with_drop.js # Uses non retryable writes. - jstests/concurrency/fsm_workloads/remove_and_bulk_insert.js diff --git a/buildscripts/resmokeconfig/suites/multi_shard_local_read_write_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/multi_shard_local_read_write_multi_stmt_txn_jscore_passthrough.yml index d77680f2395..d8b75981068 100644 --- a/buildscripts/resmokeconfig/suites/multi_shard_local_read_write_multi_stmt_txn_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/multi_shard_local_read_write_multi_stmt_txn_jscore_passthrough.yml @@ -294,7 +294,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_jscore_passthrough.yml index 18f468c702d..626e41756ac 100644 --- a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_jscore_passthrough.yml @@ -321,7 +321,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough.yml index 4ca32a5ce60..0841a15703a 100644 --- a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough.yml @@ -306,7 +306,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough.yml index bb2d81ad640..61cd190091f 100644 --- a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough.yml @@ -313,7 +313,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml b/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml index c875e22fceb..1904d857277 100644 --- a/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml +++ b/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml @@ -338,7 +338,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml index afd62b78a05..6d8f7474735 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml @@ -244,7 +244,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml index 3ab6286262f..866869f99e0 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml @@ -238,7 +238,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_stepdown_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_stepdown_jscore_passthrough.yml index 0d2e114ab28..be2ee02bdcd 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_stepdown_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_stepdown_jscore_passthrough.yml @@ -239,7 +239,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml index fc2777ff204..7b08bb1ca5c 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml @@ -236,7 +236,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml index 275065eae23..6ae6ba36d3a 100644 --- a/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml @@ -64,8 +64,6 @@ selector: - jstests/core/mr_correctness.js - jstests/core/mr_fail_invalid_js.js - jstests/core/mr_scope.js - # TODO: SERVER-30052 - - jstests/core/queryoptimizer3.js # Tests that need triaging & remediation | blacklist decision # Comments list possible problem point under review. - jstests/core/stages_delete.js # Uses stageDebug command for deletes. diff --git a/buildscripts/resmokeconfig/suites/sharded_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_multi_stmt_txn_jscore_passthrough.yml index c8156fea5a3..2e35cee3109 100644 --- a/buildscripts/resmokeconfig/suites/sharded_multi_stmt_txn_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharded_multi_stmt_txn_jscore_passthrough.yml @@ -266,7 +266,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/jstests/concurrency/fsm_workloads/yield_with_drop.js b/jstests/concurrency/fsm_workloads/yield_with_drop.js new file mode 100644 index 00000000000..f06f26beec8 --- /dev/null +++ b/jstests/concurrency/fsm_workloads/yield_with_drop.js @@ -0,0 +1,146 @@ +"use strict"; + +/** + * Executes query operations that can yield while the source collection is dropped and recreated. + */ + +var $config = (function() { + const data = { + kAllowedErrors: [ + ErrorCodes.ConflictingOperationInProgress, + ErrorCodes.CursorNotFound, + ErrorCodes.DuplicateKey, + ErrorCodes.OperationFailed, + ErrorCodes.QueryPlanKilled, + ], + nDocs: 200, + genUpdateDoc: function genUpdateDoc() { + const newVal = Random.randInt(this.nDocs); + return {$set: {a: newVal}}; + }, + // TODO SERVER-44673: Replace this function with calls to + // commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors). + assertWriteWorkedOrFailedWithExpectedCode: + function assertWriteWorkedOrFailedWithExpectedCode(cmdRes) { + if (cmdRes.ok) { + if (cmdRes.hasOwnProperty("writeErrors") && cmdRes.writeErrors.length > 0) { + assertAlways(this.kAllowedErrors.includes(cmdRes.writeErrors[0].code), + cmdRes); + } + + return; + } + + assertAlways.commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors); + }, + create: function create(db, collName) { + for (let i = 0; i < this.nDocs; i++) { + const cmdRes = db.runCommand({ + update: collName, + updates: [{ + q: {_id: i}, + u: {$set: {a: i, b: this.nDocs - i, c: i, d: this.nDocs - i, e: "foo"}}, + upsert: true + }] + }); + this.assertWriteWorkedOrFailedWithExpectedCode(cmdRes); + } + assertAlways.commandWorkedOrFailedWithCode(db[collName].createIndex({a: 1}), + this.kAllowedErrors); + } + }; + + var states = { + query: function query(db, collName) { + let cmdRes = db.runCommand( + {find: collName, filter: {c: {$lt: this.nDocs}}, batchSize: this.nDocs}); + assertAlways.commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors); + + if (cmdRes.hasOwnProperty("cursor") && cmdRes.cursor.id > 0) { + cmdRes = db.runCommand({getMore: cmdRes.cursor.id, collection: collName}); + assertAlways.commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors); + } + }, + + update: function update(db, collName) { + const cmdRes = db.runCommand({ + update: collName, + updates: [{q: {_id: Random.randInt(this.nDocs)}, u: this.genUpdateDoc()}] + }); + + this.assertWriteWorkedOrFailedWithExpectedCode(cmdRes); + }, + + remove: function remove(db, collName) { + const cmdRes = db.runCommand( + {delete: collName, deletes: [{q: {_id: Random.randInt(this.nDocs)}, limit: 1}]}); + + this.assertWriteWorkedOrFailedWithExpectedCode(cmdRes); + }, + + count: function count(db, collName) { + const cmdRes = db.runCommand({count: collName, query: {a: {$lt: this.nDocs}}}); + assertAlways.commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors); + }, + + distinct: function distinct(db, collName) { + const cmdRes = + db.runCommand({distinct: collName, key: "a", query: {a: {$lt: this.nDocs}}}); + assertAlways.commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors); + }, + + recreateColl: function recreateColl(db, collName) { + const cmdRes = db[collName].drop(); + this.create(db, collName); + }, + }; + + const kAllStatesEqual = + {update: 0.18, remove: 0.18, query: 0.18, count: 0.18, distinct: 0.18, recreateColl: 0.1}; + const transitions = { + update: kAllStatesEqual, + remove: kAllStatesEqual, + query: kAllStatesEqual, + count: kAllStatesEqual, + distinct: kAllStatesEqual, + recreateColl: kAllStatesEqual, + }; + + /** + * Lowers yielding parameters to increase frequency and sets up collection. + */ + function setup(db, collName, cluster) { + // Lower the following parameters to force even more yields. + cluster.executeOnMongodNodes(function lowerYieldParams(db) { + assertAlways.commandWorked( + db.adminCommand({setParameter: 1, internalQueryExecYieldIterations: 5})); + assertAlways.commandWorked( + db.adminCommand({setParameter: 1, internalQueryExecYieldPeriodMS: 1})); + }); + + data.create(db, collName); + } + + /** + * Disables failpoints. + */ + function teardown(db, collName, cluster) { + cluster.executeOnMongodNodes(function resetYieldParams(db) { + assertAlways.commandWorked( + db.adminCommand({setParameter: 1, internalQueryExecYieldIterations: 128})); + assertAlways.commandWorked( + db.adminCommand({setParameter: 1, internalQueryExecYieldPeriodMS: 10})); + }); + } + + return { + threadCount: 10, + iterations: 100, + startState: 'update', + states: states, + transitions: transitions, + setup: setup, + teardown: teardown, + data: data + }; +})(); diff --git a/jstests/core/queryoptimizer3.js b/jstests/core/queryoptimizer3.js deleted file mode 100644 index 9fa0585991a..00000000000 --- a/jstests/core/queryoptimizer3.js +++ /dev/null @@ -1,69 +0,0 @@ -// Validates cases where index scans are aborted due to the collection being dropped (SERVER-4400) -// -// Drop and other sharding commands can conflict with LockBusy errors in a sharding passthrough -// suite. This is because drop against a mongos takes distlocks, whereas drop against a mongod does -// not. Due to the huge number of parallel drops in this test, the other thead is very likely to be -// starved frequently. -// Note: this tag can be safely removed once PM-697 is complete and replaces distlocks with a -// LockManager that has a fairness policy, which distlocks lack. -// @tags: [ -// assumes_against_mongod_not_mongos, -// requires_non_retryable_writes, -// uses_multiple_connections, -// ] - -(function() { -'use strict'; - -var coll = db.jstests_queryoptimizer3; - -var shellWaitHandle = startParallelShell(function() { - for (var i = 0; i < 400; ++i) { - sleep(50); - try { - db.jstests_queryoptimizer3.drop(); - } catch (e) { - if (e.code === ErrorCodes.BackgroundOperationInProgressForNamespace) { - print("Background operation temporarily in progress while attempting to drop " + - "collection."); - continue; - } - throw e; - } - } -}); - -for (var i = 0; i < 100; ++i) { - coll.drop(); - assert.commandWorked(coll.ensureIndex({a: 1})); - assert.commandWorked(coll.ensureIndex({b: 1})); - - var bulk = coll.initializeUnorderedBulkOp(); - for (var j = 0; j < 100; ++j) { - bulk.insert({a: j, b: j}); - } - assert.commandWorked(bulk.execute()); - - try { - var m = i % 5; - if (m == 0) { - coll.count({a: {$gte: 0}, b: {$gte: 0}}); - } else if (m == 1) { - coll.find({a: {$gte: 0}, b: {$gte: 0}}).itcount(); - } else if (m == 2) { - coll.remove({a: {$gte: 0}, b: {$gte: 0}}); - } else if (m == 3) { - coll.update({a: {$gte: 0}, b: {$gte: 0}}, {}); - } else if (m == 4) { - coll.distinct('x', {a: {$gte: 0}, b: {$gte: 0}}); - } - } catch (e) { - print("Op killed during yield: " + e.message); - } -} - -shellWaitHandle(); - -// Ensure that the server is still responding -assert.commandWorked(db.runCommand({isMaster: 1})); -})(); diff --git a/jstests/core/queryoptimizer6.js b/jstests/core/queryoptimizer6.js deleted file mode 100644 index 8e00772aa4e..00000000000 --- a/jstests/core/queryoptimizer6.js +++ /dev/null @@ -1,16 +0,0 @@ -// Test that $ne constraints are accounted for in QueryPattern. SERVER-4665 - -t = db.jstests_queryoptimizer6; -t.drop(); - -t.save({a: 1}); - -// There is a bug in the 2.4.x indexing where the first query below returns 0 results with this -// index, but 1 result without it. -// -// t.ensureIndex( {b:1}, {sparse:true} ); - -// The sparse index will be used, and recorded for this query pattern. -assert.eq(1, t.find({a: 1, b: {$ne: 1}}).itcount()); -// The query pattern should be different, and the sparse index should not be used. -assert.eq(1, t.find({a: 1}).itcount()); -- cgit v1.2.1