diff options
author | James Wahlin <james.wahlin@mongodb.com> | 2019-12-10 13:52:57 +0000 |
---|---|---|
committer | evergreen <evergreen@mongodb.com> | 2019-12-10 13:52:57 +0000 |
commit | b5a58d9c335857ab1d414f4f48574e514f640b59 (patch) | |
tree | c33b09c289f13e1d111cb7c79147fc033ad50757 | |
parent | 2a455afd9775631c1ddb041cfdababc27b833810 (diff) | |
download | mongo-b5a58d9c335857ab1d414f4f48574e514f640b59.tar.gz |
SERVER-32567 Replace queryoptimizer3.js with yield_with_drop.js FSM test
(cherry picked from commit d86e8a5bf40f1c6edbab37bef38cd0f1d5dc062d)
18 files changed, 150 insertions, 97 deletions
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml index 0fe1e5084fe..a151e01e6fb 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml @@ -139,6 +139,7 @@ selector: - jstests/concurrency/fsm_workloads/yield_sort.js - jstests/concurrency/fsm_workloads/yield_sort_merge.js - jstests/concurrency/fsm_workloads/yield_text.js + - jstests/concurrency/fsm_workloads/yield_with_drop.js # Uses non retryable writes. - jstests/concurrency/fsm_workloads/remove_and_bulk_insert.js diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml index 68b55ab429c..4442d58c5ca 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml @@ -139,6 +139,7 @@ selector: - jstests/concurrency/fsm_workloads/yield_sort.js - jstests/concurrency/fsm_workloads/yield_sort_merge.js - jstests/concurrency/fsm_workloads/yield_text.js + - jstests/concurrency/fsm_workloads/yield_with_drop.js # Uses non retryable writes. - jstests/concurrency/fsm_workloads/remove_and_bulk_insert.js diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml index cc9b0d960f2..6aeb4648258 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml @@ -135,6 +135,7 @@ selector: - jstests/concurrency/fsm_workloads/yield_sort.js - jstests/concurrency/fsm_workloads/yield_sort_merge.js - jstests/concurrency/fsm_workloads/yield_text.js + - jstests/concurrency/fsm_workloads/yield_with_drop.js # Uses non retryable writes. - jstests/concurrency/fsm_workloads/remove_and_bulk_insert.js diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml index 5e9309df475..fdf352fc9ad 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml @@ -140,6 +140,7 @@ selector: - jstests/concurrency/fsm_workloads/yield_sort.js - jstests/concurrency/fsm_workloads/yield_sort_merge.js - jstests/concurrency/fsm_workloads/yield_text.js + - jstests/concurrency/fsm_workloads/yield_with_drop.js # Uses non retryable writes. - jstests/concurrency/fsm_workloads/remove_and_bulk_insert.js diff --git a/buildscripts/resmokeconfig/suites/multi_shard_local_read_write_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/multi_shard_local_read_write_multi_stmt_txn_jscore_passthrough.yml index b038017cd18..d98cd662280 100644 --- a/buildscripts/resmokeconfig/suites/multi_shard_local_read_write_multi_stmt_txn_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/multi_shard_local_read_write_multi_stmt_txn_jscore_passthrough.yml @@ -303,7 +303,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_jscore_passthrough.yml index 1cac4029e20..0cb1e20095a 100644 --- a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_jscore_passthrough.yml @@ -330,7 +330,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough.yml index 1d91d755423..1533aadde79 100644 --- a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough.yml @@ -315,7 +315,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough.yml index 241af05766d..068cfa1d14a 100644 --- a/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough.yml @@ -324,7 +324,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml b/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml index c9d1bc39459..40630624c7c 100644 --- a/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml +++ b/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml @@ -347,7 +347,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml index 817eceb11d1..30e40e9bab1 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml @@ -246,7 +246,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml index c09fa51e041..f3fa670b40c 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml @@ -243,7 +243,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_stepdown_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_stepdown_jscore_passthrough.yml index 7336d0cd656..da24315f156 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_stepdown_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_stepdown_jscore_passthrough.yml @@ -243,7 +243,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml index e2c550c1f52..60a615830fd 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml @@ -240,7 +240,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml index 369301b0e9e..90e4afdeb53 100644 --- a/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml @@ -69,8 +69,6 @@ selector: - jstests/core/mr1.js - jstests/core/mr3.js - jstests/core/mr4.js - # TODO: SERVER-30052 - - jstests/core/queryoptimizer3.js # TODO: SERVER-27269: mongos can't establish cursor if view has $collStats and views another view. - jstests/core/views/views_coll_stats.js # Tests that need triaging & remediation | blacklist decision diff --git a/buildscripts/resmokeconfig/suites/sharded_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_multi_stmt_txn_jscore_passthrough.yml index e225c598f99..487b96f7aff 100644 --- a/buildscripts/resmokeconfig/suites/sharded_multi_stmt_txn_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharded_multi_stmt_txn_jscore_passthrough.yml @@ -276,7 +276,6 @@ selector: - jstests/core/killop_drop_collection.js - jstests/core/loadserverscripts.js - jstests/core/mr_killop.js - - jstests/core/queryoptimizer3.js - jstests/core/remove9.js - jstests/core/removeb.js - jstests/core/removec.js diff --git a/jstests/concurrency/fsm_workloads/yield_with_drop.js b/jstests/concurrency/fsm_workloads/yield_with_drop.js new file mode 100644 index 00000000000..f06f26beec8 --- /dev/null +++ b/jstests/concurrency/fsm_workloads/yield_with_drop.js @@ -0,0 +1,146 @@ +"use strict"; + +/** + * Executes query operations that can yield while the source collection is dropped and recreated. + */ + +var $config = (function() { + const data = { + kAllowedErrors: [ + ErrorCodes.ConflictingOperationInProgress, + ErrorCodes.CursorNotFound, + ErrorCodes.DuplicateKey, + ErrorCodes.OperationFailed, + ErrorCodes.QueryPlanKilled, + ], + nDocs: 200, + genUpdateDoc: function genUpdateDoc() { + const newVal = Random.randInt(this.nDocs); + return {$set: {a: newVal}}; + }, + // TODO SERVER-44673: Replace this function with calls to + // commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors). + assertWriteWorkedOrFailedWithExpectedCode: + function assertWriteWorkedOrFailedWithExpectedCode(cmdRes) { + if (cmdRes.ok) { + if (cmdRes.hasOwnProperty("writeErrors") && cmdRes.writeErrors.length > 0) { + assertAlways(this.kAllowedErrors.includes(cmdRes.writeErrors[0].code), + cmdRes); + } + + return; + } + + assertAlways.commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors); + }, + create: function create(db, collName) { + for (let i = 0; i < this.nDocs; i++) { + const cmdRes = db.runCommand({ + update: collName, + updates: [{ + q: {_id: i}, + u: {$set: {a: i, b: this.nDocs - i, c: i, d: this.nDocs - i, e: "foo"}}, + upsert: true + }] + }); + this.assertWriteWorkedOrFailedWithExpectedCode(cmdRes); + } + assertAlways.commandWorkedOrFailedWithCode(db[collName].createIndex({a: 1}), + this.kAllowedErrors); + } + }; + + var states = { + query: function query(db, collName) { + let cmdRes = db.runCommand( + {find: collName, filter: {c: {$lt: this.nDocs}}, batchSize: this.nDocs}); + assertAlways.commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors); + + if (cmdRes.hasOwnProperty("cursor") && cmdRes.cursor.id > 0) { + cmdRes = db.runCommand({getMore: cmdRes.cursor.id, collection: collName}); + assertAlways.commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors); + } + }, + + update: function update(db, collName) { + const cmdRes = db.runCommand({ + update: collName, + updates: [{q: {_id: Random.randInt(this.nDocs)}, u: this.genUpdateDoc()}] + }); + + this.assertWriteWorkedOrFailedWithExpectedCode(cmdRes); + }, + + remove: function remove(db, collName) { + const cmdRes = db.runCommand( + {delete: collName, deletes: [{q: {_id: Random.randInt(this.nDocs)}, limit: 1}]}); + + this.assertWriteWorkedOrFailedWithExpectedCode(cmdRes); + }, + + count: function count(db, collName) { + const cmdRes = db.runCommand({count: collName, query: {a: {$lt: this.nDocs}}}); + assertAlways.commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors); + }, + + distinct: function distinct(db, collName) { + const cmdRes = + db.runCommand({distinct: collName, key: "a", query: {a: {$lt: this.nDocs}}}); + assertAlways.commandWorkedOrFailedWithCode(cmdRes, this.kAllowedErrors); + }, + + recreateColl: function recreateColl(db, collName) { + const cmdRes = db[collName].drop(); + this.create(db, collName); + }, + }; + + const kAllStatesEqual = + {update: 0.18, remove: 0.18, query: 0.18, count: 0.18, distinct: 0.18, recreateColl: 0.1}; + const transitions = { + update: kAllStatesEqual, + remove: kAllStatesEqual, + query: kAllStatesEqual, + count: kAllStatesEqual, + distinct: kAllStatesEqual, + recreateColl: kAllStatesEqual, + }; + + /** + * Lowers yielding parameters to increase frequency and sets up collection. + */ + function setup(db, collName, cluster) { + // Lower the following parameters to force even more yields. + cluster.executeOnMongodNodes(function lowerYieldParams(db) { + assertAlways.commandWorked( + db.adminCommand({setParameter: 1, internalQueryExecYieldIterations: 5})); + assertAlways.commandWorked( + db.adminCommand({setParameter: 1, internalQueryExecYieldPeriodMS: 1})); + }); + + data.create(db, collName); + } + + /** + * Disables failpoints. + */ + function teardown(db, collName, cluster) { + cluster.executeOnMongodNodes(function resetYieldParams(db) { + assertAlways.commandWorked( + db.adminCommand({setParameter: 1, internalQueryExecYieldIterations: 128})); + assertAlways.commandWorked( + db.adminCommand({setParameter: 1, internalQueryExecYieldPeriodMS: 10})); + }); + } + + return { + threadCount: 10, + iterations: 100, + startState: 'update', + states: states, + transitions: transitions, + setup: setup, + teardown: teardown, + data: data + }; +})(); diff --git a/jstests/core/queryoptimizer3.js b/jstests/core/queryoptimizer3.js deleted file mode 100644 index 9fa0585991a..00000000000 --- a/jstests/core/queryoptimizer3.js +++ /dev/null @@ -1,69 +0,0 @@ -// Validates cases where index scans are aborted due to the collection being dropped (SERVER-4400) -// -// Drop and other sharding commands can conflict with LockBusy errors in a sharding passthrough -// suite. This is because drop against a mongos takes distlocks, whereas drop against a mongod does -// not. Due to the huge number of parallel drops in this test, the other thead is very likely to be -// starved frequently. -// Note: this tag can be safely removed once PM-697 is complete and replaces distlocks with a -// LockManager that has a fairness policy, which distlocks lack. -// @tags: [ -// assumes_against_mongod_not_mongos, -// requires_non_retryable_writes, -// uses_multiple_connections, -// ] - -(function() { -'use strict'; - -var coll = db.jstests_queryoptimizer3; - -var shellWaitHandle = startParallelShell(function() { - for (var i = 0; i < 400; ++i) { - sleep(50); - try { - db.jstests_queryoptimizer3.drop(); - } catch (e) { - if (e.code === ErrorCodes.BackgroundOperationInProgressForNamespace) { - print("Background operation temporarily in progress while attempting to drop " + - "collection."); - continue; - } - throw e; - } - } -}); - -for (var i = 0; i < 100; ++i) { - coll.drop(); - assert.commandWorked(coll.ensureIndex({a: 1})); - assert.commandWorked(coll.ensureIndex({b: 1})); - - var bulk = coll.initializeUnorderedBulkOp(); - for (var j = 0; j < 100; ++j) { - bulk.insert({a: j, b: j}); - } - assert.commandWorked(bulk.execute()); - - try { - var m = i % 5; - if (m == 0) { - coll.count({a: {$gte: 0}, b: {$gte: 0}}); - } else if (m == 1) { - coll.find({a: {$gte: 0}, b: {$gte: 0}}).itcount(); - } else if (m == 2) { - coll.remove({a: {$gte: 0}, b: {$gte: 0}}); - } else if (m == 3) { - coll.update({a: {$gte: 0}, b: {$gte: 0}}, {}); - } else if (m == 4) { - coll.distinct('x', {a: {$gte: 0}, b: {$gte: 0}}); - } - } catch (e) { - print("Op killed during yield: " + e.message); - } -} - -shellWaitHandle(); - -// Ensure that the server is still responding -assert.commandWorked(db.runCommand({isMaster: 1})); -})(); diff --git a/jstests/core/queryoptimizer6.js b/jstests/core/queryoptimizer6.js deleted file mode 100644 index 8e00772aa4e..00000000000 --- a/jstests/core/queryoptimizer6.js +++ /dev/null @@ -1,16 +0,0 @@ -// Test that $ne constraints are accounted for in QueryPattern. SERVER-4665 - -t = db.jstests_queryoptimizer6; -t.drop(); - -t.save({a: 1}); - -// There is a bug in the 2.4.x indexing where the first query below returns 0 results with this -// index, but 1 result without it. -// -// t.ensureIndex( {b:1}, {sparse:true} ); - -// The sparse index will be used, and recorded for this query pattern. -assert.eq(1, t.find({a: 1, b: {$ne: 1}}).itcount()); -// The query pattern should be different, and the sparse index should not be used. -assert.eq(1, t.find({a: 1}).itcount()); |