diff options
46 files changed, 448 insertions, 218 deletions
diff --git a/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml index c73b0df611e..392c9f5dd5e 100644 --- a/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml @@ -4,9 +4,24 @@ selector: - jstests/aggregation/*.js - jstests/aggregation/bugs/*.js exclude_files: - - jstests/aggregation/bugs/server18198.js # Uses a mocked mongo client to test read preference. - - jstests/aggregation/mongos_slaveok.js # Majority read on secondary requires afterOpTime. - - jstests/aggregation/testSlave.js # Majority read on secondary requires afterOpTime. + # These test fail because afterOpTime is required to guarantee a secondary has advanced its + # majority-committed snapshot. + - jstests/aggregation/mongos_slaveok.js + - jstests/aggregation/testSlave.js + exclude_with_any_tags: + ## + # The next three tags correspond to the special errors thrown by the + # set_read_and_write_concerns.js override when it refuses to replace the readConcern or + # writeConcern of a particular command. Above each tag are the message(s) that cause the tag to + # be warranted. + ## + # "Cowardly refusing to override read concern of command: ..." + - assumes_read_concern_unchanged + # "Cowardly refusing to override write concern of command: ..." + - assumes_write_concern_unchanged + # "Cowardly refusing to run test with overridden write concern when it uses a command that can + # only perform w=1 writes: ..." + - requires_eval_command executor: js_test: @@ -14,12 +29,18 @@ executor: shell_options: global_vars: TestData: + defaultReadConcernLevel: majority enableMajorityReadConcern: '' - eval: "var testingReplication = true; load('jstests/libs/override_methods/set_majority_read_and_write_concerns.js');" + eval: >- + var testingReplication = true; + load('jstests/libs/override_methods/set_read_and_write_concerns.js'); readMode: commands hooks: - - class: ValidateCollections + # The CheckReplDBHash hook waits until all operations have replicated to and have been applied + # on the secondaries, so we run the ValidateCollections hook after it to ensure we're + # validating the entire contents of the collection. - class: CheckReplDBHash + - class: ValidateCollections fixture: class: ReplicaSetFixture mongod_options: diff --git a/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml index c706898c89e..407e083edc9 100644 --- a/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml @@ -3,70 +3,47 @@ selector: roots: - jstests/core/*.js exclude_files: - # Tests that won't work with an injected 'majority' readConcern - # and/or an injected 'majority' writeConcern. Where a function is - # listed the reason is we don't have a reliable solution to override - # the write concern for that function. - - jstests/core/batch_write_command*.js # these tests use various write concerns - - jstests/core/bench_test*.js # benchRun() used for writes - - jstests/core/capped_update.js # uses godinsert and can't run under replication. - - jstests/core/crud_api.js # has specific w:0 tests - - jstests/core/error2.js # db.eval() used - - jstests/core/eval0.js # db.eval() used - - jstests/core/eval1.js # db.eval() used - - jstests/core/eval3.js # db.eval() used - - jstests/core/eval4.js # db.eval() used - - jstests/core/eval5.js # db.eval() used - - jstests/core/eval6.js # db.eval() used - - jstests/core/eval7.js # db.eval() used - - jstests/core/eval9.js # db.eval() used - - jstests/core/evala.js # db.eval() used - - jstests/core/evalb.js # db.eval() used - - jstests/core/evalc.js # db.eval() used - - jstests/core/evald.js # db.eval() used - - jstests/core/evale.js # db.eval() used - - jstests/core/evalg.js # db.eval() used - - jstests/core/eval_mr.js # db.eval() used - - jstests/core/eval_nolock.js # db.eval() used - - jstests/core/geo_s2cursorlimitskip.js # drops system.profile collection and counts ops. - - jstests/core/js3.js # db.dbEval() used - - jstests/core/js7.js # db.eval() used - - jstests/core/js9.js # db.eval() used - - jstests/core/mr_merge.js # mr temp tables aren't replicated - - jstests/core/mr_merge2.js # mr temp tables aren't replicated - - jstests/core/mr_outreduce.js # mr temp tables aren't replicated - - jstests/core/mr_outreduce2.js # mr temp tables aren't replicated - - jstests/core/opcounters_active.js # off by n problem with opcounters - - jstests/core/opcounters_write_cmd.js # off by n problem with opcounters - - jstests/core/profile1.js # system.profile not replicated - - jstests/core/profile2.js # system.profile not replicated - - jstests/core/profile3.js # system.profile not replicated - - jstests/core/profile4.js # system.profile not replicated - - jstests/core/profile5.js # system.profile not replicated - - jstests/core/read_after_optime.js # verifies read after optime fails on standalone - - jstests/core/remove8.js # db.eval() used - - jstests/core/rename4.js # db.eval() used - - jstests/core/shell1.js # tests setSlaveOk() variations on standalone mongod - - jstests/core/shellkillop.js # db.eval() used - - jstests/core/shell_writeconcern.js # checks write concern shell helpers - - jstests/core/storefunc.js # db.eval() used - - jstests/core/write_result.js # Tests invalid writeConcern, we shouldn't override. - # Tests that need triaging & remediation | blacklist decision - # Comments list possible problem point under review. - - jstests/core/capped6.js # Uses captrunc test command. - - jstests/core/convert_to_capped_nonexistant.js # Uses convertToCapped and captrunc command. - - jstests/core/stages_delete.js # Uses stageDebug command for deletes. - + # These tests are not expected to pass with replica-sets: + - jstests/core/dbadmin.js + - jstests/core/opcounters_write_cmd.js + - jstests/core/read_after_optime.js + - jstests/core/capped_update.js + # These tests use benchRun(), which isn't configured to use the overridden writeConcern. + - jstests/core/bench_test*.js + exclude_with_any_tags: + ## + # The next three tags correspond to the special errors thrown by the + # set_read_and_write_concerns.js override when it refuses to replace the readConcern or + # writeConcern of a particular command. Above each tag are the message(s) that cause the tag to + # be warranted. + ## + # "Cowardly refusing to override read concern of command: ..." + - assumes_read_concern_unchanged + # "Cowardly refusing to override write concern of command: ..." + - assumes_write_concern_unchanged + # "Cowardly refusing to run test with overridden write concern when it uses a command that can + # only perform w=1 writes: ..." + - requires_eval_command executor: js_test: config: shell_options: - eval: "var testingReplication = true; load('jstests/libs/override_methods/set_majority_read_and_write_concerns.js');" + global_vars: + TestData: + defaultReadConcernLevel: majority + eval: >- + var testingReplication = true; + load('jstests/libs/override_methods/set_read_and_write_concerns.js'); readMode: commands hooks: - - class: ValidateCollections + # The CheckReplDBHash hook waits until all operations have replicated to and have been applied + # on the secondaries, so we run the ValidateCollections hook after it to ensure we're + # validating the entire contents of the collection. - class: CheckReplDBHash + - class: ValidateCollections + - class: CleanEveryN + n: 20 fixture: class: ReplicaSetFixture mongod_options: diff --git a/jstests/core/batch_write_command_delete.js b/jstests/core/batch_write_command_delete.js index 2aefcea6a7f..99b5f8e3a61 100644 --- a/jstests/core/batch_write_command_delete.js +++ b/jstests/core/batch_write_command_delete.js @@ -1,3 +1,5 @@ +// @tags: [assumes_write_concern_unchanged] + // // Ensures that mongod respects the batch write protocols for delete // diff --git a/jstests/core/batch_write_command_insert.js b/jstests/core/batch_write_command_insert.js index 6b42cf08ebf..ffd788cd0dd 100644 --- a/jstests/core/batch_write_command_insert.js +++ b/jstests/core/batch_write_command_insert.js @@ -1,3 +1,5 @@ +// @tags: [assumes_write_concern_unchanged] + // // Ensures that mongod respects the batch write protocol for inserts // diff --git a/jstests/core/batch_write_command_update.js b/jstests/core/batch_write_command_update.js index 2d9d2d699b2..987525e2515 100644 --- a/jstests/core/batch_write_command_update.js +++ b/jstests/core/batch_write_command_update.js @@ -1,3 +1,5 @@ +// @tags: [assumes_write_concern_unchanged] + // // Ensures that mongod respects the batch write protocols for updates // diff --git a/jstests/core/capped6.js b/jstests/core/capped6.js index d7b8a60985a..afbdf51dfa2 100644 --- a/jstests/core/capped6.js +++ b/jstests/core/capped6.js @@ -1,4 +1,11 @@ // Test NamespaceDetails::cappedTruncateAfter via "captrunc" command +// +// @tags: [ +// # This test attempts to perform read operations on a capped collection after truncating +// # documents using the captrunc command. The writes from the captrunc command aren't guaranteed +// # to become visible until a later w="majority" write occurs. +// assumes_write_concern_unchanged, +// ] (function() { var coll = db.capped6; diff --git a/jstests/core/collection_info_cache_race.js b/jstests/core/collection_info_cache_race.js index d57fc3340db..8fcde050e99 100644 --- a/jstests/core/collection_info_cache_race.js +++ b/jstests/core/collection_info_cache_race.js @@ -5,9 +5,9 @@ var coll = db.collection_info_cache_race; coll.drop(); assert.commandWorked(db.createCollection(coll.getName(), {autoIndexId: false})); // Fails when SERVER-16502 was not fixed, due to invariant -assert.writeOK(coll.save({_id: false}, {writeConcern: {w: 1}})); +assert.writeOK(coll.save({_id: false})); coll.drop(); assert.commandWorked(db.createCollection(coll.getName(), {autoIndexId: false})); assert.eq(null, coll.findOne()); -assert.writeOK(coll.save({_id: false}, {writeConcern: {w: 1}})); +assert.writeOK(coll.save({_id: false})); diff --git a/jstests/core/constructors.js b/jstests/core/constructors.js index 814766ee2c3..c4388c34c16 100644 --- a/jstests/core/constructors.js +++ b/jstests/core/constructors.js @@ -1,4 +1,6 @@ // Tests to see what validity checks are done for 10gen specific object construction +// +// @tags: [requires_eval_command] // Takes a list of constructors and returns a new list with an extra entry for each constructor with // "new" prepended diff --git a/jstests/core/crud_api.js b/jstests/core/crud_api.js index f6cc77025c3..3aceea1c07b 100644 --- a/jstests/core/crud_api.js +++ b/jstests/core/crud_api.js @@ -1,3 +1,5 @@ +// @tags: [assumes_write_concern_unchanged] + (function() { "use strict"; diff --git a/jstests/core/dropdb_race.js b/jstests/core/dropdb_race.js index b4666ecc3ad..9c62c9c886c 100644 --- a/jstests/core/dropdb_race.js +++ b/jstests/core/dropdb_race.js @@ -1,4 +1,6 @@ // test dropping a db with simultaneous commits +// +// @tags: [assumes_write_concern_unchanged] m = db.getMongo(); baseName = "jstests_dur_droprace"; diff --git a/jstests/core/error2.js b/jstests/core/error2.js index 6f0b95bc17e..fb6a8e6e3b2 100644 --- a/jstests/core/error2.js +++ b/jstests/core/error2.js @@ -1,4 +1,5 @@ // Test that client gets stack trace on failed invoke +// @tags: [requires_eval_command] f = db.jstests_error2; diff --git a/jstests/core/eval0.js b/jstests/core/eval0.js index a0c93da2cab..324816953b0 100644 --- a/jstests/core/eval0.js +++ b/jstests/core/eval0.js @@ -1,4 +1,4 @@ - +// @tags: [requires_eval_command] assert.eq(17, db.eval(function() { return 11 + 6; diff --git a/jstests/core/eval1.js b/jstests/core/eval1.js index 8b139cae02a..b5bffac892e 100644 --- a/jstests/core/eval1.js +++ b/jstests/core/eval1.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval1; t.drop(); diff --git a/jstests/core/eval3.js b/jstests/core/eval3.js index c4f8be21056..b95837a6817 100644 --- a/jstests/core/eval3.js +++ b/jstests/core/eval3.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval3; t.drop(); diff --git a/jstests/core/eval4.js b/jstests/core/eval4.js index 0d120b393de..9b0c2a49d82 100644 --- a/jstests/core/eval4.js +++ b/jstests/core/eval4.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval4; t.drop(); diff --git a/jstests/core/eval5.js b/jstests/core/eval5.js index 46bd679dd77..815365ac8b9 100644 --- a/jstests/core/eval5.js +++ b/jstests/core/eval5.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval5; t.drop(); diff --git a/jstests/core/eval6.js b/jstests/core/eval6.js index 31258f6917b..96b43b3516c 100644 --- a/jstests/core/eval6.js +++ b/jstests/core/eval6.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval6; t.drop(); diff --git a/jstests/core/eval7.js b/jstests/core/eval7.js index 89f395d5128..892bcdf6d72 100644 --- a/jstests/core/eval7.js +++ b/jstests/core/eval7.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] assert.eq(6, db.eval("5 + 1"), "A"); assert.throws(function(z) { diff --git a/jstests/core/eval9.js b/jstests/core/eval9.js index 6998345bf13..8a058f41cee 100644 --- a/jstests/core/eval9.js +++ b/jstests/core/eval9.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] a = [1, "asd", null, [2, 3], new Date(), {x: 1}]; @@ -20,4 +21,4 @@ try { db.eval("return db"); db.eval("return print"); } catch (ex) { -}
\ No newline at end of file +} diff --git a/jstests/core/eval_mr.js b/jstests/core/eval_mr.js index 84036b1e0d5..e15af9975c1 100644 --- a/jstests/core/eval_mr.js +++ b/jstests/core/eval_mr.js @@ -1,4 +1,6 @@ // Test that the eval command can't be used to invoke the mapReduce command. SERVER-17889. +// +// @tags: [requires_eval_command] (function() { "use strict"; db.eval_mr.drop(); diff --git a/jstests/core/eval_nolock.js b/jstests/core/eval_nolock.js index 9511784becb..0fde2666f5d 100644 --- a/jstests/core/eval_nolock.js +++ b/jstests/core/eval_nolock.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval_nolock; t.drop(); diff --git a/jstests/core/evala.js b/jstests/core/evala.js index 7ccf33ac754..09241eeedee 100644 --- a/jstests/core/evala.js +++ b/jstests/core/evala.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.evala; t.drop(); diff --git a/jstests/core/evalb.js b/jstests/core/evalb.js index 3391c4cc4f2..2de16f4ea83 100644 --- a/jstests/core/evalb.js +++ b/jstests/core/evalb.js @@ -1,5 +1,7 @@ // Check the return value of a db.eval function running a database query, and ensure the function's // contents are logged in the profile log. +// +// @tags: [requires_eval_command] // Use a reserved database name to avoid a conflict in the parallel test suite. var stddb = db; diff --git a/jstests/core/evalc.js b/jstests/core/evalc.js index 0d55790afe3..cf678cc6d11 100644 --- a/jstests/core/evalc.js +++ b/jstests/core/evalc.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + t = db.jstests_evalc; t.drop(); diff --git a/jstests/core/evald.js b/jstests/core/evald.js index 8049d2ba8ae..43e74fc4600 100644 --- a/jstests/core/evald.js +++ b/jstests/core/evald.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + t = db.jstests_evald; t.drop(); diff --git a/jstests/core/evale.js b/jstests/core/evale.js index 1ddc8519fc6..20384e0d741 100644 --- a/jstests/core/evale.js +++ b/jstests/core/evale.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + t = db.jstests_evale; t.drop(); diff --git a/jstests/core/evalg.js b/jstests/core/evalg.js index 570464cbce2..16863cd685d 100644 --- a/jstests/core/evalg.js +++ b/jstests/core/evalg.js @@ -1,4 +1,6 @@ // SERVER-17499: Test behavior of getMore on aggregation cursor under eval command. +// +// @tags: [requires_eval_command] db.evalg.drop(); for (var i = 0; i < 102; ++i) { db.evalg.insert({}); diff --git a/jstests/core/evalh.js b/jstests/core/evalh.js index e1058fbdce4..e222b9aede6 100644 --- a/jstests/core/evalh.js +++ b/jstests/core/evalh.js @@ -1,5 +1,7 @@ /** * Test that db.eval does not support auth. + * + * @tags: [requires_eval_command] */ (function() { 'use strict'; diff --git a/jstests/core/evalj.js b/jstests/core/evalj.js index f2326fff365..d6ef46430de 100644 --- a/jstests/core/evalj.js +++ b/jstests/core/evalj.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + (function() { "use strict"; diff --git a/jstests/core/fsync.js b/jstests/core/fsync.js index 57762ce8c78..409b8d9a962 100644 --- a/jstests/core/fsync.js +++ b/jstests/core/fsync.js @@ -6,6 +6,8 @@ * - Confirm that writes can progress after fsyncUnlock * - Confirm that the command can be run repeatedly without breaking things * - Confirm that the pseudo commands and eval can perform fsyncLock/Unlock + * + * @tags: [requires_eval_command] */ (function() { "use strict"; diff --git a/jstests/core/geo_update_btree.js b/jstests/core/geo_update_btree.js index 6376135d905..4f4e863ce47 100644 --- a/jstests/core/geo_update_btree.js +++ b/jstests/core/geo_update_btree.js @@ -1,4 +1,6 @@ // Tests whether the geospatial search is stable under btree updates +// +// @tags: [assumes_write_concern_unchanged] var coll = db.getCollection("jstests_geo_update_btree"); coll.drop(); diff --git a/jstests/core/js3.js b/jstests/core/js3.js index 36d16051135..2b5e0dfccc3 100644 --- a/jstests/core/js3.js +++ b/jstests/core/js3.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.jstests_js3; diff --git a/jstests/core/js7.js b/jstests/core/js7.js index aeaec66ff47..083c3ab3c83 100644 --- a/jstests/core/js7.js +++ b/jstests/core/js7.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + t = db.jstests_js7; t.drop(); diff --git a/jstests/core/js9.js b/jstests/core/js9.js index b29a31afdc4..4cf0bcac1c7 100644 --- a/jstests/core/js9.js +++ b/jstests/core/js9.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + c = db.jstests_js9; c.drop(); diff --git a/jstests/core/profile2.js b/jstests/core/profile2.js index bb1605abd1e..9da6410d2ac 100644 --- a/jstests/core/profile2.js +++ b/jstests/core/profile2.js @@ -24,7 +24,7 @@ assert(result.hasOwnProperty('millis')); assert(result.hasOwnProperty('query')); assert.eq('string', typeof(result.query)); // String value is truncated. -assert(result.query.match(/filter: { a: "a+\.\.\." } }$/)); +assert(result.query.match(/filter: { a: "a+\.\.\." }/)); assert.commandWorked(coll.getDB().runCommand({profile: 0})); coll.getDB().system.profile.drop(); diff --git a/jstests/core/recursion.js b/jstests/core/recursion.js index 926250be20d..1db491c8ae9 100644 --- a/jstests/core/recursion.js +++ b/jstests/core/recursion.js @@ -1,5 +1,7 @@ -// Basic tests for a form of stack recursion that's been shown to cause C++ -// side stack overflows in the past. See SERVER-19614. +// Basic tests for a form of stack recursion that's been shown to cause C++ side stack overflows in +// the past. See SERVER-19614. +// +// @tags: [requires_eval_command] (function() { "use strict"; diff --git a/jstests/core/regex_not_id.js b/jstests/core/regex_not_id.js index 1f15250f240..35b2c858867 100644 --- a/jstests/core/regex_not_id.js +++ b/jstests/core/regex_not_id.js @@ -3,10 +3,10 @@ var testColl = db.regex_not_id; testColl.drop(); -assert.writeOK(testColl.insert({_id: "ABCDEF1"}, {writeConcern: {w: 1}})); +assert.writeOK(testColl.insert({_id: "ABCDEF1"})); // Should be an error. -assert.writeError(testColl.insert({_id: /^A/}, {writeConcern: {w: 1}})); +assert.writeError(testColl.insert({_id: /^A/})); // _id doesn't have to be first; still disallowed -assert.writeError(testColl.insert({xxx: "ABCDEF", _id: /ABCDEF/}, {writeConcern: {w: 1}}));
\ No newline at end of file +assert.writeError(testColl.insert({xxx: "ABCDEF", _id: /ABCDEF/})); diff --git a/jstests/core/remove8.js b/jstests/core/remove8.js index 563e4708cf9..3c9fd6a11a1 100644 --- a/jstests/core/remove8.js +++ b/jstests/core/remove8.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.remove8; t.drop(); diff --git a/jstests/core/rename4.js b/jstests/core/rename4.js index 904709175f9..edef2e0c7bd 100644 --- a/jstests/core/rename4.js +++ b/jstests/core/rename4.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + t = db.jstests_rename4; t.drop(); diff --git a/jstests/core/shell_writeconcern.js b/jstests/core/shell_writeconcern.js index f3f190061cf..e3e7a23a9aa 100644 --- a/jstests/core/shell_writeconcern.js +++ b/jstests/core/shell_writeconcern.js @@ -1,7 +1,10 @@ "use strict"; + // check that shell writeconcern work correctly // 1.) tests that it can be set on each level and is inherited // 2.) tests that each operation (update/insert/remove/save) take and ensure a write concern +// +// @tags: [assumes_write_concern_unchanged] var collA = db.shell_wc_a; var collB = db.shell_wc_b; diff --git a/jstests/core/stages_delete.js b/jstests/core/stages_delete.js index 1624b1fcc6a..d7674ba210f 100644 --- a/jstests/core/stages_delete.js +++ b/jstests/core/stages_delete.js @@ -1,3 +1,9 @@ +// @tags: [ +// # This test attempts to remove documents using the stageDebug command, which doesn't support +// # specifying a writeConcern. +// assumes_write_concern_unchanged, +// ] + // Test basic delete stage functionality. var coll = db.stages_delete; var collScanStage = { diff --git a/jstests/core/storefunc.js b/jstests/core/storefunc.js index 8598e9cc62b..15abc56421e 100644 --- a/jstests/core/storefunc.js +++ b/jstests/core/storefunc.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + // Use a private sister database to avoid conflicts with other tests that use system.js var testdb = db.getSisterDB("storefunc"); var res; diff --git a/jstests/core/write_result.js b/jstests/core/write_result.js index 86486089c68..453be4ca1c9 100644 --- a/jstests/core/write_result.js +++ b/jstests/core/write_result.js @@ -1,6 +1,8 @@ // // Tests the behavior of single writes using write commands // +// @tags: [assumes_write_concern_unchanged] +// var coll = db.write_result; coll.drop(); diff --git a/jstests/libs/override_methods/override_helpers.js b/jstests/libs/override_methods/override_helpers.js new file mode 100644 index 00000000000..2197ec5f0ee --- /dev/null +++ b/jstests/libs/override_methods/override_helpers.js @@ -0,0 +1,117 @@ +/** + * The OverrideHelpers object defines convenience methods for overriding commands and functions in + * the mongo shell. + */ +var OverrideHelpers = (function() { + "use strict"; + + function isAggregationWithOutStage(commandName, commandObj) { + if (commandName !== "aggregate" || typeof commandObj !== "object" || commandObj === null) { + return false; + } + + if (!Array.isArray(commandObj.pipeline) || commandObj.pipeline.length === 0) { + return false; + } + + const lastStage = commandObj.pipeline[commandObj.pipeline.length - 1]; + if (typeof lastStage !== "object" || lastStage === null) { + return false; + } + + return Object.keys(lastStage)[0] === "$out"; + } + + function isMapReduceWithInlineOutput(commandName, commandObj) { + if ((commandName !== "mapReduce" && commandName !== "mapreduce") || + typeof commandObj !== "object" || commandObj === null) { + return false; + } + + if (typeof commandObj.out !== "object") { + return false; + } + + return commandObj.out.hasOwnProperty("inline"); + } + + function prependOverrideInParallelShell(overrideFile) { + const startParallelShellOriginal = startParallelShell; + + startParallelShell = function(jsCode, port, noConnect) { + var newCode; + if (typeof jsCode === "function") { + // Load the override file and immediately invoke the supplied function. + newCode = `load("${overrideFile}"); (${jsCode})();`; + } else { + newCode = `load("${overrideFile}"); ${jsCode};`; + } + + return startParallelShellOriginal(newCode, port, noConnect); + }; + } + + function overrideRunCommand(overrideFunc) { + const DBQueryOriginal = DBQuery; + const mongoRunCommandOriginal = Mongo.prototype.runCommand; + const mongoRunCommandWithMetadataOriginal = Mongo.prototype.runCommandWithMetadata; + + DBQuery = function( + mongo, db, collection, ns, query, fields, limit, skip, batchSize, options) { + // If the query isn't being run against the "$cmd" or "$cmd.sys" namespaces, then it + // represents an OP_QUERY find on that collection. We skip calling overrideFunc() in + // this case because the operation doesn't represent a command. + if (!(collection instanceof DBCollection && + (collection.getName() === "$cmd" || collection.getName().startsWith("$cmd.")))) { + return DBQueryOriginal.apply(this, arguments); + } + + // Due to the function signatures of Mongo.prototype.runCommand() and + // Mongo.prototype.runCommandWithMetadata(), the overrideFunc() function expects that + // the Mongo connection object is passed as the first argument and also represents the + // 'this' parameter. As a workaround, we bind the appropriate 'this' value to the + // DBQueryOriginal constructor ahead of time. + const commandName = Object.keys(query)[0]; + return overrideFunc( + mongo, + db.getName(), + commandName, + query, + DBQueryOriginal.bind(this), + (query) => + [mongo, db, collection, ns, query, fields, limit, skip, batchSize, options]); + }; + + // Copy any properties (e.g. DBQuery.Option) that are set on DBQueryOriginal. + Object.keys(DBQueryOriginal).forEach(function(key) { + DBQuery[key] = DBQueryOriginal[key]; + }); + + Mongo.prototype.runCommand = function(dbName, commandObj, options) { + const commandName = Object.keys(commandObj)[0]; + return overrideFunc(this, + dbName, + commandName, + commandObj, + mongoRunCommandOriginal, + (commandObj) => [dbName, commandObj, options]); + }; + + Mongo.prototype.runCommandWithMetadata = function(dbName, metadata, commandArgs) { + const commandName = Object.keys(commandArgs)[0]; + return overrideFunc(this, + dbName, + commandName, + commandArgs, + mongoRunCommandWithMetadataOriginal, + (commandArgs) => [dbName, metadata, commandArgs]); + }; + } + + return { + isAggregationWithOutStage: isAggregationWithOutStage, + isMapReduceWithInlineOutput: isMapReduceWithInlineOutput, + prependOverrideInParallelShell: prependOverrideInParallelShell, + overrideRunCommand: overrideRunCommand, + }; +})(); diff --git a/jstests/libs/override_methods/set_majority_read_and_write_concerns.js b/jstests/libs/override_methods/set_majority_read_and_write_concerns.js deleted file mode 100644 index 767134d43a4..00000000000 --- a/jstests/libs/override_methods/set_majority_read_and_write_concerns.js +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Use prototype overrides to set a read concern of "majority" and a write concern of "majority" - * while running core tests. - */ -(function() { - "use strict"; - var defaultWriteConcern = { - w: "majority", - // Use a "signature" value that won't typically match a value assigned in normal use. - wtimeout: 60321 - }; - var defaultReadConcern = { - level: "majority" - }; - - var originalDBQuery = DBQuery; - - DBQuery = function(mongo, db, collection, ns, query, fields, limit, skip, batchSize, options) { - if (ns.endsWith("$cmd")) { - if (query.hasOwnProperty("writeConcern") && - bsonWoCompare(query.writeConcern, defaultWriteConcern) !== 0) { - jsTestLog("Warning: DBQuery overriding existing writeConcern of: " + - tojson(query.writeConcern)); - query.writeConcern = defaultWriteConcern; - } - } - - return originalDBQuery.apply(this, arguments); - }; - - DBQuery.Option = originalDBQuery.Option; - - var originalStartParallelShell = startParallelShell; - startParallelShell = function(jsCode, port, noConnect) { - var newCode; - var overridesFile = "jstests/libs/override_methods/set_majority_read_and_write_concerns.js"; - - if (typeof(jsCode) === "function") { - // Load the override file and immediately invoke the supplied function. - // clang-format off - newCode = `load("${overridesFile}"); (${jsCode})();`; - // clang-format on - } else { - // clang-format off - newCode = `load("${overridesFile}"); ${jsCode};`; - // clang-format on - } - - return originalStartParallelShell(newCode, port, noConnect); - }; - - DB.prototype._runCommandImpl = function(dbName, obj, options) { - var cmdName = ""; - for (var fieldName in obj) { - cmdName = fieldName; - break; - } - - // These commands directly support a writeConcern argument. - var commandsToForceWriteConcern = [ - "applyOps", - "authSchemaUpgrade", - "createRole", - "createUser", - "delete", - "dropAllRolesFromDatabase", - "dropAllUsersFromDatabase", - "dropRole", - "dropUser", - "findAndModify", - "findandmodify", - "grantPrivilegesToRole", - "grantRolesToRole", - "grantRolesToUser", - "insert", - "revokeRolesFromRole", - "revokeRolesFromUser", - "update", - "updateRole", - "updateUser", - ]; - - // These commands do writes but do not support a writeConcern argument. Emulate it with a - // getLastError command. - var commandsToEmulateWriteConcern = ["createIndexes", ]; - - // These are reading commands that support majority readConcern. - var commandsToForceReadConcern = - ["count", "distinct", "find", "geoNear", "geoSearch", "group", ]; - - var forceWriteConcern = Array.contains(commandsToForceWriteConcern, cmdName); - var emulateWriteConcern = Array.contains(commandsToEmulateWriteConcern, cmdName); - var forceReadConcern = Array.contains(commandsToForceReadConcern, cmdName); - - if (cmdName === "aggregate") { - // Aggregate can be either a read or a write depending on whether it has a $out stage. - // $out is required to be the last stage of the pipeline. - var stages = obj.pipeline; - var hasOut = stages && (stages.length !== 0) && ('$out' in stages[stages.length - 1]); - if (hasOut) { - emulateWriteConcern = true; - } else { - forceReadConcern = true; - } - } - - if (forceWriteConcern) { - if (obj.hasOwnProperty("writeConcern")) { - if (bsonWoCompare(obj.writeConcern, defaultWriteConcern) !== 0) { - jsTestLog("Warning: _runCommandImpl overriding existing writeConcern of: " + - tojson(obj.writeConcern)); - obj.writeConcern = defaultWriteConcern; - } - } else { - obj.writeConcern = defaultWriteConcern; - } - - } else if (forceReadConcern) { - if (obj.hasOwnProperty("readConcern")) { - if (bsonWoCompare(obj.readConcern, defaultReadConcern) !== 0) { - jsTestLog("Warning: _runCommandImpl overriding existing readConcern of: " + - tojson(obj.readConcern)); - obj.readConcern = defaultReadConcern; - } - } else { - obj.readConcern = defaultReadConcern; - } - } - - var res = this.getMongo().runCommand(dbName, obj, options); - - if (res.ok && emulateWriteConcern) { - // We only emulate WriteConcern if the command succeeded to match the behavior of - // commands that support WriteConcern. - var gleCmd = Object.extend({getLastError: 1}, defaultWriteConcern); - assert.commandWorked(this.getMongo().runCommand(dbName, gleCmd, options)); - } - - return res; - }; - - // Use a majority write concern if the operation does not specify one. - DBCollection.prototype.getWriteConcern = function() { - return new WriteConcern(defaultWriteConcern); - }; - -})(); diff --git a/jstests/libs/override_methods/set_read_and_write_concerns.js b/jstests/libs/override_methods/set_read_and_write_concerns.js new file mode 100644 index 00000000000..56ae306b796 --- /dev/null +++ b/jstests/libs/override_methods/set_read_and_write_concerns.js @@ -0,0 +1,190 @@ +/** + * Use prototype overrides to set read concern and write concern while running tests. + */ +(function() { + "use strict"; + + load("jstests/libs/override_methods/override_helpers.js"); + + if (typeof TestData === "undefined" || !TestData.hasOwnProperty("defaultReadConcernLevel")) { + throw new Error( + "The readConcern level to use must be set as the 'defaultReadConcernLevel'" + + " property on the global TestData object"); + } + + const kDefaultReadConcern = {level: TestData.defaultReadConcernLevel}; + const kDefaultWriteConcern = + (TestData.hasOwnProperty("defaultWriteConcern")) ? TestData.defaultWriteConcern : { + w: "majority", + // Use a "signature" value that won't typically match a value assigned in normal use. + // This way the wtimeout set by this override is distinguishable in the server logs. + wtimeout: 5 * 60 * 1000 + 321, // 300321ms + }; + + const kCommandsSupportingReadConcern = new Set([ + "aggregate", + "count", + "distinct", + "find", + "geoNear", + "geoSearch", + "group", + "parallelCollectionScan", + ]); + + const kCommandsSupportingWriteConcern = new Set([ + "applyOps", + "authSchemaUpgrade", + "createRole", + "createUser", + "delete", + "dropAllRolesFromDatabase", + "dropAllUsersFromDatabase", + "dropRole", + "dropUser", + "findAndModify", + "findandmodify", + "grantPrivilegesToRole", + "grantRolesToRole", + "grantRolesToUser", + "insert", + "revokeRolesFromRole", + "revokeRolesFromUser", + "update", + "updateRole", + "updateUser", + ]); + + function runCommandWithReadAndWriteConcerns( + conn, dbName, commandName, commandObj, func, makeFuncArgs) { + if (typeof commandObj !== "object" || commandObj === null) { + return func.apply(conn, makeFuncArgs(commandObj)); + } + + // If the command is in a wrapped form, then we look for the actual command object inside + // the query/$query object. + var commandObjUnwrapped = commandObj; + if (commandName === "query" || commandName === "$query") { + commandObjUnwrapped = commandObj[commandName]; + commandName = Object.keys(commandObjUnwrapped)[0]; + } + + if (commandName === "eval" || commandName === "$eval") { + throw new Error("Cowardly refusing to run test with overridden write concern when it" + + " uses a command that can only perform w=1 writes: " + + tojson(commandObj)); + } + + var shouldForceReadConcern = kCommandsSupportingReadConcern.has(commandName); + var shouldForceWriteConcern = kCommandsSupportingWriteConcern.has(commandName); + var shouldEmulateWriteConcern = + (commandName === "aggregate" || commandName === "createIndexes" || + commandName === "mapReduce" || commandName === "mapreduce" || + commandName === "mapreduce.shardedfinish"); + + if (commandName === "aggregate") { + if (OverrideHelpers.isAggregationWithOutStage(commandName, commandObjUnwrapped)) { + // The $out stage can only be used with readConcern={level: "local"}. + shouldForceReadConcern = false; + } else { + // Only a $out aggregation does writes. + shouldEmulateWriteConcern = false; + } + + if (commandObjUnwrapped.explain) { + // Attempting to specify a readConcern while explaining an aggregation would always + // return an error prior to SERVER-30582 and it otherwise only compatible with + // readConcern={level: "local"}. + shouldForceReadConcern = false; + } + } else if (OverrideHelpers.isMapReduceWithInlineOutput(commandName, commandObjUnwrapped)) { + // A writeConcern can only be used with non-inline output. + shouldForceWriteConcern = false; + } else if (commandObj[commandName] === "system.profile") { + // Writes to the "system.profile" collection aren't guaranteed to be visible in the same + // majority-committed snapshot as the command they originated from. We don't override + // the readConcern for operations on the "system.profile" collection so that tests which + // assert on its contents continue to succeed. + shouldForceReadConcern = false; + } + + const inWrappedForm = commandObj !== commandObjUnwrapped; + + if (shouldForceReadConcern) { + // We create a copy of 'commandObj' to avoid mutating the parameter the caller + // specified. + commandObj = Object.assign({}, commandObj); + if (inWrappedForm) { + commandObjUnwrapped = Object.assign({}, commandObjUnwrapped); + commandObj[Object.keys(commandObj)[0]] = commandObjUnwrapped; + } else { + commandObjUnwrapped = commandObj; + } + + if (commandObjUnwrapped.hasOwnProperty("readConcern")) { + var readConcern = commandObjUnwrapped.readConcern; + + if (typeof readConcern !== "object" || readConcern === null || + (readConcern.hasOwnProperty("level") && + bsonWoCompare({_: readConcern.level}, {_: kDefaultReadConcern.level}) !== 0)) { + throw new Error("Cowardly refusing to override read concern of command: " + + tojson(commandObj)); + } + + // We create a copy of the readConcern object to avoid mutating the parameter the + // caller specified. + readConcern = Object.assign({}, readConcern, kDefaultReadConcern); + commandObjUnwrapped.readConcern = readConcern; + } else { + commandObjUnwrapped.readConcern = kDefaultReadConcern; + } + } + + if (shouldForceWriteConcern) { + // We create a copy of 'commandObj' to avoid mutating the parameter the caller + // specified. + commandObj = Object.assign({}, commandObj); + if (inWrappedForm) { + commandObjUnwrapped = Object.assign({}, commandObjUnwrapped); + commandObj[Object.keys(commandObj)[0]] = commandObjUnwrapped; + } else { + commandObjUnwrapped = commandObj; + } + + if (commandObjUnwrapped.hasOwnProperty("writeConcern")) { + var writeConcern = commandObjUnwrapped.writeConcern; + + if (typeof writeConcern !== "object" || writeConcern === null || + (writeConcern.hasOwnProperty("w") && + bsonWoCompare({_: writeConcern.w}, {_: kDefaultWriteConcern.w}) !== 0)) { + throw new Error("Cowardly refusing to override write concern of command: " + + tojson(commandObj)); + } + + // We create a copy of the writeConcern object to avoid mutating the parameter the + // caller specified. + writeConcern = Object.assign({}, writeConcern, kDefaultWriteConcern); + commandObjUnwrapped.writeConcern = writeConcern; + } else { + commandObjUnwrapped.writeConcern = kDefaultWriteConcern; + } + } + + const serverResponse = func.apply(conn, makeFuncArgs(commandObj)); + + if (shouldEmulateWriteConcern && serverResponse.ok === 1) { + // We only wait for the write concern if the command succeeded to match what the + // server's behavior would have been if the command supports the "writeConcern" option + // itself. + assert.commandWorked( + conn.runCommand(dbName, Object.assign({getLastError: 1}, kDefaultWriteConcern), 0)); + } + + return serverResponse; + } + + OverrideHelpers.prependOverrideInParallelShell( + "jstests/libs/override_methods/set_read_and_write_concerns.js"); + + OverrideHelpers.overrideRunCommand(runCommandWithReadAndWriteConcerns); +})(); |