diff options
author | Lingzhi Deng <lingzhi.deng@mongodb.com> | 2019-05-20 00:49:33 -0400 |
---|---|---|
committer | Lingzhi Deng <lingzhi.deng@mongodb.com> | 2019-05-23 16:46:30 -0400 |
commit | 8a89b076d28a904c200e491698ef1169dd8a7254 (patch) | |
tree | dc32fa3bdc62cdd9f8b9b84777f0df8f03f43cec | |
parent | 47d4eca3fcdfa8eed0f1bef28021c8603452dec3 (diff) | |
download | mongo-8a89b076d28a904c200e491698ef1169dd8a7254.tar.gz |
SERVER-40938: disallow afterClusterTime and ignore prepare conflicts for dbhash and map-reduce
71 files changed, 470 insertions, 113 deletions
diff --git a/buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough.yml index 0d42534222c..695a79ef218 100644 --- a/buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough.yml @@ -117,6 +117,7 @@ selector: # "Cowardly refusing to run test with overridden read preference when it reads from a # non-replicated collection: ..." - assumes_read_preference_unchanged + - does_not_support_causal_consistency - requires_collstats executor: diff --git a/buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough_auth.yml b/buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough_auth.yml index 81c98f9ec44..e3487009836 100644 --- a/buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough_auth.yml +++ b/buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough_auth.yml @@ -146,6 +146,7 @@ selector: - assumes_read_preference_unchanged # TODO SERVER-35447: Multiple users cannot be authenticated on one connection within a session. - creates_and_authenticates_user + - does_not_support_causal_consistency - requires_collstats executor: diff --git a/buildscripts/resmokeconfig/suites/causally_consistent_jscore_txns_passthrough.yml b/buildscripts/resmokeconfig/suites/causally_consistent_jscore_txns_passthrough.yml index 61d2fbfd9d1..f97c8781099 100644 --- a/buildscripts/resmokeconfig/suites/causally_consistent_jscore_txns_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/causally_consistent_jscore_txns_passthrough.yml @@ -17,6 +17,7 @@ selector: - jstests/core/txns/prepare_conflict_aggregation_behavior.js exclude_with_any_tags: + - does_not_support_causal_consistency # Transactions are not allowed to operate on capped collections. - requires_capped diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_causal_consistency.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_causal_consistency.yml index c4a59b7355c..169a4d1fad5 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_replication_causal_consistency.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_replication_causal_consistency.yml @@ -17,6 +17,7 @@ selector: - jstests/concurrency/fsm_workloads/reindex_background.js exclude_with_any_tags: + - does_not_support_causal_consistency - uses_transactions # collStats is not causally consistent - requires_collstats diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn.yml index 9291a7eb561..b4859b0bda5 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn.yml @@ -27,6 +27,7 @@ selector: - jstests/concurrency/fsm_workloads/view_catalog_direct_system_writes.js exclude_with_any_tags: + - does_not_support_causal_consistency - requires_sharding # Sharing cursors between state functions will fail in this suite because it will attempt to use # the same cursor in multiple transactions. diff --git a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml index 0d8a244567e..2350e6b5408 100644 --- a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml @@ -48,6 +48,9 @@ selector: # Unacknowledged writes prohibited in an explicit session. - jstests/core/batch_write_command_w0.js + exclude_with_any_tags: + - does_not_support_causal_consistency + executor: archive: hooks: diff --git a/buildscripts/resmokeconfig/suites/secondary_reads_passthrough.yml b/buildscripts/resmokeconfig/suites/secondary_reads_passthrough.yml index 6c8b8d151e2..1dfdbdb80e9 100644 --- a/buildscripts/resmokeconfig/suites/secondary_reads_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/secondary_reads_passthrough.yml @@ -72,6 +72,7 @@ selector: # "Cowardly refusing to run test with overridden read preference when it reads from a # non-replicated collection: ..." - assumes_read_preference_unchanged + - does_not_support_causal_consistency ## # collStats and dbStats are not causally consistent - requires_collstats diff --git a/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml index da9ff39d989..d27fcb1ba63 100644 --- a/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml @@ -151,6 +151,7 @@ selector: # "Cowardly refusing to run test with overridden read preference when it reads from a # non-replicated collection: ..." - assumes_read_preference_unchanged + - does_not_support_causal_consistency executor: archive: diff --git a/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_txns_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_txns_passthrough.yml index 47570ea19d1..d1eb538e77b 100644 --- a/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_txns_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_txns_passthrough.yml @@ -44,6 +44,7 @@ selector: - jstests/core/txns/non_transactional_operations_on_session_with_transaction.js exclude_with_any_tags: + - does_not_support_causal_consistency # Transactions are not allowed to operate on capped collections. - requires_capped # Prepare is not a command on mongos. diff --git a/jstests/concurrency/fsm_workloads/map_reduce_drop.js b/jstests/concurrency/fsm_workloads/map_reduce_drop.js index 2ea1ffd5572..17900fb74a1 100644 --- a/jstests/concurrency/fsm_workloads/map_reduce_drop.js +++ b/jstests/concurrency/fsm_workloads/map_reduce_drop.js @@ -11,6 +11,10 @@ * * This workload serves as a regression test for SERVER-6757, SERVER-15087, * and SERVER-15842. + * @tags: [ + * # mapReduce does not support afterClusterTime. + * does_not_support_causal_consistency, + * ] */ var $config = (function() { diff --git a/jstests/concurrency/fsm_workloads/map_reduce_inline.js b/jstests/concurrency/fsm_workloads/map_reduce_inline.js index ade5a8aa369..22589f1afad 100644 --- a/jstests/concurrency/fsm_workloads/map_reduce_inline.js +++ b/jstests/concurrency/fsm_workloads/map_reduce_inline.js @@ -8,6 +8,10 @@ * counts of the 'value' field in memory. * * Used as the base workload for the other map-reduce workloads. + * @tags: [ + * # mapReduce does not support afterClusterTime. + * does_not_support_causal_consistency, + * ] */ var $config = (function() { diff --git a/jstests/concurrency/fsm_workloads/map_reduce_interrupt.js b/jstests/concurrency/fsm_workloads/map_reduce_interrupt.js index bbabdd954ac..e33622c0d93 100644 --- a/jstests/concurrency/fsm_workloads/map_reduce_interrupt.js +++ b/jstests/concurrency/fsm_workloads/map_reduce_interrupt.js @@ -7,7 +7,11 @@ * operation. This workload is intended to test that there are no deadlocks or unhandled exceptions * when tearing down a map-reduce command following an interrupt. * - * @tags: [uses_curop_agg_stage] + * @tags: [ + * # mapReduce does not support afterClusterTime. + * does_not_support_causal_consistency, + * uses_curop_agg_stage, + * ] */ load('jstests/concurrency/fsm_libs/extend_workload.js'); // for extendWorkload load('jstests/concurrency/fsm_workloads/map_reduce_replace_nonexistent.js'); // for $config diff --git a/jstests/concurrency/fsm_workloads/map_reduce_merge.js b/jstests/concurrency/fsm_workloads/map_reduce_merge.js index 6359318ab05..125a2b35261 100644 --- a/jstests/concurrency/fsm_workloads/map_reduce_merge.js +++ b/jstests/concurrency/fsm_workloads/map_reduce_merge.js @@ -12,6 +12,10 @@ * of the output collection. * * Writes the results of each thread to the same collection. + * @tags: [ + * # mapReduce does not support afterClusterTime. + * does_not_support_causal_consistency, + * ] */ load('jstests/concurrency/fsm_libs/extend_workload.js'); // for extendWorkload load('jstests/concurrency/fsm_workloads/map_reduce_inline.js'); // for $config diff --git a/jstests/concurrency/fsm_workloads/map_reduce_merge_nonatomic.js b/jstests/concurrency/fsm_workloads/map_reduce_merge_nonatomic.js index 8078c8b65d9..32b324b08bf 100644 --- a/jstests/concurrency/fsm_workloads/map_reduce_merge_nonatomic.js +++ b/jstests/concurrency/fsm_workloads/map_reduce_merge_nonatomic.js @@ -12,6 +12,10 @@ * of the output collection. * * Specifies nonAtomic=true. + * @tags: [ + * # mapReduce does not support afterClusterTime. + * does_not_support_causal_consistency, + * ] */ load('jstests/concurrency/fsm_libs/extend_workload.js'); // for extendWorkload load('jstests/concurrency/fsm_workloads/map_reduce_inline.js'); // for $config diff --git a/jstests/concurrency/fsm_workloads/map_reduce_reduce.js b/jstests/concurrency/fsm_workloads/map_reduce_reduce.js index f5a18bdb522..4cb9d8241ca 100644 --- a/jstests/concurrency/fsm_workloads/map_reduce_reduce.js +++ b/jstests/concurrency/fsm_workloads/map_reduce_reduce.js @@ -10,6 +10,10 @@ * * Uses the "reduce" action to combine the results with the contents * of the output collection. + * @tags: [ + * # mapReduce does not support afterClusterTime. + * does_not_support_causal_consistency, + * ] */ load('jstests/concurrency/fsm_libs/extend_workload.js'); // for extendWorkload load('jstests/concurrency/fsm_workloads/map_reduce_inline.js'); // for $config diff --git a/jstests/concurrency/fsm_workloads/map_reduce_reduce_nonatomic.js b/jstests/concurrency/fsm_workloads/map_reduce_reduce_nonatomic.js index 0afcf7bdb9b..286e2023c66 100644 --- a/jstests/concurrency/fsm_workloads/map_reduce_reduce_nonatomic.js +++ b/jstests/concurrency/fsm_workloads/map_reduce_reduce_nonatomic.js @@ -13,6 +13,10 @@ * * Specifies nonAtomic=true and writes the results of each thread to * the same collection. + * @tags: [ + * # mapReduce does not support afterClusterTime. + * does_not_support_causal_consistency, + * ] */ load('jstests/concurrency/fsm_libs/extend_workload.js'); // for extendWorkload load('jstests/concurrency/fsm_workloads/map_reduce_inline.js'); // for $config diff --git a/jstests/concurrency/fsm_workloads/map_reduce_replace.js b/jstests/concurrency/fsm_workloads/map_reduce_replace.js index 252808a498b..0eda604b7ec 100644 --- a/jstests/concurrency/fsm_workloads/map_reduce_replace.js +++ b/jstests/concurrency/fsm_workloads/map_reduce_replace.js @@ -10,6 +10,10 @@ * * Uses the "replace" action to overwrite the entire contents of the * collection. + * @tags: [ + * # mapReduce does not support afterClusterTime. + * does_not_support_causal_consistency, + * ] */ load('jstests/concurrency/fsm_libs/extend_workload.js'); // for extendWorkload load('jstests/concurrency/fsm_workloads/map_reduce_inline.js'); // for $config diff --git a/jstests/concurrency/fsm_workloads/map_reduce_replace_nonexistent.js b/jstests/concurrency/fsm_workloads/map_reduce_replace_nonexistent.js index a87f07a8c73..623a7b2a936 100644 --- a/jstests/concurrency/fsm_workloads/map_reduce_replace_nonexistent.js +++ b/jstests/concurrency/fsm_workloads/map_reduce_replace_nonexistent.js @@ -9,6 +9,10 @@ * * Uses the "replace" action to write the results to a nonexistent * output collection. + * @tags: [ + * # mapReduce does not support afterClusterTime. + * does_not_support_causal_consistency, + * ] */ load('jstests/concurrency/fsm_libs/extend_workload.js'); // for extendWorkload load('jstests/concurrency/fsm_workloads/map_reduce_inline.js'); // for $config diff --git a/jstests/concurrency/fsm_workloads/map_reduce_replace_remove.js b/jstests/concurrency/fsm_workloads/map_reduce_replace_remove.js index f14390c1968..bf9768d9639 100644 --- a/jstests/concurrency/fsm_workloads/map_reduce_replace_remove.js +++ b/jstests/concurrency/fsm_workloads/map_reduce_replace_remove.js @@ -10,6 +10,10 @@ * yields. * * This workload was designed to reproduce SERVER-15539. + * @tags: [ + * # mapReduce does not support afterClusterTime. + * does_not_support_causal_consistency, + * ] */ load('jstests/concurrency/fsm_libs/extend_workload.js'); // for extendWorkload load('jstests/concurrency/fsm_workloads/map_reduce_replace.js'); // for $config diff --git a/jstests/core/bypass_doc_validation.js b/jstests/core/bypass_doc_validation.js index d2b41c531c2..08b101a3c64 100644 --- a/jstests/core/bypass_doc_validation.js +++ b/jstests/core/bypass_doc_validation.js @@ -1,6 +1,12 @@ // TODO SERVER-40402: Remove 'assumes_write_concern_unchanged' tag. -// @tags: [does_not_support_stepdowns, requires_non_retryable_commands, requires_fastcount, -// assumes_write_concern_unchanged] +// @tags: [ +// assumes_write_concern_unchanged, +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// requires_fastcount, +// requires_non_retryable_commands, +// ] /** * Tests that various database commands respect the 'bypassDocumentValidation' flag: diff --git a/jstests/core/collation.js b/jstests/core/collation.js index c4dc36fb914..af87212346c 100644 --- a/jstests/core/collation.js +++ b/jstests/core/collation.js @@ -1,7 +1,13 @@ // Cannot implicitly shard accessed collections because of collection existing when none // expected. -// @tags: [assumes_no_implicit_collection_creation_after_drop, does_not_support_stepdowns, -// requires_non_retryable_commands, requires_non_retryable_writes] +// @tags: [ +// assumes_no_implicit_collection_creation_after_drop, +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// requires_non_retryable_commands, +// requires_non_retryable_writes, +// ] // Integration tests for the collation feature. (function() { diff --git a/jstests/core/commands_namespace_parsing.js b/jstests/core/commands_namespace_parsing.js index ab9750bfbb5..59863f52f33 100644 --- a/jstests/core/commands_namespace_parsing.js +++ b/jstests/core/commands_namespace_parsing.js @@ -3,6 +3,8 @@ // the "code" property. // @tags: [ // assumes_unsharded_collection, +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, // does_not_support_stepdowns, // requires_fastcount, // requires_getmore, diff --git a/jstests/core/commands_that_do_not_write_do_not_accept_wc.js b/jstests/core/commands_that_do_not_write_do_not_accept_wc.js index 17396961a74..76b6c0c0d1b 100644 --- a/jstests/core/commands_that_do_not_write_do_not_accept_wc.js +++ b/jstests/core/commands_that_do_not_write_do_not_accept_wc.js @@ -5,6 +5,8 @@ * * @tags: [ * assumes_write_concern_unchanged, + * # mapReduce does not support afterClusterTime. + * does_not_support_causal_consistency, * does_not_support_stepdowns, * ] */ diff --git a/jstests/core/constructors.js b/jstests/core/constructors.js index 27b0b7f7406..0fc15cb313f 100644 --- a/jstests/core/constructors.js +++ b/jstests/core/constructors.js @@ -1,6 +1,8 @@ // Tests to see what validity checks are done for 10gen specific object construction // // @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, // does_not_support_stepdowns, // requires_non_retryable_commands, // ] diff --git a/jstests/core/expr.js b/jstests/core/expr.js index 5a5284474d2..6e7bfa2abb8 100644 --- a/jstests/core/expr.js +++ b/jstests/core/expr.js @@ -1,4 +1,10 @@ -// @tags: [does_not_support_stepdowns, requires_getmore, requires_non_retryable_writes] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// requires_getmore, +// requires_non_retryable_writes, +// ] // Tests for $expr in the CRUD commands. (function() { diff --git a/jstests/core/function_string_representations.js b/jstests/core/function_string_representations.js index 448bc8321b8..be39e8df894 100644 --- a/jstests/core/function_string_representations.js +++ b/jstests/core/function_string_representations.js @@ -1,4 +1,8 @@ -// @tags: [does_not_support_stepdowns] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] /** Demonstrate that mapReduce can accept functions represented by strings. * Some drivers do not have a type which represents a Javascript function. These languages represent diff --git a/jstests/core/geo_big_polygon3.js b/jstests/core/geo_big_polygon3.js index fdcd3f3b5cf..97dc5386fff 100644 --- a/jstests/core/geo_big_polygon3.js +++ b/jstests/core/geo_big_polygon3.js @@ -1,4 +1,10 @@ -// @tags: [does_not_support_stepdowns, requires_non_retryable_writes, requires_fastcount] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// requires_fastcount, +// requires_non_retryable_writes, +// ] // // Big Polygon edge cases diff --git a/jstests/core/geo_mapreduce.js b/jstests/core/geo_mapreduce.js index ea83d1e4474..8bf93d7fd81 100644 --- a/jstests/core/geo_mapreduce.js +++ b/jstests/core/geo_mapreduce.js @@ -1,4 +1,8 @@ -// @tags: [does_not_support_stepdowns] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] // Test script from SERVER-1742 diff --git a/jstests/core/geo_mapreduce2.js b/jstests/core/geo_mapreduce2.js index 9a62d5222d7..d7f73ce3d69 100644 --- a/jstests/core/geo_mapreduce2.js +++ b/jstests/core/geo_mapreduce2.js @@ -1,4 +1,8 @@ -// @tags: [does_not_support_stepdowns] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] // Geo mapreduce 2 from SERVER-3478 diff --git a/jstests/core/index_stats.js b/jstests/core/index_stats.js index 1834e7bb05f..fa5b723cb3d 100644 --- a/jstests/core/index_stats.js +++ b/jstests/core/index_stats.js @@ -1,12 +1,14 @@ // @tags: [ -// # Cannot implicitly shard accessed collections because of following errmsg: A single -// # update/delete on a sharded collection must contain an exact match on _id or contain the shard -// # key. -// assumes_unsharded_collection, // # This test attempts to perform write operations and get index usage statistics using the // # $indexStats stage. The former operation must be routed to the primary in a replica set, // # whereas the latter may be routed to a secondary. // assumes_read_preference_unchanged, +// # Cannot implicitly shard accessed collections because of following errmsg: A single +// # update/delete on a sharded collection must contain an exact match on _id or contain the shard +// # key. +// assumes_unsharded_collection, +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, // does_not_support_stepdowns, // requires_non_retryable_writes, // ] diff --git a/jstests/core/mr1.js b/jstests/core/mr1.js index 465706fbf8a..245059de523 100644 --- a/jstests/core/mr1.js +++ b/jstests/core/mr1.js @@ -1,4 +1,10 @@ -// @tags: [does_not_support_stepdowns, requires_getmore, requires_fastcount] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// requires_fastcount, +// requires_getmore, +// ] t = db.mr1; t.drop(); diff --git a/jstests/core/mr2.js b/jstests/core/mr2.js index 53da098835d..9b336a949c5 100644 --- a/jstests/core/mr2.js +++ b/jstests/core/mr2.js @@ -1,6 +1,11 @@ // Cannot implicitly shard accessed collections because the "jsMode" option to the "mapReduce" // command cannot be used on a sharded collection. -// @tags: [assumes_unsharded_collection, does_not_support_stepdowns] +// @tags: [ +// assumes_unsharded_collection, +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] t = db.mr2; t.drop(); diff --git a/jstests/core/mr3.js b/jstests/core/mr3.js index 3aaf43c12a5..19c8abbf419 100644 --- a/jstests/core/mr3.js +++ b/jstests/core/mr3.js @@ -1,4 +1,8 @@ -// @tags: [does_not_support_stepdowns] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] t = db.mr3; t.drop(); diff --git a/jstests/core/mr4.js b/jstests/core/mr4.js index 9db2d998604..987218ab926 100644 --- a/jstests/core/mr4.js +++ b/jstests/core/mr4.js @@ -1,4 +1,8 @@ -// @tags: [does_not_support_stepdowns] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] t = db.mr4; t.drop(); diff --git a/jstests/core/mr5.js b/jstests/core/mr5.js index b0bf29c4478..c78ce1d8f4e 100644 --- a/jstests/core/mr5.js +++ b/jstests/core/mr5.js @@ -1,4 +1,8 @@ -// @tags: [does_not_support_stepdowns] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] (function() { "use strict"; diff --git a/jstests/core/mr_bigobject.js b/jstests/core/mr_bigobject.js index 4b58661330f..513d48d25a2 100644 --- a/jstests/core/mr_bigobject.js +++ b/jstests/core/mr_bigobject.js @@ -1,4 +1,9 @@ -// @tags: [does_not_support_stepdowns, requires_fastcount] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// requires_fastcount, +// ] t = db.mr_bigobject; t.drop(); diff --git a/jstests/core/mr_bigobject_replace.js b/jstests/core/mr_bigobject_replace.js index 6b7c2724686..3c32e6de8af 100644 --- a/jstests/core/mr_bigobject_replace.js +++ b/jstests/core/mr_bigobject_replace.js @@ -1,6 +1,11 @@ // Cannot implicitly shard accessed collections because of following errmsg: Cannot output to a // non-sharded collection because sharded collection exists already. -// @tags: [assumes_unsharded_collection, does_not_support_stepdowns] +// @tags: [ +// assumes_unsharded_collection, +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] /** * Test that the server returns an error response for map-reduce operations that attempt to insert a diff --git a/jstests/core/mr_comments.js b/jstests/core/mr_comments.js index 127a7cb5da1..d5a797256b6 100644 --- a/jstests/core/mr_comments.js +++ b/jstests/core/mr_comments.js @@ -1,4 +1,8 @@ -// @tags: [does_not_support_stepdowns] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] t = db.mr_comments; t.drop(); diff --git a/jstests/core/mr_errorhandling.js b/jstests/core/mr_errorhandling.js index 2ded89baecf..47ba211e7af 100644 --- a/jstests/core/mr_errorhandling.js +++ b/jstests/core/mr_errorhandling.js @@ -1,6 +1,11 @@ // Cannot implicitly shard accessed collections because of collection existing when none // expected. -// @tags: [assumes_no_implicit_collection_creation_after_drop, does_not_support_stepdowns] +// @tags: [ +// assumes_no_implicit_collection_creation_after_drop, +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] t = db.mr_errorhandling; t.drop(); diff --git a/jstests/core/mr_index.js b/jstests/core/mr_index.js index de509a357cd..9c4a6edd21d 100644 --- a/jstests/core/mr_index.js +++ b/jstests/core/mr_index.js @@ -1,4 +1,8 @@ -// @tags: [does_not_support_stepdowns] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] t = db.mr_index; t.drop(); diff --git a/jstests/core/mr_index2.js b/jstests/core/mr_index2.js index c3e514e699b..6b85639a400 100644 --- a/jstests/core/mr_index2.js +++ b/jstests/core/mr_index2.js @@ -1,4 +1,8 @@ -// @tags: [does_not_support_stepdowns] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] t = db.mr_index2; t.drop(); diff --git a/jstests/core/mr_index3.js b/jstests/core/mr_index3.js index 64ab0d1aea8..51ed88b77a9 100644 --- a/jstests/core/mr_index3.js +++ b/jstests/core/mr_index3.js @@ -1,4 +1,8 @@ -// @tags: [does_not_support_stepdowns] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] t = db.mr_index3; t.drop(); diff --git a/jstests/core/mr_killop.js b/jstests/core/mr_killop.js index bbe7fe509fa..56a025dc4b5 100644 --- a/jstests/core/mr_killop.js +++ b/jstests/core/mr_killop.js @@ -3,6 +3,8 @@ // when the "finalize" option to the "mapReduce" command is used on a sharded collection. // @tags: [ // assumes_unsharded_collection, +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, // does_not_support_stepdowns, // uses_multiple_connections, // ] diff --git a/jstests/core/mr_merge.js b/jstests/core/mr_merge.js index 70445b89cf5..4402d1a56e1 100644 --- a/jstests/core/mr_merge.js +++ b/jstests/core/mr_merge.js @@ -1,6 +1,11 @@ // Cannot implicitly shard accessed collections because of following errmsg: Cannot output to a // non-sharded collection because sharded collection exists already. -// @tags: [assumes_unsharded_collection, does_not_support_stepdowns] +// @tags: [ +// assumes_unsharded_collection, +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] t = db.mr_merge; t.drop(); diff --git a/jstests/core/mr_merge2.js b/jstests/core/mr_merge2.js index 8bdc2a90216..b0ba9bcbaf0 100644 --- a/jstests/core/mr_merge2.js +++ b/jstests/core/mr_merge2.js @@ -1,6 +1,11 @@ // Cannot implicitly shard accessed collections because of following errmsg: Cannot output to a // non-sharded collection because sharded collection exists already. -// @tags: [assumes_unsharded_collection, does_not_support_stepdowns] +// @tags: [ +// assumes_unsharded_collection, +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] t = db.mr_merge2; t.drop(); diff --git a/jstests/core/mr_mutable_properties.js b/jstests/core/mr_mutable_properties.js index 8c9eacd6e97..25087a63a8e 100644 --- a/jstests/core/mr_mutable_properties.js +++ b/jstests/core/mr_mutable_properties.js @@ -1,4 +1,8 @@ -// @tags: [does_not_support_stepdowns] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] // See SERVER-9448 // Test argument and receiver (aka 'this') objects and their children can be mutated diff --git a/jstests/core/mr_optim.js b/jstests/core/mr_optim.js index d0fc6c8d371..d4d088f93e5 100644 --- a/jstests/core/mr_optim.js +++ b/jstests/core/mr_optim.js @@ -1,4 +1,9 @@ -// @tags: [does_not_support_stepdowns, requires_getmore] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// requires_getmore, +// ] t = db.mr_optim; t.drop(); diff --git a/jstests/core/mr_outreduce.js b/jstests/core/mr_outreduce.js index 089faf4a87b..daed8169800 100644 --- a/jstests/core/mr_outreduce.js +++ b/jstests/core/mr_outreduce.js @@ -1,6 +1,11 @@ // Cannot implicitly shard accessed collections because of following errmsg: Cannot output to a // non-sharded collection because sharded collection exists already. -// @tags: [assumes_unsharded_collection, does_not_support_stepdowns] +// @tags: [ +// assumes_unsharded_collection, +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] t = db.mr_outreduce; t.drop(); diff --git a/jstests/core/mr_outreduce2.js b/jstests/core/mr_outreduce2.js index 1a7a7597ff7..e849d5209bc 100644 --- a/jstests/core/mr_outreduce2.js +++ b/jstests/core/mr_outreduce2.js @@ -1,6 +1,11 @@ // Cannot implicitly shard accessed collections because of following errmsg: Cannot output to a // non-sharded collection because sharded collection exists already. -// @tags: [assumes_unsharded_collection, does_not_support_stepdowns] +// @tags: [ +// assumes_unsharded_collection, +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] normal = "mr_outreduce2"; out = normal + "_out"; diff --git a/jstests/core/mr_replaceIntoDB.js b/jstests/core/mr_replaceIntoDB.js index 2fbac26665b..536397e735e 100644 --- a/jstests/core/mr_replaceIntoDB.js +++ b/jstests/core/mr_replaceIntoDB.js @@ -1,4 +1,8 @@ -// @tags: [does_not_support_stepdowns] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] t = db.mr_replace; t.drop(); diff --git a/jstests/core/mr_sort.js b/jstests/core/mr_sort.js index d53b741f5c5..54465fa2d5d 100644 --- a/jstests/core/mr_sort.js +++ b/jstests/core/mr_sort.js @@ -1,6 +1,11 @@ // Cannot implicitly shard accessed collections because the "limit" option to the "mapReduce" // command cannot be used on a sharded collection. -// @tags: [assumes_unsharded_collection, does_not_support_stepdowns] +// @tags: [ +// assumes_unsharded_collection, +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] t = db.mr_sort; t.drop(); diff --git a/jstests/core/mr_stored.js b/jstests/core/mr_stored.js index 4d59da9df2b..e2c8d1450ec 100644 --- a/jstests/core/mr_stored.js +++ b/jstests/core/mr_stored.js @@ -1,6 +1,12 @@ // This test expects a function stored in the system.js collection to be available for a map/reduce, // which may not be the case if it is implicitly sharded in a passthrough. -// @tags: [does_not_support_stepdowns, requires_non_retryable_writes, assumes_unsharded_collection] +// @tags: [ +// assumes_unsharded_collection, +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// requires_non_retryable_writes, +// ] (function() { "use strict"; diff --git a/jstests/core/mr_tolerates_js_exception.js b/jstests/core/mr_tolerates_js_exception.js index 20e96eb47dc..29de4cf795d 100644 --- a/jstests/core/mr_tolerates_js_exception.js +++ b/jstests/core/mr_tolerates_js_exception.js @@ -3,6 +3,8 @@ * that the user gets back a JavaScript stacktrace. * * @tags: [ + * # mapReduce does not support afterClusterTime. + * does_not_support_causal_consistency, * does_not_support_stepdowns, * requires_scripting, * ] diff --git a/jstests/core/mr_undef.js b/jstests/core/mr_undef.js index 88e5936af4e..eae17573283 100644 --- a/jstests/core/mr_undef.js +++ b/jstests/core/mr_undef.js @@ -1,4 +1,8 @@ -// @tags: [does_not_support_stepdowns] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// ] t = db.mr_undef; t.drop(); diff --git a/jstests/core/opcounters_active.js b/jstests/core/opcounters_active.js index 283415ba8b1..25f91b9c41c 100644 --- a/jstests/core/opcounters_active.js +++ b/jstests/core/opcounters_active.js @@ -1,4 +1,8 @@ // checks that db.serverStatus will not throw errors when metrics tree is not present +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// ] (function() { "use strict"; diff --git a/jstests/core/or4.js b/jstests/core/or4.js index cd3c42424b9..02cae46c13e 100644 --- a/jstests/core/or4.js +++ b/jstests/core/or4.js @@ -1,8 +1,10 @@ // @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, // does_not_support_stepdowns, +// requires_fastcount, // requires_getmore, // requires_non_retryable_writes, -// requires_fastcount, // ] (function() { diff --git a/jstests/core/profile_mapreduce.js b/jstests/core/profile_mapreduce.js index 7d111779344..117689c4d73 100644 --- a/jstests/core/profile_mapreduce.js +++ b/jstests/core/profile_mapreduce.js @@ -1,4 +1,9 @@ -// @tags: [does_not_support_stepdowns, requires_profiling] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// requires_profiling, +// ] // Confirms that profiled findAndModify execution contains all expected metrics with proper values. diff --git a/jstests/core/recursion.js b/jstests/core/recursion.js index 6f6e5c906af..a7c8c9013f7 100644 --- a/jstests/core/recursion.js +++ b/jstests/core/recursion.js @@ -2,6 +2,8 @@ // the past. See SERVER-19614. // // @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, // does_not_support_stepdowns, // requires_non_retryable_commands, // ] diff --git a/jstests/core/system_profile.js b/jstests/core/system_profile.js index 2a805ba8942..812007c0334 100644 --- a/jstests/core/system_profile.js +++ b/jstests/core/system_profile.js @@ -1,9 +1,11 @@ // @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, // does_not_support_stepdowns, +// requires_capped, +// requires_collstats, // requires_non_retryable_commands, // requires_non_retryable_writes, -// requires_collstats, -// requires_capped, // ] // Test various user operations against "system.profile" collection. SERVER-18111. diff --git a/jstests/core/temp_cleanup.js b/jstests/core/temp_cleanup.js index f9e0432c890..200a82d2d13 100644 --- a/jstests/core/temp_cleanup.js +++ b/jstests/core/temp_cleanup.js @@ -1,4 +1,9 @@ -// @tags: [does_not_support_stepdowns, requires_fastcount] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// does_not_support_stepdowns, +// requires_fastcount, +// ] mydb = db.getSisterDB("temp_cleanup_test"); diff --git a/jstests/core/txns/commands_banning_txnnumber_outside_transactions.js b/jstests/core/txns/commands_banning_txnnumber_outside_transactions.js index ead98683049..f1048698b47 100644 --- a/jstests/core/txns/commands_banning_txnnumber_outside_transactions.js +++ b/jstests/core/txns/commands_banning_txnnumber_outside_transactions.js @@ -1,5 +1,9 @@ // Test that commands other than retryable writes may not use txnNumber outside transactions. -// @tags: [requires_document_locking] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// requires_document_locking, +// ] (function() { "use strict"; diff --git a/jstests/core/txns/commands_not_allowed_in_txn.js b/jstests/core/txns/commands_not_allowed_in_txn.js index 7300de24598..f23d6bb81c1 100644 --- a/jstests/core/txns/commands_not_allowed_in_txn.js +++ b/jstests/core/txns/commands_not_allowed_in_txn.js @@ -1,5 +1,10 @@ // Test commands that are not allowed in multi-document transactions. -// @tags: [uses_transactions, uses_snapshot_read_concern] +// @tags: [ +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, +// uses_snapshot_read_concern, +// uses_transactions, +// ] (function() { "use strict"; diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js index 916728d2c90..e172628b843 100644 --- a/jstests/core/views/views_all_commands.js +++ b/jstests/core/views/views_all_commands.js @@ -1,5 +1,7 @@ // @tags: [ // assumes_superuser_permissions, +// # mapReduce does not support afterClusterTime. +// does_not_support_causal_consistency, // does_not_support_stepdowns, // requires_fastcount, // requires_getmore, diff --git a/jstests/noPassthrough/shell_can_use_read_concern.js b/jstests/noPassthrough/shell_can_use_read_concern.js index 45e4755d8c9..183da4686ec 100644 --- a/jstests/noPassthrough/shell_can_use_read_concern.js +++ b/jstests/noPassthrough/shell_can_use_read_concern.js @@ -196,36 +196,6 @@ } // - // Tests for the "mapReduce" command. - // - - testCommandCanBeCausallyConsistent(function() { - const res = assert.commandWorked(db.runCommand({ - mapReduce: coll.getName(), - map: function() { - emit("x", 1); - }, - reduce: function(key, values) { - return values.length; - }, - out: {inline: 1} - })); - assert.eq([{_id: "x", value: 5}], res.results, tojson(res)); - }); - - testCommandCanBeCausallyConsistent(function() { - const res = coll.mapReduce( - function() { - emit("x", 1); - }, - function(key, values) { - return values.length; - }, - {out: {inline: 1}}); - assert.eq([{_id: "x", value: 5}], res.results, tojson(res)); - }); - - // // Tests for the "geoSearch" command. // diff --git a/jstests/replsets/prepare_conflict_read_concern_behavior.js b/jstests/replsets/prepare_conflict_read_concern_behavior.js index 4327f7d86c5..ebf714f5fb2 100644 --- a/jstests/replsets/prepare_conflict_read_concern_behavior.js +++ b/jstests/replsets/prepare_conflict_read_concern_behavior.js @@ -3,6 +3,13 @@ * and afterClusterTime reads are the only reads that should block on a prepared transaction. Reads * that happen as part of a write should also block on a prepared transaction. * + * Also test that dbHash and mapReduce, which acquire collection S locks for reads, do not block on + * a prepared transaction on secondaries. Otherwise, it would cause deadlocks when the prepared + * transaction reacquires locks (since locks were yielded on secondaries) at commit time. This test + * makes sure dbHash and mapReduce do not accept a non local read concern or afterClusterTime and so + * it is safe for the two commands to ignore prepare conflicts for reads. This test also makes sure + * mapReduce that does writes is not allowed to run on secondaries. + * * @tags: [uses_transactions, uses_prepare_transaction] */ @@ -10,7 +17,7 @@ "use strict"; load("jstests/core/txns/libs/prepare_helpers.js"); - const replTest = new ReplSetTest({nodes: 1}); + const replTest = new ReplSetTest({nodes: 2}); replTest.startSet(); replTest.initiate(); @@ -25,6 +32,9 @@ const testColl = testDB.getCollection(collName); const testColl2 = testDB.getCollection(collName2); + const secondary = replTest.getSecondary(); + const secondaryTestDB = secondary.getDB(dbName); + // Turn off timestamp reaping so that clusterTimeBeforePrepare doesn't get too old. assert.commandWorked(testDB.adminCommand({ configureFailPoint: "WTPreserveSnapshotHistoryIndefinitely", @@ -57,6 +67,34 @@ return res; }; + const dbHash = function(read_concern, timeout, db) { + let res = db.runCommand({ + dbHash: 1, + readConcern: read_concern, + maxTimeMS: timeout, + }); + + return res; + }; + + const mapReduce = function(read_concern, timeout, db, outOptions = {inline: 1}) { + let map = function() { + emit(this.a, this.a); + }; + let reduce = function(key, vals) { + return 1; + }; + let res = db.runCommand({ + mapReduce: collName, + map: map, + reduce: reduce, + out: outOptions, + readConcern: read_concern, + maxTimeMS: timeout, + }); + return res; + }; + assert.commandWorked( testColl.insert({_id: 1, in_prepared_txn: false}, {writeConcern: {w: "majority"}})); assert.commandWorked(testColl.insert({_id: 2, in_prepared_txn: false})); @@ -125,6 +163,77 @@ collName2, 1)); + // dbHash does not accept a non local read concern or afterClusterTime and it also sets + // ignore_prepare=true during its execution. Therefore, dbHash should never get prepare + // conflicts on secondaries. dbHash acquires collection S lock for reads and it will be + // blocked by a prepared transaction that writes to the same collection if it is run on + // primaries. + jsTestLog("Test dbHash doesn't support afterClusterTime read."); + assert.commandFailedWithCode( + dbHash({level: 'local', afterClusterTime: clusterTimeAfterPrepare}, + failureTimeout, + secondaryTestDB), + ErrorCodes.InvalidOptions); + + jsTestLog("Test dbHash doesn't support read concern other than local."); + assert.commandWorked(dbHash({level: 'local'}, successTimeout, secondaryTestDB)); + assert.commandFailedWithCode(dbHash({level: 'available'}, failureTimeout, secondaryTestDB), + ErrorCodes.InvalidOptions); + assert.commandFailedWithCode(dbHash({level: 'majority'}, failureTimeout, secondaryTestDB), + ErrorCodes.InvalidOptions); + assert.commandFailedWithCode(dbHash({level: 'snapshot'}, failureTimeout, secondaryTestDB), + ErrorCodes.InvalidOptions); + assert.commandFailedWithCode( + dbHash({level: 'linearizable'}, failureTimeout, secondaryTestDB), + ErrorCodes.InvalidOptions); + + jsTestLog("Test dbHash on secondary doesn't block on a prepared transaction."); + assert.commandWorked(dbHash({}, successTimeout, secondaryTestDB)); + jsTestLog("Test dbHash on primary blocks on collection S lock which conflicts with " + + "a prepared transaction."); + assert.commandFailedWithCode(dbHash({}, failureTimeout, testDB), + ErrorCodes.MaxTimeMSExpired); + + // mapReduce does not accept a non local read concern or afterClusterTime and it also sets + // ignore_prepare=true during its read phase. As mapReduce that writes is not allowed to run + // on secondaries, mapReduce should never get prepare conflicts on secondaries. mapReduce + // acquires collection S lock for reads and it will be blocked by a prepared transaction + // that writes to the same collection if it is run on primaries. + jsTestLog("Test mapReduce doesn't support afterClusterTime read."); + assert.commandFailedWithCode( + mapReduce({level: 'local', afterClusterTime: clusterTimeAfterPrepare}, + failureTimeout, + secondaryTestDB), + ErrorCodes.InvalidOptions); + + jsTestLog("Test mapReduce doesn't support read concern other than local."); + assert.commandWorked(mapReduce({level: 'local'}, successTimeout, secondaryTestDB)); + assert.commandFailedWithCode( + mapReduce({level: 'available'}, failureTimeout, secondaryTestDB), + ErrorCodes.InvalidOptions); + assert.commandFailedWithCode( + mapReduce({level: 'majority'}, failureTimeout, secondaryTestDB), + ErrorCodes.InvalidOptions); + assert.commandFailedWithCode( + mapReduce({level: 'snapshot'}, failureTimeout, secondaryTestDB), + ErrorCodes.InvalidOptions); + assert.commandFailedWithCode( + mapReduce({level: 'linearizable'}, failureTimeout, secondaryTestDB), + ErrorCodes.InvalidOptions); + + jsTestLog("Test mapReduce that writes is not allowed to run on secondaries."); + // It currently returns ErrorCodes.PrimarySteppedDown in this case. + assert.commandFailedWithCode(mapReduce({}, failureTimeout, secondaryTestDB, "outColl"), + [ErrorCodes.InvalidOptions, ErrorCodes.PrimarySteppedDown]); + + jsTestLog("Test mapReduce on secondary doesn't block on a prepared transaction."); + assert.commandWorked(mapReduce({}, successTimeout, secondaryTestDB)); + + jsTestLog("Test mapReduce on primary blocks on collection S lock which conflicts with " + + "a prepared transaction."); + assert.commandFailedWithCode(mapReduce({}, failureTimeout, testDB), + ErrorCodes.MaxTimeMSExpired); + jsTestLog("Test read from an update blocks on a prepared transaction."); assert.commandFailedWithCode(testDB.runCommand({ update: collName, @@ -145,7 +254,7 @@ jsTestLog("Test read with read concern 'snapshot' and a read timestamp after " + "prepareTimestamp on non-prepared documents doesn't block on a prepared " + "transaction."); - assert.commandWorked(read({}, failureTimeout, sessionDB2, collName2, 1)); + assert.commandWorked(read({}, successTimeout, sessionDB2, collName2, 1)); jsTestLog("Test read with read concern 'snapshot' and a read timestamp after " + "prepareTimestamp blocks on a prepared transaction."); diff --git a/src/mongo/db/commands.cpp b/src/mongo/db/commands.cpp index ea2641218d7..8efc03ab32d 100644 --- a/src/mongo/db/commands.cpp +++ b/src/mongo/db/commands.cpp @@ -616,6 +616,10 @@ private: return _command->allowsAfterClusterTime(cmdObj()); } + bool canIgnorePrepareConflicts() const override { + return _command->canIgnorePrepareConflicts(); + } + void doCheckAuthorization(OperationContext* opCtx) const override { uassertStatusOK(_command->checkAuthForOperation( opCtx, _request->getDatabase().toString(), _request->body)); diff --git a/src/mongo/db/commands/dbhash.cpp b/src/mongo/db/commands/dbhash.cpp index d93488bbacf..7e861b43abb 100644 --- a/src/mongo/db/commands/dbhash.cpp +++ b/src/mongo/db/commands/dbhash.cpp @@ -68,6 +68,14 @@ public: return false; } + bool allowsAfterClusterTime(const BSONObj& cmd) const override { + return false; + } + + bool canIgnorePrepareConflicts() const override { + return true; + } + ReadWriteType getReadWriteType() const override { return ReadWriteType::kRead; } diff --git a/src/mongo/db/commands/mr.cpp b/src/mongo/db/commands/mr.cpp index 083a682cbab..fb811ddc370 100644 --- a/src/mongo/db/commands/mr.cpp +++ b/src/mongo/db/commands/mr.cpp @@ -169,6 +169,9 @@ void assertCollectionNotNull(const NamespaceString& nss, AutoT& autoT) { void dropTempCollections(OperationContext* cleanupOpCtx, const NamespaceString& tempNamespace, const NamespaceString& incLong) { + // Make sure we enforce prepare conflicts before writing. + EnforcePrepareConflictsBlock enforcePrepare(cleanupOpCtx); + if (!tempNamespace.isEmpty()) { writeConflictRetry( cleanupOpCtx, @@ -499,6 +502,9 @@ void State::prepTempCollection() { if (!_onDisk) return; + // Make sure we enforce prepare conflicts before writing. + EnforcePrepareConflictsBlock enforcePrepare(_opCtx); + dropTempCollections( _opCtx, _config.tempNamespace, _useIncremental ? _config.incLong : NamespaceString()); @@ -715,6 +721,9 @@ long long State::postProcessCollection(OperationContext* opCtx, CurOp* curOp) { long long State::postProcessCollectionNonAtomic(OperationContext* opCtx, CurOp* curOp, bool callerHoldsGlobalLock) { + // Make sure we enforce prepare conflicts before writing. + EnforcePrepareConflictsBlock enforcePrepare(opCtx); + if (_config.outputOptions.finalNamespace == _config.tempNamespace) return collectionCount(opCtx, _config.outputOptions.finalNamespace, callerHoldsGlobalLock); @@ -806,6 +815,9 @@ long long State::postProcessCollectionNonAtomic(OperationContext* opCtx, void State::insert(const NamespaceString& nss, const BSONObj& o) { invariant(_onDisk); + // Make sure we enforce prepare conflicts before writing. + EnforcePrepareConflictsBlock enforcePrepare(_opCtx); + writeConflictRetry(_opCtx, "M/R insert", nss.ns(), [this, &nss, &o] { AutoGetCollection autoColl(_opCtx, nss, MODE_IX); uassert( @@ -845,6 +857,9 @@ void State::insert(const NamespaceString& nss, const BSONObj& o) { void State::_insertToInc(BSONObj& o) { verify(_onDisk); + // Make sure we enforce prepare conflicts before writing. + EnforcePrepareConflictsBlock enforcePrepare(_opCtx); + writeConflictRetry(_opCtx, "M/R insertToInc", _config.incLong.ns(), [this, &o] { AutoGetCollection autoColl(_opCtx, _config.incLong, MODE_IX); assertCollectionNotNull(_config.incLong, autoColl); @@ -1399,6 +1414,17 @@ public: return mrSupportsWriteConcern(cmd); } + bool allowsAfterClusterTime(const BSONObj& cmd) const override { + return false; + } + + bool canIgnorePrepareConflicts() const override { + // Map-Reduce is a special case for prepare conflicts. It may do writes to an output + // collection, but it enables enforcement of prepare conflicts before doing so. See use of + // EnforcePrepareConflictsBlock. + return true; + } + virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, std::vector<Privilege>* out) const { diff --git a/src/mongo/db/db_raii.h b/src/mongo/db/db_raii.h index e6293022bf7..c8ff0b6bef1 100644 --- a/src/mongo/db/db_raii.h +++ b/src/mongo/db/db_raii.h @@ -222,4 +222,40 @@ private: */ LockMode getLockModeForQuery(OperationContext* opCtx, const boost::optional<NamespaceString>& nss); +/** + * When in scope, enforces prepare conflicts in the storage engine. Reads and writes in this scope + * will block on accessing an already updated document which is in prepared state. And they will + * unblock after the prepared transaction that performed the update commits/aborts. + */ +class EnforcePrepareConflictsBlock { +public: + explicit EnforcePrepareConflictsBlock(OperationContext* opCtx) + : _opCtx(opCtx), _originalValue(opCtx->recoveryUnit()->getIgnorePrepared()) { + // It is illegal to call setIgnorePrepared() while any storage transaction is active. + // setIgnorePrepared() invariants that there is no active storage transaction. + _opCtx->recoveryUnit()->setIgnorePrepared(false); + } + + ~EnforcePrepareConflictsBlock() { + // If we are still holding locks, we might still have open storage transactions. However, we + // did not start with any active transactions when we first entered the scope. And + // transactions started within this scope cannot be reused outside of the scope. So we need + // to call abandonSnapshot() to close any open transactions on destruction. Any reads or + // writes should have already completed as we are exiting the scope. Therefore, this call is + // safe. + if (_opCtx->lockState()->isLocked()) { + _opCtx->recoveryUnit()->abandonSnapshot(); + } + // It is illegal to call setIgnorePrepared() while any storage transaction is active. There + // should not be any active transaction if we are not holding locks. If locks are still + // being held, the above abandonSnapshot() call should have already closed all storage + // transactions. + _opCtx->recoveryUnit()->setIgnorePrepared(_originalValue); + } + +private: + OperationContext* _opCtx; + bool _originalValue; +}; + } // namespace mongo diff --git a/src/mongo/db/introspect.cpp b/src/mongo/db/introspect.cpp index a82e67a0e6c..425d100f183 100644 --- a/src/mongo/db/introspect.cpp +++ b/src/mongo/db/introspect.cpp @@ -80,39 +80,6 @@ void _appendUserInfo(const CurOp& c, BSONObjBuilder& builder, AuthorizationSessi builder.append("user", bestUser.getUser().empty() ? "" : bestUser.getFullName()); } -/** - * When in scope, closes any active storage transactions and enforces prepare conflicts for reads. - * - * Locks must be held while this is in scope because both constructor and destructor access the - * storage engine. - */ -class EnforcePrepareConflictsBlock { -public: - explicit EnforcePrepareConflictsBlock(OperationContext* opCtx) - : _opCtx(opCtx), _originalValue(opCtx->recoveryUnit()->getIgnorePrepared()) { - dassert(_opCtx->lockState()->isLocked()); - dassert(!_opCtx->lockState()->inAWriteUnitOfWork()); - - // It is illegal to call setIgnorePrepared() while any storage transaction is active. This - // call is also harmless because any previous reads or writes should have already completed, - // as profile() is called at the end of an operation. - _opCtx->recoveryUnit()->abandonSnapshot(); - _opCtx->recoveryUnit()->setIgnorePrepared(false); - } - - ~EnforcePrepareConflictsBlock() { - dassert(_opCtx->lockState()->isLocked()); - dassert(!_opCtx->lockState()->inAWriteUnitOfWork()); - - _opCtx->recoveryUnit()->abandonSnapshot(); - _opCtx->recoveryUnit()->setIgnorePrepared(_originalValue); - } - -private: - OperationContext* _opCtx; - bool _originalValue; -}; - } // namespace @@ -188,6 +155,12 @@ void profile(OperationContext* opCtx, NetworkOp op) { Lock::CollectionLock collLock(opCtx, db->getProfilingNS(), MODE_IX); + // We are about to enforce prepare conflicts for the OperationContext. But it is illegal + // to change the behavior of ignoring prepare conflicts while any storage transaction is + // still active. So we need to call abandonSnapshot() to close any open transactions. + // This call is also harmless because any previous reads or writes should have already + // completed, as profile() is called at the end of an operation. + opCtx->recoveryUnit()->abandonSnapshot(); // The profiler performs writes even after read commands. Ignoring prepare conflicts is // not allowed while performing writes, so temporarily enforce prepare conflicts. EnforcePrepareConflictsBlock enforcePrepare(opCtx); diff --git a/src/mongo/db/pipeline/document_source_out.h b/src/mongo/db/pipeline/document_source_out.h index 674aea38657..ce9e4282b14 100644 --- a/src/mongo/db/pipeline/document_source_out.h +++ b/src/mongo/db/pipeline/document_source_out.h @@ -29,6 +29,7 @@ #pragma once +#include "mongo/db/db_raii.h" #include "mongo/db/pipeline/document_source.h" #include "mongo/db/pipeline/document_source_out_gen.h" #include "mongo/db/write_concern_options.h" @@ -46,23 +47,21 @@ class OutStageWriteBlock { OperationContext* _opCtx; repl::ReadConcernArgs _originalArgs; RecoveryUnit::ReadSource _originalSource; - bool _originalIgnorePrepared; + EnforcePrepareConflictsBlock _enforcePrepareConflictsBlock; public: - OutStageWriteBlock(OperationContext* opCtx) : _opCtx(opCtx) { + OutStageWriteBlock(OperationContext* opCtx) + : _opCtx(opCtx), _enforcePrepareConflictsBlock(opCtx) { _originalArgs = repl::ReadConcernArgs::get(_opCtx); _originalSource = _opCtx->recoveryUnit()->getTimestampReadSource(); - _originalIgnorePrepared = _opCtx->recoveryUnit()->getIgnorePrepared(); repl::ReadConcernArgs::get(_opCtx) = repl::ReadConcernArgs(); _opCtx->recoveryUnit()->setTimestampReadSource(RecoveryUnit::kUnset); - _opCtx->recoveryUnit()->setIgnorePrepared(false); } ~OutStageWriteBlock() { repl::ReadConcernArgs::get(_opCtx) = _originalArgs; _opCtx->recoveryUnit()->setTimestampReadSource(_originalSource); - _opCtx->recoveryUnit()->setIgnorePrepared(_originalIgnorePrepared); } }; |