diff options
author | Randolph Tan <randolph@10gen.com> | 2018-12-05 17:08:48 -0500 |
---|---|---|
committer | Randolph Tan <randolph@10gen.com> | 2019-01-04 16:17:30 -0500 |
commit | 62076130c3321f842f3ce4fb426aefc078074206 (patch) | |
tree | c7b2c756fc95bb18794ac91212d8fc386be00d4a | |
parent | e3eaddbf01d4c0939370b117a44b60069b68158e (diff) | |
download | mongo-62076130c3321f842f3ce4fb426aefc078074206.tar.gz |
SERVER-37853 Create sharded_core_txn with balancer suite
25 files changed, 615 insertions, 20 deletions
diff --git a/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml b/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml new file mode 100644 index 00000000000..8d8a2457b99 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml @@ -0,0 +1,450 @@ +test_kind: multi_stmt_txn_passthrough + +selector: + roots: + - jstests/core/**/*.js + exclude_files: + # These tests run in the jscore_txn passthrough suites. + - jstests/core/txns/**/*.js + + # The following tests fail because a certain command or functionality is not supported by + # mongos. This command or functionality is placed in a comment next to the failing test. + - jstests/core/apitest_db.js # serverStatus output doesn't have storageEngine. + - jstests/core/apitest_db_profile_level.js # profiling. + - jstests/core/apply_ops*.js # applyOps, SERVER-1439. + - jstests/core/awaitdata_getmore_cmd.js # capped collections. + - jstests/core/bypass_doc_validation.js # sharded $out output not permitted + - jstests/core/capped*.js # capped collections. + - jstests/core/check_shard_index.js # checkShardingIndex. + - jstests/core/collection_truncate.js # emptycapped. + - jstests/core/collmod_without_uuid.js # applyOps, SERVER-1439 + - jstests/core/compact_keeps_indexes.js # compact. + - jstests/core/currentop.js # uses fsync. + - jstests/core/dbadmin.js # "local" database. + - jstests/core/dbhash.js # dbhash. + - jstests/core/dbhash2.js # dbhash. + - jstests/core/diagdata.js # Command not supported in mongos + - jstests/core/do_txn*.js # doTxn + - jstests/core/dropdb_race.js # syncdelay. + - jstests/core/fsync.js # uses fsync. + - jstests/core/geo_haystack*.js # geoSearch. + - jstests/core/geo_s2cursorlimitskip.js # profiling. + - jstests/core/geo_update_btree2.js # notablescan. + - jstests/core/index_id_options.js # "local" database. + - jstests/core/index9.js # "local" database. + - jstests/core/max_time_ms.js # sleep, SERVER-2212. + - jstests/core/mr_replaceIntoDB.js # MapReduceResult, SERVER-20495. + - jstests/core/notablescan.js # notablescan. + - jstests/core/profile*.js # profiling. + - jstests/core/queryoptimizera.js # "local" database. + - jstests/core/rename*.js # renameCollection. + - jstests/core/stages*.js # stageDebug. + - jstests/core/startup_log.js # "local" database. + - jstests/core/tailable_cursor_invalidation.js # capped collections. + - jstests/core/tailable_getmore_batch_size.js # capped collections. + - jstests/core/tailable_skip_limit.js # capped collections. + - jstests/core/top.js # top. + # The following tests fail because mongos behaves differently from mongod when testing certain + # functionality. The differences are in a comment next to the failing test. + - jstests/core/explain_missing_database.js # Behavior with no db different on mongos, SERVER-18047. + - jstests/core/geo_2d_explain.js # executionSuccess in different spot in explain(). + - jstests/core/geo_s2explain.js # inputStage in different spot in explain(). + - jstests/core/geo_s2sparse.js # keysPerIndex in different spot in validate(). + - jstests/core/operation_latency_histogram.js # Stats are counted differently on mongos, SERVER-24880. + # Test expects to run against a standalone, but shards in this passthrough are repl sets. + - jstests/core/read_after_optime.js + # The following tests fail because they count indexes. These counts do not take into account the + # additional hashed shard key indexes that are automatically added by this passthrough. + - jstests/core/apitest_dbcollection.js + - jstests/core/bad_index_plugin.js + - jstests/core/create_indexes.js + - jstests/core/list_indexes_non_existent_ns.js + - jstests/core/rename6.js + # The following tests fail because they expect no databases to be created. However a DB is created + # automatically when we shard a collection. + - jstests/core/dbcase.js + - jstests/core/dbcase2.js + - jstests/core/no_db_created.js + - jstests/core/ns_length.js + - jstests/core/views/*.js # Views tests aren't expected to work when collections are implicitly sharded. + - jstests/core/killop_drop_collection.js # Uses fsyncLock. + # These tests fail because sharded clusters do not clean up correctly after failed index builds. + # See SERVER-33207 as an example. + - jstests/core/geo_borders.js + # TODO: Remove after fixing SERVER-32555. executionStats.nReturned is incorrect for sharded + # queries with a limit or for distinct commands. + - jstests/core/distinct_index1.js + - jstests/core/explain1.js + - jstests/core/explain4.js + - jstests/core/sortk.js + # TODO: Remove after fixing SERVER-32563. The implementation of explain for the count command is + # incorrect on sharded collections. + - jstests/core/explain_count.js + # TODO: Remove after fixing SERVER-14324. mapReduce behaves unpredictably when the out collection + # is sharded on anything other than {_id: 1}. + - jstests/core/mr_undef.js + # TODO SERVER-32311: These tests use plan stage helpers which can't handle sharded explain output. + - jstests/core/expr_index_use.js + - jstests/core/index_multikey.js + - jstests/core/optimized_match_explain.js + - jstests/core/sort_array.js + # TODO: SERVER-16605 MapReduce into sharded collection with hashed index fails + - jstests/core/mr1.js + - jstests/core/mr3.js + - jstests/core/mr4.js + # Create index command arbitrarily picks one of the errors if there are multiple errors from the + # shards. In this suite, it is possible that one of the shards don't have the collection created + # so it can return cannot implicitly create collection. + - jstests/core/index_create_too_many.js + - jstests/core/index_create_with_nul_in_name.js + + ## + ## Limitations with the way the runner file injects transactions. + ## + + # These tests expects some statements to error, which will cause txns to abort entirely. + - jstests/core/bulk_api_ordered.js + - jstests/core/bulk_api_unordered.js + - jstests/core/bulk_legacy_enforce_gle.js + - jstests/core/capped5.js + - jstests/core/commands_with_uuid.js + - jstests/core/explain_execution_error.js + - jstests/core/expr.js + - jstests/core/find9.js + - jstests/core/find_getmore_bsonsize.js + - jstests/core/find_getmore_cmd.js + - jstests/core/geo_allowedcomparisons.js + - jstests/core/geo_big_polygon2.js + - jstests/core/geonear_key.js + - jstests/core/in.js + - jstests/core/index8.js # No explicit check for failed command. + - jstests/core/index_decimal.js + - jstests/core/index_multiple_compatibility.js + - jstests/core/index_partial_write_ops.js + - jstests/core/indexa.js # No explicit check for failed command. + - jstests/core/indexes_multiple_commands.js + - jstests/core/js2.js + - jstests/core/json_schema/json_schema.js + - jstests/core/mr_bigobject.js + - jstests/core/not2.js + - jstests/core/notablescan.js + - jstests/core/null_query_semantics.js + - jstests/core/or1.js + - jstests/core/or2.js + - jstests/core/or3.js + - jstests/core/orj.js + - jstests/core/ref.js + - jstests/core/ref4.js + - jstests/core/regex_limit.js + - jstests/core/remove_undefined.js + - jstests/core/set7.js + - jstests/core/sortb.js + - jstests/core/sortf.js + - jstests/core/sortg.js + - jstests/core/sortj.js + - jstests/core/tailable_skip_limit.js + - jstests/core/type_array.js + - jstests/core/uniqueness.js + - jstests/core/unset2.js + - jstests/core/update_addToSet.js + - jstests/core/update_arrayFilters.js + - jstests/core/update_find_and_modify_id.js + - jstests/core/update_modifier_pop.js + - jstests/core/updateh.js + - jstests/core/updatej.js + + # SERVER-38096 Mongos opCounter doesn't increase when last op causes the transaction to implicitly abort + - jstests/core/opcounters_write_cmd.js + + # SERVER-36311 Remove txn_override.js restriction of snapshot readConcern + - jstests/core/read_after_optime.js + + # Consecutive writes totalling more than the size upper threshold will cause the txn to + # abort with a TransactionTooLarge error. + - jstests/core/batch_size.js + - jstests/core/single_batch.js + # For insert1.js, only Windows DEBUG builds are affected. We compute the in-memory size + # of the transaction, which uses the size of the oplog entry and is 8 bytes larger on + # Windows debug builds than other builds. + - jstests/core/insert1.js + + # TODO: SERVER-38207 Cannot insert document with MaxKey shard key. + - jstests/core/type8.js + + # TODO: SERVER-38326 mapReduce cleanup doesn't obey writeConcern, rendering + # txn_passthrough_cmd_massage.js ineffective for preventing 2 phase drops + # from contesting db/coll locks. + - jstests/core/function_string_representations.js + - jstests/core/geo_mapreduce2.js + - jstests/core/mr*.js + - jstests/core/or4.js + - jstests/core/temp_cleanup.js + + # TODO: SERVER-38570 txn_override.js does not check writeError field. + - jstests/core/update_setOnInsert.js + + # TODO: SERVER-38588 Hybrid index builds races with prepare in replication. + - jstests/core/background_index_multikey.js + + # TODO: SERVER-38686 createIndex should prioritize other errors over + # CannotImplicitlyCreateCollection + - jstests/core/fts_spanish.js + - jstests/core/geo_invalid_polygon.js + - jstests/core/geo_s2index.js + - jstests/core/wildcard_index_validindex.js + + # TODO: SERVER-38215 delx.js touches 2 different collections, which makes it susceptible for + # migration to happen in between and cause the transaction to abort due to StaleConfigException + # because it is not the first op of the transaction. + - jstests/core/delx.js + + # benchRun bypasses the txn_override.js so it doesn't run under the same transaction and can + # cause test to timeout because of the 3 hr transactions lifetime setting and moveChunk + # getting blocked trying to acquire collection X lock while doing chunk refresh. + - jstests/core/bench_test*.js + + ## + ## Some aggregation stages don't support snapshot readconcern. + ## + + # $explain (requires read concern local) + - jstests/core/agg_hint.js + - jstests/core/and.js + - jstests/core/collation.js + - jstests/core/explain_shell_helpers.js + - jstests/core/index_partial_read_ops.js + - jstests/core/optimized_match_explain.js + - jstests/core/sort_array.js + - jstests/core/views/views_collation.js + - jstests/core/wildcard_index_count.js + + # $out + - jstests/core/bypass_doc_validation.js + - jstests/core/views/views_aggregation.js + + # $listSessions + - jstests/core/list_all_local_sessions.js + - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js + - jstests/core/list_sessions.js + + # $indexStats + - jstests/core/index_stats.js + + # $collStats + - jstests/core/operation_latency_histogram.js + - jstests/core/views/views_coll_stats.js + - jstests/core/views/views_stats.js + + ## + ## WriteErrors get converted to WriteCommandErrors if part of a txn. + ## + + # The same error code, but with ok:0. + - jstests/core/json_schema/additional_items.js + - jstests/core/json_schema/additional_properties.js + - jstests/core/json_schema/bsontype.js + - jstests/core/json_schema/dependencies.js + - jstests/core/json_schema/items.js + - jstests/core/json_schema/logical_keywords.js + - jstests/core/json_schema/min_max_items.js + - jstests/core/json_schema/min_max_properties.js + - jstests/core/json_schema/pattern_properties.js + - jstests/core/json_schema/required.js + - jstests/core/json_schema/unique_items.js + + - jstests/core/field_name_validation.js + - jstests/core/fts_array.js + - jstests/core/inc-SERVER-7446.js + - jstests/core/invalid_db_name.js + - jstests/core/push_sort.js + + # Checks for "WriteErrors" explicitly from the response of db.runCommand() + - jstests/core/max_doc_size.js + + # Calls res.getWriteError() or res.hasWriteError(). + - jstests/core/bulk_api_ordered.js + - jstests/core/bulk_api_unordered.js + - jstests/core/bulk_legacy_enforce_gle.js + - jstests/core/cappeda.js + - jstests/core/doc_validation.js + - jstests/core/doc_validation_options.js + - jstests/core/geo_multinest0.js + - jstests/core/insert_illegal_doc.js + - jstests/core/ns_length.js + - jstests/core/push2.js + - jstests/core/remove6.js + - jstests/core/removeb.js + - jstests/core/rename4.js + - jstests/core/shell_writeconcern.js + - jstests/core/storefunc.js + - jstests/core/update_arrayFilters.js + - jstests/core/update_dbref.js + - jstests/core/updatel.js + - jstests/core/write_result.js + + # Multiple writes in a txn, some of which fail because the collection doesn't exist. + # We create the collection and retry the last write, but previous writes would have + # still failed. + - jstests/core/dbref1.js + - jstests/core/dbref2.js + - jstests/core/ref3.js + - jstests/core/update3.js + - jstests/core/rename3.js + + ## + ## Error: Unable to acquire lock within a max lock request timeout of '0ms' milliseconds + ## + + # Collection drops done through applyOps are not converted to w:majority + - jstests/core/views/invalid_system_views.js + + ## + ## Misc. reasons. + ## + + # SERVER-34868 Cannot run a legacy query on a session. + - jstests/core/exhaust.js + - jstests/core/validate_cmd_ns.js + + # SERVER-34772 Tailable Cursors are not allowed with snapshot readconcern. + - jstests/core/awaitdata_getmore_cmd.js + - jstests/core/getmore_cmd_maxtimems.js + - jstests/core/tailable_cursor_invalidation.js + - jstests/core/tailable_getmore_batch_size.js + + # SERVER-34918 The "max" option of a capped collection can be exceeded until the next insert. + # The reason is that we don't update the count of a collection until a transaction commits, + # by which point it is too late to complain that "max" has been exceeded. + - jstests/core/capped_max1.js + + # The "max" option of a capped collection can be temporarily exceeded before a + # txn is committed. + - jstests/core/bulk_insert_capped.js + + # Wrong count for top info (WriteLock) + - jstests/core/top.js + + # Expects collection to not have been created + - jstests/core/insert_id_undefined.js + + # Creates sessions explicitly, resulting in txns being run through different sessions + # using a single txnNumber. + - jstests/core/json_schema/misc_validation.js + - jstests/core/views/views_all_commands.js + + # Fails with implicit sessions because it will use multiple sessions on the same Mongo connection. + - jstests/core/dropdb.js + + # Committing a transaction when the server is fsync locked fails. + - jstests/core/fsync.js + + # Expects legacy errors ($err). + - jstests/core/constructors.js + + # txn interrupted by command outside of txn before getMore runs. + - jstests/core/commands_namespace_parsing.js + - jstests/core/drop3.js + - jstests/core/ensure_sorted.js + - jstests/core/geo_s2cursorlimitskip.js + - jstests/core/getmore_invalidated_cursors.js + - jstests/core/getmore_invalidated_documents.js + - jstests/core/kill_cursors.js + - jstests/core/list_collections1.js + - jstests/core/list_indexes.js + - jstests/core/max_time_ms.js + - jstests/core/oro.js + + # Expects certain number of operations in the system.profile collection. + - jstests/core/profile*.js + + # Parallel Shell - we do not signal the override to end a txn when a parallel shell closes. + - jstests/core/awaitdata_getmore_cmd.js + - jstests/core/compact_keeps_indexes.js + - jstests/core/count10.js + - jstests/core/count_plan_summary.js + - jstests/core/coveredIndex3.js + - jstests/core/currentop.js + - jstests/core/distinct3.js + - jstests/core/find_and_modify_concurrent_update.js + - jstests/core/fsync.js + - jstests/core/geo_update_btree.js + - jstests/core/killop_drop_collection.js + - jstests/core/loadserverscripts.js + - jstests/core/mr_killop.js + - jstests/core/queryoptimizer3.js + - jstests/core/remove9.js + - jstests/core/removeb.js + - jstests/core/removec.js + - jstests/core/shellstartparallel.js + - jstests/core/updatef.js + + # Command expects to see result from parallel operation. + # E.g. Suppose the following sequence of events: op1, join() op2 in parallel shell, op3. + # op3 will still be using the snapshot from op1, and not see op2 at all. + - jstests/core/cursora.js + - jstests/core/bench_test1.js + + exclude_with_any_tags: + # "Cowardly refusing to override read concern of command: ..." + - assumes_read_concern_unchanged + # "writeConcern is not allowed within a multi-statement transaction" + - assumes_write_concern_unchanged + - assumes_against_mongod_not_mongos + # Tests tagged with the following will fail because they assume collections are not sharded. + - assumes_no_implicit_collection_creation_after_drop + - assumes_no_implicit_index_creation + - assumes_unsharded_collection + - cannot_create_unique_index_when_using_hashed_shard_key + - assumes_balancer_off + - requires_fastcount + +executor: + config: + shell_options: + eval: >- + var testingReplication = true; + load('jstests/libs/override_methods/implicitly_shard_accessed_collections.js'); + ImplicitlyShardAccessCollSettings.setMode(ImplicitlyShardAccessCollSettings.Modes.kHashedMoveToSingleShard); + load('jstests/libs/override_methods/enable_sessions.js'); + load('jstests/libs/override_methods/txn_passthrough_cmd_massage.js'); + load('jstests/libs/txns/txn_override.js'); + global_vars: + TestData: + sessionOptions: + # Tests in this suite only read from primaries and only one node is electable, so causal + # consistency is not required to read your own writes. + causalConsistency: false + readMode: commands + hooks: + # The CheckReplDBHash hook waits until all operations have replicated to and have been applied + # on the secondaries, so we run the ValidateCollections hook after it to ensure we're + # validating the entire contents of the collection. + - class: CheckReplDBHash + - class: ValidateCollections + - class: DropShardedCollections + - class: CleanEveryN + n: 20 + fixture: + class: ShardedClusterFixture + mongos_options: + set_parameters: + enableTestCommands: 1 + mongod_options: + set_parameters: + logComponentVerbosity: + verbosity: 0 + command: 2 + replication: + election: 4 + heartbeats: 2 + initialSync: 2 + rollback: 2 + storage: + recovery: 2 + transaction: 4 + enableTestCommands: 1 + numInitialSyncAttempts: 1 + writePeriodicNoops: true + num_rs_nodes_per_shard: 2 + num_shards: 2 diff --git a/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py b/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py new file mode 100644 index 00000000000..90752571909 --- /dev/null +++ b/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py @@ -0,0 +1,22 @@ +"""Hook for cleaning up sharded collections created during tests.""" + +from __future__ import absolute_import + +import os.path + +from . import jsfile + + +class DropShardedCollections(jsfile.JSHook): + """Drops all sharded collections. + + With the exception of internal collections like config.system.sessions. + """ + + def __init__( # pylint: disable=super-init-not-called + self, hook_logger, fixture, shell_options=None): + """.""" + description = "Drop all sharded collections" + js_filename = os.path.join("jstests", "hooks", "drop_sharded_collections.js") + jsfile.JSHook.__init__( # pylint: disable=non-parent-init-called + self, hook_logger, fixture, js_filename, description, shell_options=shell_options) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 30ad3ddc457..b16ba792c16 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -6084,6 +6084,15 @@ tasks: vars: resmoke_args: --suites=multi_shard_local_read_write_multi_stmt_txn_jscore_passthrough --storageEngine=wiredTiger + name: multi_stmt_txn_jscore_passthrough_with_migration + depends_on: + - name: jsCore + commands: + - func: "do setup" + - func: "run tests" + vars: + resmoke_args: --suites=multi_stmt_txn_jscore_passthrough_with_migration --storageEngine=wiredTiger + - <<: *task_template name: parallel depends_on: @@ -8791,6 +8800,7 @@ buildvariants: - name: mongosTest - name: multi_shard_local_read_write_multi_stmt_txn_jscore_passthrough - name: multi_shard_multi_stmt_txn_jscore_passthrough + - name: multi_stmt_txn_jscore_passthrough_with_migration - name: noPassthrough - name: noPassthroughWithMongod - name: parallel @@ -10472,6 +10482,8 @@ buildvariants: - name: logical_session_cache_sharding_1sec_refresh_jscore_passthrough - name: logical_session_cache_standalone_1sec_refresh_jscore_passthrough - name: mongosTest + - name: multi_shard_multi_stmt_txn_jscore_passthrough + - name: multi_stmt_txn_jscore_passthrough_with_migration - name: noPassthrough - name: noPassthroughWithMongod - name: rollback_fuzzer_gen @@ -12124,6 +12136,9 @@ buildvariants: - name: multi_shard_multi_stmt_txn_jscore_passthrough distros: - rhel62-large + - name: multi_stmt_txn_jscore_passthrough_with_migration + distros: + - rhel62-large - name: multiversion_auth - name: multiversion - name: noPassthroughWithMongod diff --git a/jstests/core/batch_size.js b/jstests/core/batch_size.js index cdd60e77cd2..b280b8ceb77 100644 --- a/jstests/core/batch_size.js +++ b/jstests/core/batch_size.js @@ -1,4 +1,4 @@ -// @tags: [requires_getmore] +// @tags: [requires_getmore, assumes_balancer_off] // Test subtleties of batchSize and limit. diff --git a/jstests/core/distinct_compound_index.js b/jstests/core/distinct_compound_index.js index 446313fee7f..2af57ffa3bb 100644 --- a/jstests/core/distinct_compound_index.js +++ b/jstests/core/distinct_compound_index.js @@ -1,3 +1,4 @@ +// @tags [assumes_balancer_off] (function() { "use strict"; diff --git a/jstests/core/explain5.js b/jstests/core/explain5.js index 35841ac0789..03fcec4ddc6 100644 --- a/jstests/core/explain5.js +++ b/jstests/core/explain5.js @@ -1,4 +1,5 @@ // Check explain results for a plan that uses an index to obtain the requested sort order. +// @tags: [assumes_balancer_off] t = db.jstests_explain5; t.drop(); diff --git a/jstests/core/explain_execution_error.js b/jstests/core/explain_execution_error.js index ff999f48bb9..a59a4b7930f 100644 --- a/jstests/core/explain_execution_error.js +++ b/jstests/core/explain_execution_error.js @@ -1,4 +1,4 @@ -// @tags: [requires_getmore] +// @tags: [requires_getmore, assumes_balancer_off] // Test that even when the execution of a query fails, explain reports query // planner information. diff --git a/jstests/core/getmore_invalidated_cursors.js b/jstests/core/getmore_invalidated_cursors.js index 37f2a508013..788fb662cba 100644 --- a/jstests/core/getmore_invalidated_cursors.js +++ b/jstests/core/getmore_invalidated_cursors.js @@ -1,4 +1,5 @@ -// @tags: [requires_getmore, requires_non_retryable_commands, requires_collstats] +// @tags: [requires_getmore, requires_non_retryable_commands, requires_collstats, +// assumes_balancer_off] // Tests that running a getMore on a cursor that has been invalidated by something like a collection // drop will return an appropriate error message. diff --git a/jstests/core/index_check7.js b/jstests/core/index_check7.js index fda248db467..d02725616a8 100644 --- a/jstests/core/index_check7.js +++ b/jstests/core/index_check7.js @@ -1,3 +1,4 @@ +// @tags: [assumes_balancer_off] t = db.index_check7; t.drop(); diff --git a/jstests/core/index_diag.js b/jstests/core/index_diag.js index 3779d5f3c78..2064527ef72 100644 --- a/jstests/core/index_diag.js +++ b/jstests/core/index_diag.js @@ -1,3 +1,4 @@ +// @tags: [assumes_balancer_off] load("jstests/libs/fixture_helpers.js"); t = db.index_diag; diff --git a/jstests/core/min_max_bounds.js b/jstests/core/min_max_bounds.js index 1e4a80efbf6..713dc75b844 100644 --- a/jstests/core/min_max_bounds.js +++ b/jstests/core/min_max_bounds.js @@ -1,5 +1,6 @@ /** * This test ensures that queries using simple ranges handle bound inclusion properly. + * @tags: [assumes_balancer_off] */ (function() { 'use strict'; diff --git a/jstests/core/minmax.js b/jstests/core/minmax.js index 72653b23811..43a321d9c80 100644 --- a/jstests/core/minmax.js +++ b/jstests/core/minmax.js @@ -1,4 +1,5 @@ // Test min / max query parameters. +// @tags: [assumes_balancer_off] (function() { "use strict"; diff --git a/jstests/core/orf.js b/jstests/core/orf.js index 5d58e59c74f..66d943aa764 100644 --- a/jstests/core/orf.js +++ b/jstests/core/orf.js @@ -1,4 +1,5 @@ // Test a query with 200 $or clauses +// @tags: [assumes_balancer_off] t = db.jstests_orf; t.drop(); diff --git a/jstests/core/remove7.js b/jstests/core/remove7.js index 226a4cfb150..2cda8945089 100644 --- a/jstests/core/remove7.js +++ b/jstests/core/remove7.js @@ -25,7 +25,7 @@ for (i = 0; i < 200; i++) { t.save({tags: getTags(100)}); var q = {tags: {$in: getTags(10)}}; var before = t.find(q).count(); - var res = t.remove(q); + var res = assert.writeOK(t.remove(q)); var after = t.find(q).count(); assert.eq(0, after, "not zero after!"); assert.writeOK(res); diff --git a/jstests/core/wildcard_and_text_indexes.js b/jstests/core/wildcard_and_text_indexes.js index 4fd7ad34ac1..a4b552e3220 100644 --- a/jstests/core/wildcard_and_text_indexes.js +++ b/jstests/core/wildcard_and_text_indexes.js @@ -1,5 +1,6 @@ /** * Tests that a {$**: 1} index can coexist with a {$**: 'text'} index in the same collection. + * @tags: [assumes_balancer_off] */ (function() { "use strict"; diff --git a/jstests/core/wildcard_index_basic_index_bounds.js b/jstests/core/wildcard_index_basic_index_bounds.js index c5e1f87f7e1..e2ee7da8710 100644 --- a/jstests/core/wildcard_index_basic_index_bounds.js +++ b/jstests/core/wildcard_index_basic_index_bounds.js @@ -4,7 +4,7 @@ * Does not support stepdowns because the test issues getMores, which the stepdown/kill_primary * passthroughs will reject. * - * @tags: [does_not_support_stepdowns] + * @tags: [does_not_support_stepdowns, assumes_balancer_off] */ (function() { "use strict"; diff --git a/jstests/core/wildcard_index_cached_plans.js b/jstests/core/wildcard_index_cached_plans.js index d0c2a50abe9..96b8cfa4055 100644 --- a/jstests/core/wildcard_index_cached_plans.js +++ b/jstests/core/wildcard_index_cached_plans.js @@ -5,7 +5,7 @@ * $planCacheStats aggregation source. Both operations must be routed to the primary, and the latter * only supports 'local' readConcern. * @tags: [assumes_read_preference_unchanged, assumes_read_concern_unchanged, - * does_not_support_stepdowns] + * does_not_support_stepdowns, assumes_balancer_off] */ (function() { "use strict"; diff --git a/jstests/core/wildcard_index_multikey.js b/jstests/core/wildcard_index_multikey.js index 93e9492d6de..d9704963720 100644 --- a/jstests/core/wildcard_index_multikey.js +++ b/jstests/core/wildcard_index_multikey.js @@ -1,5 +1,6 @@ /** * Tests that queries using a multikey $** index, return correct results. + * @tags: [assumes_balancer_off] */ (function() { "use strict"; diff --git a/jstests/core/wildcard_index_nonblocking_sort.js b/jstests/core/wildcard_index_nonblocking_sort.js index fa400fb89e6..c21a0cacdb5 100644 --- a/jstests/core/wildcard_index_nonblocking_sort.js +++ b/jstests/core/wildcard_index_nonblocking_sort.js @@ -1,3 +1,4 @@ +// @tags: [assumes_balancer_off] (function() { "use strict"; diff --git a/jstests/hooks/drop_sharded_collections.js b/jstests/hooks/drop_sharded_collections.js new file mode 100644 index 00000000000..5758e3027e5 --- /dev/null +++ b/jstests/hooks/drop_sharded_collections.js @@ -0,0 +1,33 @@ +/** + * Drops all sharded collections (except for collections used internally, + * like config.system.sessions). + */ +(function() { + "use strict"; + + load("jstests/libs/fixture_helpers.js"); // For isMongos. + + assert.neq(typeof db, 'undefined', 'No `db` object, is the shell connected to a mongod?'); + assert(FixtureHelpers.isMongos(db), "not connected to mongos"); + + let balSettingResult = assert.commandWorked(db.adminCommand({balancerStatus: 1})); + if (balSettingResult.mode !== 'off') { + assert.commandWorked(db.adminCommand({balancerStop: 1})); + } + + db.getSiblingDB('config').collections.find().forEach(collEntry => { + if (collEntry._id !== 'config.system.sessions') { + let nsSplit = collEntry._id.split('.'); + const dbName = nsSplit.shift(); + const collName = nsSplit.join('.'); + + // Note: drop also cleans up tags and chunks associated with ns. + assert.commandWorked(db.getSiblingDB(dbName).runCommand({drop: collName})); + } + }); + + // Turn balancer back on if it was not off earlier. + if (balSettingResult.mode !== 'off') { + assert.commandWorked(db.adminCommand({balancerStart: 1})); + } +})(); diff --git a/jstests/libs/override_methods/implicitly_shard_accessed_collections.js b/jstests/libs/override_methods/implicitly_shard_accessed_collections.js index 93f3cbd3e15..a6cb5a6c2a0 100644 --- a/jstests/libs/override_methods/implicitly_shard_accessed_collections.js +++ b/jstests/libs/override_methods/implicitly_shard_accessed_collections.js @@ -9,6 +9,30 @@ * dropped in a sharded cluster. */ +/** + * Settings for the converting implictily accessed collections to sharded collections. + */ +const ImplicitlyShardAccessCollSettings = (function() { + let mode = 0; // Default to hashed shard key. + + return { + Modes: { + kUseHashedSharding: 0, + kHashedMoveToSingleShard: 1, + }, + setMode: function(newMode) { + if (newMode !== 0 && newMode !== 1) { + throw new Error("Cannot set mode to unknown mode: " + newMode); + } + + mode = newMode; + }, + getMode: function() { + return mode; + }, + }; +})(); + (function() { 'use strict'; @@ -31,6 +55,8 @@ /\.system\./, ]; + const kZoneName = 'moveToHereForMigrationPassthrough'; + function shardCollection(collection) { var db = collection.getDB(); var dbName = db.getName(); @@ -51,18 +77,45 @@ res = db.adminCommand( {shardCollection: fullName, key: {_id: 'hashed'}, collation: {locale: "simple"}}); - if (res.ok === 0 && testMayRunDropInParallel) { - // We ignore ConflictingOperationInProgress error responses from the - // "shardCollection" command if it's possible the test was running a "drop" command - // concurrently. We could retry running the "shardCollection" command, but tests - // that are likely to trigger this case are also likely running the "drop" command - // in a loop. We therefore just let the test continue with the collection being - // unsharded. - assert.commandFailedWithCode(res, ErrorCodes.ConflictingOperationInProgress); - print("collection '" + fullName + - "' failed to be sharded due to a concurrent drop operation"); - } else { - assert.commandWorked(res, "sharding '" + fullName + "' with a hashed _id key failed"); + + let checkResult = function(res, opDescription) { + if (res.ok === 0 && testMayRunDropInParallel) { + // We ignore ConflictingOperationInProgress error responses from the + // "shardCollection" command if it's possible the test was running a "drop" command + // concurrently. We could retry running the "shardCollection" command, but tests + // that are likely to trigger this case are also likely running the "drop" command + // in a loop. We therefore just let the test continue with the collection being + // unsharded. + assert.commandFailedWithCode(res, ErrorCodes.ConflictingOperationInProgress); + jsTest.log("Ignoring failure while " + opDescription + + " due to a concurrent drop operation: " + tojson(res)); + } else { + assert.commandWorked(res, opDescription + " failed"); + } + }; + + checkResult(res, 'shard ' + fullName); + + // Set the entire chunk range to a single zone, so balancer will be forced to move the + // evenly distributed chunks to a shard (selected at random). + if (res.ok === 1 && + ImplicitlyShardAccessCollSettings.getMode() === + ImplicitlyShardAccessCollSettings.Modes.kHashedMoveToSingleShard) { + let shardName = + db.getSiblingDB('config').shards.aggregate([{$sample: {size: 1}}]).toArray()[0]._id; + + checkResult(db.adminCommand({addShardToZone: shardName, zone: kZoneName}), + 'add ' + shardName + ' to zone ' + kZoneName); + checkResult(db.adminCommand({ + updateZoneKeyRange: fullName, + min: {_id: MinKey}, + max: {_id: MaxKey}, + zone: kZoneName + }), + 'set zone for ' + fullName); + + // Wake up the balancer. + checkResult(db.adminCommand({balancerStart: 1}), 'turn on balancer'); } } diff --git a/src/mongo/db/s/balancer/balancer.cpp b/src/mongo/db/s/balancer/balancer.cpp index 86a80ee5032..5e1f2ed131d 100644 --- a/src/mongo/db/s/balancer/balancer.cpp +++ b/src/mongo/db/s/balancer/balancer.cpp @@ -661,4 +661,9 @@ void Balancer::_splitOrMarkJumbo(OperationContext* opCtx, } } +void Balancer::notifyPersistedBalancerSettingsChanged() { + stdx::unique_lock<stdx::mutex> lock(_mutex); + _condVar.notify_all(); +} + } // namespace mongo diff --git a/src/mongo/db/s/balancer/balancer.h b/src/mongo/db/s/balancer/balancer.h index eb0b143b6bc..db11d7d0c64 100644 --- a/src/mongo/db/s/balancer/balancer.h +++ b/src/mongo/db/s/balancer/balancer.h @@ -144,6 +144,11 @@ public: */ void report(OperationContext* opCtx, BSONObjBuilder* builder); + /** + * Informs the balancer that a setting that affects it changed. + */ + void notifyPersistedBalancerSettingsChanged(); + private: /** * Possible runtime states of the balancer. The comments indicate the allowed next state. diff --git a/src/mongo/db/s/config/configsvr_control_balancer_command.cpp b/src/mongo/db/s/config/configsvr_control_balancer_command.cpp index 85e949d0d4e..b7bb313ece7 100644 --- a/src/mongo/db/s/config/configsvr_control_balancer_command.cpp +++ b/src/mongo/db/s/config/configsvr_control_balancer_command.cpp @@ -105,6 +105,7 @@ private: void _run(OperationContext* opCtx, BSONObjBuilder* result) override { uassertStatusOK(Grid::get(opCtx)->getBalancerConfiguration()->setBalancerMode( opCtx, BalancerSettingsType::kFull)); + Balancer::get(opCtx)->notifyPersistedBalancerSettingsChanged(); } }; @@ -121,6 +122,7 @@ private: uassertStatusOK(Grid::get(opCtx)->getBalancerConfiguration()->setBalancerMode( opCtx, BalancerSettingsType::kOff)); + Balancer::get(opCtx)->notifyPersistedBalancerSettingsChanged(); Balancer::get(opCtx)->joinCurrentRound(opCtx); } }; diff --git a/src/mongo/db/s/split_chunk.cpp b/src/mongo/db/s/split_chunk.cpp index b1cb6c5a93e..ddb7d5a9f53 100644 --- a/src/mongo/db/s/split_chunk.cpp +++ b/src/mongo/db/s/split_chunk.cpp @@ -136,13 +136,12 @@ StatusWith<boost::optional<ChunkRange>> splitChunk(OperationContext* opCtx, const std::string& shardName, const OID& expectedCollectionEpoch) { // - // Lock the collection's metadata and get highest version for the current shard // TODO(SERVER-25086): Remove distLock acquisition from split chunk // const std::string whyMessage( str::stream() << "splitting chunk " << chunkRange.toString() << " in " << nss.toString()); auto scopedDistLock = Grid::get(opCtx)->catalogClient()->getDistLockManager()->lock( - opCtx, nss.ns(), whyMessage, DistLockManager::kSingleLockAttemptTimeout); + opCtx, nss.ns(), whyMessage, DistLockManager::kDefaultLockTimeout); if (!scopedDistLock.isOK()) { return scopedDistLock.getStatus().withContext( str::stream() << "could not acquire collection lock for " << nss.toString() |