diff options
author | Marcos José Grillo Ramírez <marcos.grillo@10gen.com> | 2019-11-21 10:55:40 +0000 |
---|---|---|
committer | evergreen <evergreen@mongodb.com> | 2019-11-21 10:55:40 +0000 |
commit | f48da7a0f83762d214128799923e4bcede800dbe (patch) | |
tree | 4235e34ce3bd254e3513fef90996da252e4fc645 | |
parent | 57f2fc733c2db98f3cab55250f4d31458234836d (diff) | |
download | mongo-f48da7a0f83762d214128799923e4bcede800dbe.tar.gz |
SERVER-43990 A command to confirm that all chunks have been moved to the right zone after the initial split
28 files changed, 683 insertions, 50 deletions
diff --git a/buildscripts/resmokeconfig/suites/sharding_misc.yml b/buildscripts/resmokeconfig/suites/sharding_misc.yml index 662b8fe7dde..321e15c9128 100644 --- a/buildscripts/resmokeconfig/suites/sharding_misc.yml +++ b/buildscripts/resmokeconfig/suites/sharding_misc.yml @@ -347,6 +347,7 @@ selector: - jstests/sharding/sharding_options.js - jstests/sharding/advance_cluster_time_action_type.js - jstests/sharding/mongos_wait_csrs_initiate.js + - jstests/sharding/balancer_collection_status.js - jstests/sharding/config_rs_change.js - jstests/sharding/current_op_no_shards.js - jstests/sharding/movechunk_include.js diff --git a/jstests/auth/lib/commands_lib.js b/jstests/auth/lib/commands_lib.js index 408b1fdbea2..4f1f1d88899 100644 --- a/jstests/auth/lib/commands_lib.js +++ b/jstests/auth/lib/commands_lib.js @@ -5664,6 +5664,22 @@ var authCommandsLib = { }, ] }, + { + testname: "balancerCollectionStatus", + command: {shardCollection: "test.x"}, + skipUnlessSharded: true, + testcases: [ + { + runOnDb: adminDbName, + roles: Object.extend({enableSharding: 1}, roles_clusterManager), + privileges: + [{resource: {db: "test", collection: "x"}, actions: ["enableSharding"]}], + expectFail: true + }, + {runOnDb: firstDbName, roles: {}}, + {runOnDb: secondDbName, roles: {}} + ] + }, ], /************* SHARED TEST LOGIC ****************/ diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js index 20fc61b2fdd..9a7e4f778a8 100644 --- a/jstests/core/views/views_all_commands.js +++ b/jstests/core/views/views_all_commands.js @@ -75,6 +75,7 @@ let viewsCommandTests = { _cloneCollectionOptionsFromPrimaryShard: {skip: isAnInternalCommand}, _configsvrAddShard: {skip: isAnInternalCommand}, _configsvrAddShardToZone: {skip: isAnInternalCommand}, + _configsvrBalancerCollectionStatus: {skip: isAnInternalCommand}, _configsvrBalancerStart: {skip: isAnInternalCommand}, _configsvrBalancerStatus: {skip: isAnInternalCommand}, _configsvrBalancerStop: {skip: isAnInternalCommand}, @@ -128,6 +129,16 @@ let viewsCommandTests = { }, authenticate: {skip: isUnrelated}, availableQueryOptions: {skip: isAnInternalCommand}, + balancerCollectionStatus: { + command: {balancerCollectionStatus: "test.view"}, + setup: function(conn) { + assert.commandWorked(conn.adminCommand({enableSharding: "test"})); + }, + skipStandalone: true, + expectFailure: true, + isAdminCommand: true, + expectedErrorCode: ErrorCodes.NamespaceNotSharded, + }, balancerStart: {skip: isUnrelated}, balancerStatus: {skip: isUnrelated}, balancerStop: {skip: isUnrelated}, diff --git a/jstests/sharding/balancer_collection_status.js b/jstests/sharding/balancer_collection_status.js new file mode 100644 index 00000000000..e1cbb8f48c0 --- /dev/null +++ b/jstests/sharding/balancer_collection_status.js @@ -0,0 +1,102 @@ +/** + * Test the balancerCollectionStatus command and its possible outputs + * + * Remove requires_fcv_44 tag if SERVER-43990 is backported or 4.4 becomes last-stable. + * @tags: [requires_fcv_44] + */ + +(function() { +'use strict'; + +var st = new ShardingTest({mongos: 1, shards: 3}); + +function runBalancer(rounds) { + st.startBalancer(); + let numRounds = 0; + + // Let the balancer run for caller specified number of rounds. + assert.soon(() => { + st.awaitBalancerRound(); + st.printShardingStatus(true); + numRounds++; + return (numRounds === rounds); + }, 'Balancer failed to run for ' + rounds + ' rounds', 1000 * 60 * (3 * rounds)); + + st.stopBalancer(); +} + +// only fully quilified namespaces are allowed on the command +assert.commandFailedWithCode(st.s0.adminCommand({balancerCollectionStatus: 'db'}), + ErrorCodes.InvalidNamespace); + +// only sharded databases are allowed +assert.commandFailedWithCode(st.s0.adminCommand({balancerCollectionStatus: 'db.col'}), + ErrorCodes.NamespaceNotFound); + +// setup the collection for the test +assert.commandWorked(st.s0.adminCommand({enableSharding: 'db'})); +assert.commandWorked(st.s0.adminCommand({shardCollection: 'db.col', key: {key: 1}})); + +// only sharded collections are allowed +assert.commandFailedWithCode(st.s0.adminCommand({balancerCollectionStatus: 'db.col2'}), + ErrorCodes.NamespaceNotSharded); + +var result = assert.commandWorked(st.s0.adminCommand({balancerCollectionStatus: 'db.col'})); + +// new collections must be balanced +assert.eq(result.status, 'balanced'); + +// get shardIds +var shards = st.s0.getDB('config').shards.find().toArray(); + +// manually split and place the 3 chunks on the same shard +assert.commandWorked(st.s0.adminCommand({split: 'db.col', middle: {key: 10}})); +assert.commandWorked(st.s0.adminCommand({split: 'db.col', middle: {key: 20}})); +assert.commandWorked(st.s0.adminCommand({moveChunk: 'db.col', find: {key: 0}, to: shards[0]._id})); +assert.commandWorked(st.s0.adminCommand({moveChunk: 'db.col', find: {key: 10}, to: shards[0]._id})); +assert.commandWorked(st.s0.adminCommand({moveChunk: 'db.col', find: {key: 20}, to: shards[0]._id})); + +// check the current status +result = assert.commandWorked(st.s0.adminCommand({balancerCollectionStatus: 'db.col'})); + +// chunksImbalanced expected +assert.eq(result.status, 'chunksImbalance'); + +// run balancer with 3 rounds +runBalancer(3); + +// the chunks must be balanced now +result = assert.commandWorked(st.s0.adminCommand({balancerCollectionStatus: 'db.col'})); + +assert.eq(result.status, 'balanced'); + +// manually move a chunk to a shard before creating zones (this will help +// testing the zone violation) +assert.commandWorked(st.s0.adminCommand({moveChunk: 'db.col', find: {key: 10}, to: shards[2]._id})); + +// create zones on first two shards only +assert.commandWorked(st.s0.adminCommand({addShardToZone: shards[0]._id, zone: 'zone0'})); +assert.commandWorked(st.s0.adminCommand( + {updateZoneKeyRange: 'db.col', min: {key: MinKey}, max: {key: 10}, zone: 'zone0'})); + +assert.commandWorked(st.s0.adminCommand({addShardToZone: shards[1]._id, zone: 'zone1'})); +assert.commandWorked(st.s0.adminCommand( + {updateZoneKeyRange: 'db.col', min: {key: 10}, max: {key: 20}, zone: 'zone1'})); + +result = assert.commandWorked(st.s0.adminCommand({balancerCollectionStatus: 'db.col'})); + +// having a chunk on a different zone will cause a zone violation +assert.eq(result.status, 'zoneViolation'); + +// run balancer, we don't know exactly where the first run moved the chunks +// so lets run 3 rounds just in case +runBalancer(3); + +// the chunks must be balanced now +result = assert.commandWorked(st.s0.adminCommand({balancerCollectionStatus: 'db.col'})); + +// final check: all chunks are balanced and in the correct zone +assert.eq(result.status, 'balanced'); + +st.stop(); +})();
\ No newline at end of file diff --git a/jstests/sharding/database_versioning_all_commands.js b/jstests/sharding/database_versioning_all_commands.js index 10754c7f8ef..5ce6d28e2a2 100644 --- a/jstests/sharding/database_versioning_all_commands.js +++ b/jstests/sharding/database_versioning_all_commands.js @@ -252,6 +252,7 @@ let testCases = { }, authenticate: {skip: "does not forward command to primary shard"}, availableQueryOptions: {skip: "executes locally on mongos (not sent to any remote node)"}, + balancerCollectionStatus: {skip: "does not forward command to primary shard"}, balancerStart: {skip: "not on a user database"}, balancerStatus: {skip: "not on a user database"}, balancerStop: {skip: "not on a user database"}, diff --git a/jstests/sharding/safe_secondary_reads_drop_recreate.js b/jstests/sharding/safe_secondary_reads_drop_recreate.js index aa08f7a6732..b7455ba3de1 100644 --- a/jstests/sharding/safe_secondary_reads_drop_recreate.js +++ b/jstests/sharding/safe_secondary_reads_drop_recreate.js @@ -41,6 +41,7 @@ let testCases = { _shardsvrCloneCatalogData: {skip: "primary only"}, _configsvrAddShard: {skip: "primary only"}, _configsvrAddShardToZone: {skip: "primary only"}, + _configsvrBalancerCollectionStatus: {skip: "primary only"}, _configsvrBalancerStart: {skip: "primary only"}, _configsvrBalancerStatus: {skip: "primary only"}, _configsvrBalancerStop: {skip: "primary only"}, @@ -87,6 +88,7 @@ let testCases = { authSchemaUpgrade: {skip: "primary only"}, authenticate: {skip: "does not return user data"}, availableQueryOptions: {skip: "does not return user data"}, + balancerCollectionStatus: {skip: "primary only"}, balancerStart: {skip: "primary only"}, balancerStatus: {skip: "primary only"}, balancerStop: {skip: "primary only"}, diff --git a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js index aa47096339a..96f2efd2978 100644 --- a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js +++ b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js @@ -48,6 +48,7 @@ let testCases = { _shardsvrCloneCatalogData: {skip: "primary only"}, _configsvrAddShard: {skip: "primary only"}, _configsvrAddShardToZone: {skip: "primary only"}, + _configsvrBalancerCollectionStatus: {skip: "primary only"}, _configsvrBalancerStart: {skip: "primary only"}, _configsvrBalancerStatus: {skip: "primary only"}, _configsvrBalancerStop: {skip: "primary only"}, @@ -100,6 +101,7 @@ let testCases = { authSchemaUpgrade: {skip: "primary only"}, authenticate: {skip: "does not return user data"}, availableQueryOptions: {skip: "does not return user data"}, + balancerCollectionStatus: {skip: "primary only"}, balancerStart: {skip: "primary only"}, balancerStatus: {skip: "primary only"}, balancerStop: {skip: "primary only"}, diff --git a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js index efa99b0c6ad..d310da83d9e 100644 --- a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js +++ b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js @@ -41,6 +41,7 @@ let testCases = { _shardsvrCloneCatalogData: {skip: "primary only"}, _configsvrAddShard: {skip: "primary only"}, _configsvrAddShardToZone: {skip: "primary only"}, + _configsvrBalancerCollectionStatus: {skip: "primary only"}, _configsvrBalancerStart: {skip: "primary only"}, _configsvrBalancerStatus: {skip: "primary only"}, _configsvrBalancerStop: {skip: "primary only"}, @@ -88,6 +89,7 @@ let testCases = { authenticate: {skip: "does not return user data"}, authSchemaUpgrade: {skip: "primary only"}, availableQueryOptions: {skip: "does not return user data"}, + balancerCollectionStatus: {skip: "primary only"}, balancerStart: {skip: "primary only"}, balancerStatus: {skip: "primary only"}, balancerStop: {skip: "primary only"}, diff --git a/jstests/sharding/track_unsharded_collections_check_shard_version.js b/jstests/sharding/track_unsharded_collections_check_shard_version.js index 3020d78a397..9ff6e225d6b 100644 --- a/jstests/sharding/track_unsharded_collections_check_shard_version.js +++ b/jstests/sharding/track_unsharded_collections_check_shard_version.js @@ -72,6 +72,7 @@ let testCases = { }, authenticate: {skip: "does not forward command to primary shard"}, availableQueryOptions: {skip: "executes locally on mongos (not sent to any remote node)"}, + balancerCollectionStatus: {skip: "does not forward command to primary shard"}, balancerStart: {skip: "not on a user database"}, balancerStatus: {skip: "not on a user database"}, balancerStop: {skip: "not on a user database"}, diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript index e584d103fd3..5db7844a060 100644 --- a/src/mongo/db/s/SConscript +++ b/src/mongo/db/s/SConscript @@ -263,6 +263,7 @@ env.Library( 'clone_collection_options_from_primary_shard_cmd.cpp', 'config/configsvr_add_shard_command.cpp', 'config/configsvr_add_shard_to_zone_command.cpp', + 'config/configsvr_balancer_collection_status_command.cpp', 'config/configsvr_clear_jumbo_flag_command.cpp', 'config/configsvr_commit_chunk_migration_command.cpp', 'config/configsvr_commit_move_primary_command.cpp', diff --git a/src/mongo/db/s/balancer/balancer.cpp b/src/mongo/db/s/balancer/balancer.cpp index 67bcb23220f..63785789aa3 100644 --- a/src/mongo/db/s/balancer/balancer.cpp +++ b/src/mongo/db/s/balancer/balancer.cpp @@ -51,6 +51,7 @@ #include "mongo/s/catalog_cache.h" #include "mongo/s/client/shard_registry.h" #include "mongo/s/grid.h" +#include "mongo/s/request_types/balancer_collection_status_gen.h" #include "mongo/s/shard_util.h" #include "mongo/util/concurrency/idle_thread_block.h" #include "mongo/util/exit.h" @@ -80,6 +81,14 @@ const Seconds kShortBalanceRoundInterval(1); const auto getBalancer = ServiceContext::declareDecoration<std::unique_ptr<Balancer>>(); /** + * Balancer status response + */ +static constexpr StringData kBalancerPolicyStatusBalanced = "balanced"_sd; +static constexpr StringData kBalancerPolicyStatusDraining = "draining"_sd; +static constexpr StringData kBalancerPolicyStatusZoneViolation = "zoneViolation"_sd; +static constexpr StringData kBalancerPolicyStatusChunksImbalance = "chunksImbalance"_sd; + +/** * Utility class to generate timing and statistics for a single balancer round. */ class BalanceRoundDetails { @@ -285,7 +294,8 @@ Status Balancer::moveSingleChunk(OperationContext* opCtx, MigrateInfo(newShardId, chunk, forceJumbo ? MoveChunkRequest::ForceJumbo::kForceManual - : MoveChunkRequest::ForceJumbo::kDoNotForce), + : MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance), maxChunkSizeBytes, secondaryThrottle, waitForDelete); @@ -690,4 +700,27 @@ void Balancer::notifyPersistedBalancerSettingsChanged() { _condVar.notify_all(); } +StringData Balancer::getBalancerStatusForNs(OperationContext* opCtx, const NamespaceString& ns) { + auto splitChunks = uassertStatusOK(_chunkSelectionPolicy->selectChunksToSplit(opCtx, ns)); + if (!splitChunks.empty()) { + return kBalancerPolicyStatusZoneViolation; + } + auto chunksToMove = uassertStatusOK(_chunkSelectionPolicy->selectChunksToMove(opCtx, ns)); + if (chunksToMove.empty()) { + return kBalancerPolicyStatusBalanced; + } + const auto& migrationInfo = chunksToMove.front(); + + switch (migrationInfo.reason) { + case MigrateInfo::drain: + return kBalancerPolicyStatusDraining; + case MigrateInfo::zoneViolation: + return kBalancerPolicyStatusZoneViolation; + case MigrateInfo::chunksImbalance: + return kBalancerPolicyStatusChunksImbalance; + } + + return kBalancerPolicyStatusBalanced; +} + } // namespace mongo diff --git a/src/mongo/db/s/balancer/balancer.h b/src/mongo/db/s/balancer/balancer.h index b4d8c9e02a0..92ed1aea2c0 100644 --- a/src/mongo/db/s/balancer/balancer.h +++ b/src/mongo/db/s/balancer/balancer.h @@ -149,6 +149,12 @@ public: */ void notifyPersistedBalancerSettingsChanged(); + /** + * Returns if a given collection is draining due to a removed shard, has chunks on an invalid + * zone or the number of chunks is imbalanced across the cluster + */ + StringData getBalancerStatusForNs(OperationContext* opCtx, const NamespaceString& nss); + private: /** * Possible runtime states of the balancer. The comments indicate the allowed next state. diff --git a/src/mongo/db/s/balancer/balancer_chunk_selection_policy.h b/src/mongo/db/s/balancer/balancer_chunk_selection_policy.h index 2557cebd602..79a539c3599 100644 --- a/src/mongo/db/s/balancer/balancer_chunk_selection_policy.h +++ b/src/mongo/db/s/balancer/balancer_chunk_selection_policy.h @@ -91,11 +91,25 @@ public: virtual StatusWith<SplitInfoVector> selectChunksToSplit(OperationContext* opCtx) = 0; /** + * Given a valid namespace returns all the Migrations the balancer would need to perform + * with the current state + */ + virtual StatusWith<SplitInfoVector> selectChunksToSplit(OperationContext* opCtx, + const NamespaceString& nss) = 0; + + /** * Potentially blocking method, which gives out a set of chunks to be moved. */ virtual StatusWith<MigrateInfoVector> selectChunksToMove(OperationContext* opCtx) = 0; /** + * Given a valid namespace returns all the Migrations the balancer would need to perform + * with the current state + */ + virtual StatusWith<MigrateInfoVector> selectChunksToMove(OperationContext* opCtx, + const NamespaceString& nss) = 0; + + /** * Requests a single chunk to be relocated to a different shard, if possible. If some error * occurs while trying to determine the best location for the chunk, a failed status is * returned. If the chunk is already at the best shard that it can be, returns boost::none. diff --git a/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.cpp b/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.cpp index 4e24a95ea91..f387556b02b 100644 --- a/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.cpp +++ b/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.cpp @@ -189,7 +189,7 @@ StatusWith<SplitInfoVector> BalancerChunkSelectionPolicyImpl::selectChunksToSpli return shardStatsStatus.getStatus(); } - const auto shardStats = std::move(shardStatsStatus.getValue()); + const auto& shardStats = shardStatsStatus.getValue(); auto swCollections = Grid::get(opCtx)->catalogClient()->getCollections(opCtx, nullptr, nullptr); if (!swCollections.isOK()) { @@ -231,6 +231,19 @@ StatusWith<SplitInfoVector> BalancerChunkSelectionPolicyImpl::selectChunksToSpli return splitCandidates; } +StatusWith<SplitInfoVector> BalancerChunkSelectionPolicyImpl::selectChunksToSplit( + OperationContext* opCtx, const NamespaceString& nss) { + + auto shardStatsStatus = _clusterStats->getStats(opCtx); + if (!shardStatsStatus.isOK()) { + return shardStatsStatus.getStatus(); + } + + const auto& shardStats = shardStatsStatus.getValue(); + + return _getSplitCandidatesForCollection(opCtx, nss, shardStats); +} + StatusWith<MigrateInfoVector> BalancerChunkSelectionPolicyImpl::selectChunksToMove( OperationContext* opCtx) { auto shardStatsStatus = _clusterStats->getStats(opCtx); @@ -238,7 +251,7 @@ StatusWith<MigrateInfoVector> BalancerChunkSelectionPolicyImpl::selectChunksToMo return shardStatsStatus.getStatus(); } - const auto shardStats = std::move(shardStatsStatus.getValue()); + const auto& shardStats = shardStatsStatus.getValue(); if (shardStats.size() < 2) { return MigrateInfoVector{}; @@ -292,6 +305,38 @@ StatusWith<MigrateInfoVector> BalancerChunkSelectionPolicyImpl::selectChunksToMo return candidateChunks; } +StatusWith<MigrateInfoVector> BalancerChunkSelectionPolicyImpl::selectChunksToMove( + OperationContext* opCtx, const NamespaceString& nss) { + auto shardStatsStatus = _clusterStats->getStats(opCtx); + if (!shardStatsStatus.isOK()) { + return shardStatsStatus.getStatus(); + } + + const auto& shardStats = shardStatsStatus.getValue(); + + // Validate collection information + auto swCollection = Grid::get(opCtx)->catalogClient()->getCollection(opCtx, nss); + if (!swCollection.isOK()) { + return swCollection.getStatus(); + } + + const auto& collection = swCollection.getValue().value; + + if (collection.getDropped()) { + return Status(ErrorCodes::NamespaceNotFound, + str::stream() << "collection " << nss.ns() << " not found"); + } + + std::set<ShardId> usedShards; + + auto candidatesStatus = _getMigrateCandidatesForCollection(opCtx, nss, shardStats, &usedShards); + if (!candidatesStatus.isOK()) { + return candidatesStatus.getStatus(); + } + + return candidatesStatus; +} + StatusWith<boost::optional<MigrateInfo>> BalancerChunkSelectionPolicyImpl::selectSpecificChunkToMove(OperationContext* opCtx, const ChunkType& chunk) { diff --git a/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.h b/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.h index c5a0d3fe07c..4262ff63dd0 100644 --- a/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.h +++ b/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.h @@ -43,8 +43,14 @@ public: StatusWith<SplitInfoVector> selectChunksToSplit(OperationContext* opCtx) override; + StatusWith<SplitInfoVector> selectChunksToSplit(OperationContext* opCtx, + const NamespaceString& ns) override; + StatusWith<MigrateInfoVector> selectChunksToMove(OperationContext* opCtx) override; + StatusWith<MigrateInfoVector> selectChunksToMove(OperationContext* opCtx, + const NamespaceString& ns) override; + StatusWith<boost::optional<MigrateInfo>> selectSpecificChunkToMove( OperationContext* opCtx, const ChunkType& chunk) override; diff --git a/src/mongo/db/s/balancer/balancer_policy.cpp b/src/mongo/db/s/balancer/balancer_policy.cpp index 894c40dea3d..022379992f0 100644 --- a/src/mongo/db/s/balancer/balancer_policy.cpp +++ b/src/mongo/db/s/balancer/balancer_policy.cpp @@ -349,7 +349,8 @@ MigrateInfo chooseRandomMigration(const ShardStatisticsVector& shardStats, return {destShardId, chunks[getRandomIndex(chunks.size())], - MoveChunkRequest::ForceJumbo::kDoNotForce}; + MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance}; } vector<MigrateInfo> BalancerPolicy::balance(const ShardStatisticsVector& shardStats, @@ -408,7 +409,8 @@ vector<MigrateInfo> BalancerPolicy::balance(const ShardStatisticsVector& shardSt } invariant(to != stat.shardId); - migrations.emplace_back(to, chunk, MoveChunkRequest::ForceJumbo::kForceBalancer); + migrations.emplace_back( + to, chunk, MoveChunkRequest::ForceJumbo::kForceBalancer, MigrateInfo::drain); invariant(usedShards->insert(stat.shardId).second); invariant(usedShards->insert(to).second); break; @@ -458,7 +460,8 @@ vector<MigrateInfo> BalancerPolicy::balance(const ShardStatisticsVector& shardSt migrations.emplace_back(to, chunk, forceJumbo ? MoveChunkRequest::ForceJumbo::kForceBalancer - : MoveChunkRequest::ForceJumbo::kDoNotForce); + : MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::zoneViolation); invariant(usedShards->insert(stat.shardId).second); invariant(usedShards->insert(to).second); break; @@ -525,7 +528,8 @@ boost::optional<MigrateInfo> BalancerPolicy::balanceSingleChunk( return boost::optional<MigrateInfo>(); } - return MigrateInfo(newShardId, chunk, MoveChunkRequest::ForceJumbo::kDoNotForce); + return MigrateInfo( + newShardId, chunk, MoveChunkRequest::ForceJumbo::kDoNotForce, MigrateInfo::chunksImbalance); } bool BalancerPolicy::_singleZoneBalance(const ShardStatisticsVector& shardStats, @@ -585,7 +589,7 @@ bool BalancerPolicy::_singleZoneBalance(const ShardStatisticsVector& shardStats, continue; } - migrations->emplace_back(to, chunk, forceJumbo); + migrations->emplace_back(to, chunk, forceJumbo, MigrateInfo::chunksImbalance); invariant(usedShards->insert(chunk.getShard()).second); invariant(usedShards->insert(to).second); return true; @@ -609,7 +613,8 @@ string ZoneRange::toString() const { MigrateInfo::MigrateInfo(const ShardId& a_to, const ChunkType& a_chunk, - const MoveChunkRequest::ForceJumbo a_forceJumbo) { + const MoveChunkRequest::ForceJumbo a_forceJumbo, + MigrationReason a_reason) { invariant(a_chunk.validate()); invariant(a_to.isValid()); @@ -621,6 +626,7 @@ MigrateInfo::MigrateInfo(const ShardId& a_to, maxKey = a_chunk.getMax(); version = a_chunk.getVersion(); forceJumbo = a_forceJumbo; + reason = a_reason; } std::string MigrateInfo::getName() const { diff --git a/src/mongo/db/s/balancer/balancer_policy.h b/src/mongo/db/s/balancer/balancer_policy.h index c75c5ceda2d..daf1d16098b 100644 --- a/src/mongo/db/s/balancer/balancer_policy.h +++ b/src/mongo/db/s/balancer/balancer_policy.h @@ -42,6 +42,7 @@ namespace mongo { + struct ZoneRange { ZoneRange(const BSONObj& a_min, const BSONObj& a_max, const std::string& _zone); @@ -53,9 +54,12 @@ struct ZoneRange { }; struct MigrateInfo { + enum MigrationReason { drain, zoneViolation, chunksImbalance }; + MigrateInfo(const ShardId& a_to, const ChunkType& a_chunk, - MoveChunkRequest::ForceJumbo a_forceJumbo); + MoveChunkRequest::ForceJumbo a_forceJumbo, + MigrationReason a_reason); std::string getName() const; @@ -70,6 +74,7 @@ struct MigrateInfo { BSONObj maxKey; ChunkVersion version; MoveChunkRequest::ForceJumbo forceJumbo; + MigrationReason reason; }; typedef std::vector<ClusterStatistics::ShardStatistics> ShardStatisticsVector; diff --git a/src/mongo/db/s/balancer/balancer_policy_test.cpp b/src/mongo/db/s/balancer/balancer_policy_test.cpp index 81eb8366e80..d7b82b2240a 100644 --- a/src/mongo/db/s/balancer/balancer_policy_test.cpp +++ b/src/mongo/db/s/balancer/balancer_policy_test.cpp @@ -132,6 +132,7 @@ TEST(BalancerPolicy, Basic) { ASSERT_EQ(kShardId1, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::chunksImbalance, migrations[0].reason); } TEST(BalancerPolicy, SmallClusterShouldBePerfectlyBalanced) { @@ -147,6 +148,7 @@ TEST(BalancerPolicy, SmallClusterShouldBePerfectlyBalanced) { ASSERT_EQ(kShardId2, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::chunksImbalance, migrations[0].reason); } TEST(BalancerPolicy, SingleChunkShouldNotMove) { @@ -190,11 +192,13 @@ TEST(BalancerPolicy, ParallelBalancing) { ASSERT_EQ(kShardId2, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::chunksImbalance, migrations[0].reason); ASSERT_EQ(kShardId1, migrations[1].from); ASSERT_EQ(kShardId3, migrations[1].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMin(), migrations[1].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMax(), migrations[1].maxKey); + ASSERT_EQ(MigrateInfo::chunksImbalance, migrations[1].reason); } TEST(BalancerPolicy, ParallelBalancingDoesNotPutChunksOnShardsAboveTheOptimal) { @@ -214,11 +218,13 @@ TEST(BalancerPolicy, ParallelBalancingDoesNotPutChunksOnShardsAboveTheOptimal) { ASSERT_EQ(kShardId4, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::chunksImbalance, migrations[0].reason); ASSERT_EQ(kShardId1, migrations[1].from); ASSERT_EQ(kShardId5, migrations[1].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMin(), migrations[1].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMax(), migrations[1].maxKey); + ASSERT_EQ(MigrateInfo::chunksImbalance, migrations[1].reason); } TEST(BalancerPolicy, ParallelBalancingDoesNotMoveChunksFromShardsBelowOptimal) { @@ -236,6 +242,7 @@ TEST(BalancerPolicy, ParallelBalancingDoesNotMoveChunksFromShardsBelowOptimal) { ASSERT_EQ(kShardId3, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::chunksImbalance, migrations[0].reason); } TEST(BalancerPolicy, ParallelBalancingNotSchedulingOnInUseSourceShardsWithMoveNecessary) { @@ -255,6 +262,7 @@ TEST(BalancerPolicy, ParallelBalancingNotSchedulingOnInUseSourceShardsWithMoveNe ASSERT_EQ(kShardId2, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::chunksImbalance, migrations[0].reason); } TEST(BalancerPolicy, ParallelBalancingNotSchedulingOnInUseSourceShardsWithMoveNotNecessary) { @@ -288,6 +296,7 @@ TEST(BalancerPolicy, ParallelBalancingNotSchedulingOnInUseDestinationShards) { ASSERT_EQ(kShardId3, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::chunksImbalance, migrations[0].reason); } TEST(BalancerPolicy, JumboChunksNotMoved) { @@ -307,6 +316,7 @@ TEST(BalancerPolicy, JumboChunksNotMoved) { ASSERT_EQ(kShardId1, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][1].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][1].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::chunksImbalance, migrations[0].reason); } TEST(BalancerPolicy, JumboChunksNotMovedParallel) { @@ -334,11 +344,13 @@ TEST(BalancerPolicy, JumboChunksNotMovedParallel) { ASSERT_EQ(kShardId1, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][1].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][1].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::chunksImbalance, migrations[0].reason); ASSERT_EQ(kShardId2, migrations[1].from); ASSERT_EQ(kShardId3, migrations[1].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][2].getMin(), migrations[1].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][2].getMax(), migrations[1].maxKey); + ASSERT_EQ(MigrateInfo::chunksImbalance, migrations[1].reason); } TEST(BalancerPolicy, DrainingSingleChunk) { @@ -354,6 +366,7 @@ TEST(BalancerPolicy, DrainingSingleChunk) { ASSERT_EQ(kShardId1, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::drain, migrations[0].reason); } TEST(BalancerPolicy, DrainingSingleChunkPerShard) { @@ -372,11 +385,13 @@ TEST(BalancerPolicy, DrainingSingleChunkPerShard) { ASSERT_EQ(kShardId1, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::drain, migrations[0].reason); ASSERT_EQ(kShardId2, migrations[1].from); ASSERT_EQ(kShardId3, migrations[1].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMin(), migrations[1].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMax(), migrations[1].maxKey); + ASSERT_EQ(MigrateInfo::drain, migrations[1].reason); } TEST(BalancerPolicy, DrainingWithTwoChunksFirstOneSelected) { @@ -392,6 +407,7 @@ TEST(BalancerPolicy, DrainingWithTwoChunksFirstOneSelected) { ASSERT_EQ(kShardId1, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::drain, migrations[0].reason); } TEST(BalancerPolicy, DrainingMultipleShardsFirstOneSelected) { @@ -409,6 +425,7 @@ TEST(BalancerPolicy, DrainingMultipleShardsFirstOneSelected) { ASSERT_EQ(kShardId2, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::drain, migrations[0].reason); } TEST(BalancerPolicy, DrainingMultipleShardsWontAcceptChunks) { @@ -439,6 +456,7 @@ TEST(BalancerPolicy, DrainingSingleAppropriateShardFoundDueToTag) { ASSERT_EQ(kShardId1, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::drain, migrations[0].reason); } TEST(BalancerPolicy, DrainingNoAppropriateShardsFoundDueToTag) { @@ -516,6 +534,7 @@ TEST(BalancerPolicy, BalancerRespectsTagsWhenDraining) { ASSERT_EQ(kShardId0, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::drain, migrations[0].reason); } TEST(BalancerPolicy, BalancerRespectsTagPolicyBeforeImbalance) { @@ -535,6 +554,7 @@ TEST(BalancerPolicy, BalancerRespectsTagPolicyBeforeImbalance) { ASSERT_EQ(kShardId0, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::zoneViolation, migrations[0].reason); } TEST(BalancerPolicy, BalancerFixesIncorrectTagsWithCrossShardViolationOfTags) { @@ -555,6 +575,7 @@ TEST(BalancerPolicy, BalancerFixesIncorrectTagsWithCrossShardViolationOfTags) { ASSERT_EQ(kShardId2, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::zoneViolation, migrations[0].reason); } TEST(BalancerPolicy, BalancerFixesIncorrectTagsInOtherwiseBalancedCluster) { @@ -573,6 +594,7 @@ TEST(BalancerPolicy, BalancerFixesIncorrectTagsInOtherwiseBalancedCluster) { ASSERT_EQ(kShardId0, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::zoneViolation, migrations[0].reason); } TEST(BalancerPolicy, BalancerTagAlreadyBalanced) { @@ -650,11 +672,13 @@ TEST(BalancerPolicy, BalancerFixesIncorrectTagsInOtherwiseBalancedClusterParalle ASSERT_EQ(kShardId0, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMin(), migrations[0].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrateInfo::zoneViolation, migrations[0].reason); ASSERT_EQ(kShardId3, migrations[1].from); ASSERT_EQ(kShardId1, migrations[1].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId3][0].getMin(), migrations[1].minKey); ASSERT_BSONOBJ_EQ(cluster.second[kShardId3][0].getMax(), migrations[1].maxKey); + ASSERT_EQ(MigrateInfo::zoneViolation, migrations[0].reason); } TEST(BalancerPolicy, BalancerHandlesNoShardsWithTag) { diff --git a/src/mongo/db/s/balancer/migration_manager_test.cpp b/src/mongo/db/s/balancer/migration_manager_test.cpp index 61ee28ba11e..74c6e0168a1 100644 --- a/src/mongo/db/s/balancer/migration_manager_test.cpp +++ b/src/mongo/db/s/balancer/migration_manager_test.cpp @@ -120,9 +120,14 @@ TEST_F(MigrationManagerTest, OneCollectionTwoMigrations) { setUpChunk(collName, BSON(kPattern << 49), kKeyPattern.globalMax(), kShardId2, version); // Going to request that these two chunks get migrated. - const std::vector<MigrateInfo> migrationRequests{ - {kShardId1, chunk1, MoveChunkRequest::ForceJumbo::kDoNotForce}, - {kShardId3, chunk2, MoveChunkRequest::ForceJumbo::kDoNotForce}}; + const std::vector<MigrateInfo> migrationRequests{{kShardId1, + chunk1, + MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance}, + {kShardId3, + chunk2, + MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance}}; auto future = launchAsync([this, migrationRequests] { ThreadClient tc("Test", getGlobalServiceContext()); @@ -181,11 +186,22 @@ TEST_F(MigrationManagerTest, TwoCollectionsTwoMigrationsEach) { setUpChunk(collName2, BSON(kPattern << 49), kKeyPattern.globalMax(), kShardId2, version2); // Going to request that these four chunks get migrated. - const std::vector<MigrateInfo> migrationRequests{ - {kShardId1, chunk1coll1, MoveChunkRequest::ForceJumbo::kDoNotForce}, - {kShardId3, chunk2coll1, MoveChunkRequest::ForceJumbo::kDoNotForce}, - {kShardId1, chunk1coll2, MoveChunkRequest::ForceJumbo::kDoNotForce}, - {kShardId3, chunk2coll2, MoveChunkRequest::ForceJumbo::kDoNotForce}}; + const std::vector<MigrateInfo> migrationRequests{{kShardId1, + chunk1coll1, + MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance}, + {kShardId3, + chunk2coll1, + MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance}, + {kShardId1, + chunk1coll2, + MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance}, + {kShardId3, + chunk2coll2, + MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance}}; auto future = launchAsync([this, migrationRequests] { ThreadClient tc("Test", getGlobalServiceContext()); @@ -239,9 +255,14 @@ TEST_F(MigrationManagerTest, SourceShardNotFound) { setUpChunk(collName, BSON(kPattern << 49), kKeyPattern.globalMax(), kShardId2, version); // Going to request that these two chunks get migrated. - const std::vector<MigrateInfo> migrationRequests{ - {kShardId1, chunk1, MoveChunkRequest::ForceJumbo::kDoNotForce}, - {kShardId3, chunk2, MoveChunkRequest::ForceJumbo::kDoNotForce}}; + const std::vector<MigrateInfo> migrationRequests{{kShardId1, + chunk1, + MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance}, + {kShardId3, + chunk2, + MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance}}; auto future = launchAsync([this, chunk1, chunk2, migrationRequests] { ThreadClient tc("Test", getGlobalServiceContext()); @@ -288,8 +309,10 @@ TEST_F(MigrationManagerTest, JumboChunkResponseBackwardsCompatibility) { setUpChunk(collName, kKeyPattern.globalMin(), kKeyPattern.globalMax(), kShardId0, version); // Going to request that this chunk gets migrated. - const std::vector<MigrateInfo> migrationRequests{ - {kShardId1, chunk1, MoveChunkRequest::ForceJumbo::kDoNotForce}}; + const std::vector<MigrateInfo> migrationRequests{{kShardId1, + chunk1, + MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance}}; auto future = launchAsync([this, chunk1, migrationRequests] { ThreadClient tc("Test", getGlobalServiceContext()); @@ -338,13 +361,16 @@ TEST_F(MigrationManagerTest, InterruptMigration) { // up a dummy host for kShardHost0. shardTargeterMock(opCtx.get(), kShardId0)->setFindHostReturnValue(kShardHost0); - ASSERT_EQ(ErrorCodes::BalancerInterrupted, - _migrationManager->executeManualMigration( - opCtx.get(), - {kShardId1, chunk, MoveChunkRequest::ForceJumbo::kDoNotForce}, - 0, - kDefaultSecondaryThrottle, - false)); + ASSERT_EQ( + ErrorCodes::BalancerInterrupted, + _migrationManager->executeManualMigration(opCtx.get(), + {kShardId1, + chunk, + MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance}, + 0, + kDefaultSecondaryThrottle, + false)); }); // Wait till the move chunk request gets sent and pretend that it is stuck by never responding @@ -367,12 +393,14 @@ TEST_F(MigrationManagerTest, InterruptMigration) { // Ensure that no new migrations can be scheduled ASSERT_EQ(ErrorCodes::BalancerInterrupted, - _migrationManager->executeManualMigration( - operationContext(), - {kShardId1, chunk, MoveChunkRequest::ForceJumbo::kDoNotForce}, - 0, - kDefaultSecondaryThrottle, - false)); + _migrationManager->executeManualMigration(operationContext(), + {kShardId1, + chunk, + MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance}, + 0, + kDefaultSecondaryThrottle, + false)); // Ensure that the migration manager is no longer handling any migrations. _migrationManager->drainActiveMigrations(); @@ -435,12 +463,15 @@ TEST_F(MigrationManagerTest, RestartMigrationManager) { // up a dummy host for kShardHost0. shardTargeterMock(opCtx.get(), kShardId0)->setFindHostReturnValue(kShardHost0); - ASSERT_OK(_migrationManager->executeManualMigration( - opCtx.get(), - {kShardId1, chunk1, MoveChunkRequest::ForceJumbo::kDoNotForce}, - 0, - kDefaultSecondaryThrottle, - false)); + ASSERT_OK( + _migrationManager->executeManualMigration(opCtx.get(), + {kShardId1, + chunk1, + MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance}, + 0, + kDefaultSecondaryThrottle, + false)); }); // Expect only one moveChunk command to be called. @@ -587,9 +618,14 @@ TEST_F(MigrationManagerTest, RemoteCallErrorConversionToOperationFailed) { setUpChunk(collName, BSON(kPattern << 49), kKeyPattern.globalMax(), kShardId2, version); // Going to request that these two chunks get migrated. - const std::vector<MigrateInfo> migrationRequests{ - {kShardId1, chunk1, MoveChunkRequest::ForceJumbo::kDoNotForce}, - {kShardId3, chunk2, MoveChunkRequest::ForceJumbo::kDoNotForce}}; + const std::vector<MigrateInfo> migrationRequests{{kShardId1, + chunk1, + MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance}, + {kShardId3, + chunk2, + MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance}}; auto future = launchAsync([&] { ThreadClient tc("Test", getGlobalServiceContext()); diff --git a/src/mongo/db/s/balancer/scoped_migration_request_test.cpp b/src/mongo/db/s/balancer/scoped_migration_request_test.cpp index a60f8f9450f..42f71a10613 100644 --- a/src/mongo/db/s/balancer/scoped_migration_request_test.cpp +++ b/src/mongo/db/s/balancer/scoped_migration_request_test.cpp @@ -111,7 +111,10 @@ MigrateInfo makeMigrateInfo() { ChunkType chunkType = assertGet(ChunkType::parseFromConfigBSONCommand(chunkBuilder.obj())); ASSERT_OK(chunkType.validate()); - return MigrateInfo(kToShard, chunkType, MoveChunkRequest::ForceJumbo::kDoNotForce); + return MigrateInfo(kToShard, + chunkType, + MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance); } TEST_F(ScopedMigrationRequestTest, CreateScopedMigrationRequest) { diff --git a/src/mongo/db/s/balancer/type_migration.cpp b/src/mongo/db/s/balancer/type_migration.cpp index 4bedb26300c..16085881e8b 100644 --- a/src/mongo/db/s/balancer/type_migration.cpp +++ b/src/mongo/db/s/balancer/type_migration.cpp @@ -161,7 +161,10 @@ MigrateInfo MigrationType::toMigrateInfo() const { chunk.setMax(_max); chunk.setVersion(_chunkVersion); - return MigrateInfo(_toShard, chunk, MoveChunkRequest::parseForceJumbo(_forceJumbo)); + return MigrateInfo(_toShard, + chunk, + MoveChunkRequest::parseForceJumbo(_forceJumbo), + MigrateInfo::chunksImbalance); } } // namespace mongo diff --git a/src/mongo/db/s/balancer/type_migration_test.cpp b/src/mongo/db/s/balancer/type_migration_test.cpp index a1d2e9193b9..186cd0172be 100644 --- a/src/mongo/db/s/balancer/type_migration_test.cpp +++ b/src/mongo/db/s/balancer/type_migration_test.cpp @@ -61,7 +61,10 @@ TEST(MigrationTypeTest, ConvertFromMigrationInfo) { ChunkType chunkType = assertGet(ChunkType::fromConfigBSON(chunkBuilder.obj())); ASSERT_OK(chunkType.validate()); - MigrateInfo migrateInfo(kToShard, chunkType, MoveChunkRequest::ForceJumbo::kDoNotForce); + MigrateInfo migrateInfo(kToShard, + chunkType, + MoveChunkRequest::ForceJumbo::kDoNotForce, + MigrateInfo::chunksImbalance); MigrationType migrationType(migrateInfo, kWaitForDelete); BSONObjBuilder builder; diff --git a/src/mongo/db/s/config/configsvr_balancer_collection_status_command.cpp b/src/mongo/db/s/config/configsvr_balancer_collection_status_command.cpp new file mode 100644 index 00000000000..2cd769b989d --- /dev/null +++ b/src/mongo/db/s/config/configsvr_balancer_collection_status_command.cpp @@ -0,0 +1,118 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kSharding + +#include "mongo/platform/basic.h" + +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/catalog_raii.h" +#include "mongo/db/client.h" +#include "mongo/db/commands.h" +#include "mongo/db/operation_context.h" +#include "mongo/db/repl/repl_client_info.h" +#include "mongo/db/s/balancer/balancer.h" +#include "mongo/db/s/database_sharding_state.h" +#include "mongo/db/s/migration_source_manager.h" +#include "mongo/db/s/operation_sharding_state.h" +#include "mongo/db/s/shard_filtering_metadata_refresh.h" +#include "mongo/db/s/sharding_state.h" +#include "mongo/s/catalog_cache_loader.h" +#include "mongo/s/grid.h" +#include "mongo/s/request_types/balancer_collection_status_gen.h" +#include "mongo/util/log.h" + +namespace mongo { +namespace { + +class ConfigsvrBalancerCollectionStatusCmd final + : public TypedCommand<ConfigsvrBalancerCollectionStatusCmd> { +public: + using Request = ConfigsvrBalancerCollectionStatus; + using Response = BalancerCollectionStatusResponse; + + class Invocation final : public InvocationBase { + public: + using InvocationBase::InvocationBase; + + Response typedRun(OperationContext* opCtx) { + uassert(ErrorCodes::IllegalOperation, + str::stream() << ConfigsvrBalancerCollectionStatus::kCommandName + << " can only be run on config servers", + serverGlobalParams.clusterRole == ClusterRole::ConfigServer); + + const NamespaceString& nss = ns(); + + uassert(ErrorCodes::InvalidNamespace, + str::stream() << "Invalid namespace specified '" << nss.ns() << "'", + nss.isValid()); + + return Response(Balancer::get(opCtx)->getBalancerStatusForNs(opCtx, nss).toString()); + } + + private: + NamespaceString ns() const override { + return request().getCommandParameter(); + } + + bool supportsWriteConcern() const override { + return false; + } + + void doCheckAuthorization(OperationContext* opCtx) const override { + uassert(ErrorCodes::Unauthorized, + "Unauthorized", + AuthorizationSession::get(opCtx->getClient()) + ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(), + ActionType::internal)); + } + }; + + std::string help() const override { + return "Internal command, which is exported by the sharding config server. Do not call " + "directly. Checks whether the chunks of a given collection are in a quiesced state " + "or there are any which need to be moved because of (1) draining shards, (2) zone " + "violation or (3) imbalance between shards."; + } + + bool adminOnly() const override { + return true; + } + + AllowedOnSecondary secondaryAllowed(ServiceContext*) const override { + return AllowedOnSecondary::kNever; + } + +} configsvrBalancerCollectionStatusCmd; + +} // namespace +} // namespace mongo
\ No newline at end of file diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript index 20cf7b4a6fd..2096d9c1a47 100644 --- a/src/mongo/s/SConscript +++ b/src/mongo/s/SConscript @@ -179,8 +179,10 @@ env.Library( env.Idlc('catalog/type_shard_collection.idl')[0], env.Idlc('chunk_version.idl')[0], env.Idlc('database_version.idl')[0], + env.Idlc('request_types/balancer_collection_status.idl')[0], env.Idlc('request_types/clone_catalog_data.idl')[0], env.Idlc('request_types/clear_jumbo_flag.idl')[0], + env.Idlc('request_types/clone_collection_options_from_primary_shard.idl')[0], env.Idlc('request_types/create_collection.idl')[0], env.Idlc('request_types/create_database.idl')[0], env.Idlc('request_types/flush_database_cache_updates.idl')[0], @@ -188,7 +190,6 @@ env.Library( env.Idlc('request_types/get_database_version.idl')[0], env.Idlc('request_types/move_primary.idl')[0], env.Idlc('request_types/shard_collection.idl')[0], - env.Idlc('request_types/clone_collection_options_from_primary_shard.idl')[0], env.Idlc('request_types/refine_collection_shard_key.idl')[0], env.Idlc('request_types/rename_collection.idl')[0], env.Idlc('request_types/wait_for_fail_point.idl')[0], diff --git a/src/mongo/s/commands/SConscript b/src/mongo/s/commands/SConscript index 617ed80e8de..2fbcc76d81b 100644 --- a/src/mongo/s/commands/SConscript +++ b/src/mongo/s/commands/SConscript @@ -26,6 +26,7 @@ env.Library( 'cluster_add_shard_cmd.cpp', 'cluster_add_shard_to_zone_cmd.cpp', 'cluster_available_query_options_cmd.cpp', + 'cluster_balancer_collection_status_cmd.cpp', 'cluster_build_info.cpp', 'cluster_clear_jumbo_flag_cmd.cpp', 'cluster_coll_stats_cmd.cpp', diff --git a/src/mongo/s/commands/cluster_balancer_collection_status_cmd.cpp b/src/mongo/s/commands/cluster_balancer_collection_status_cmd.cpp new file mode 100644 index 00000000000..1cc17bf7ece --- /dev/null +++ b/src/mongo/s/commands/cluster_balancer_collection_status_cmd.cpp @@ -0,0 +1,124 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kSharding + +#include "mongo/platform/basic.h" + +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/catalog_raii.h" +#include "mongo/db/client.h" +#include "mongo/db/commands.h" +#include "mongo/db/operation_context.h" +#include "mongo/db/repl/repl_client_info.h" +#include "mongo/db/s/database_sharding_state.h" +#include "mongo/db/s/migration_source_manager.h" +#include "mongo/db/s/operation_sharding_state.h" +#include "mongo/db/s/shard_filtering_metadata_refresh.h" +#include "mongo/db/s/sharding_state.h" +#include "mongo/idl/idl_parser.h" +#include "mongo/s/catalog_cache_loader.h" +#include "mongo/s/grid.h" +#include "mongo/s/request_types/balancer_collection_status_gen.h" + +#include "mongo/util/log.h" + +namespace mongo { +namespace { + +class BalancerCollectionStatusCmd final : public TypedCommand<BalancerCollectionStatusCmd> { +public: + using Request = BalancerCollectionStatus; + using Response = BalancerCollectionStatusResponse; + + class Invocation final : public InvocationBase { + public: + using InvocationBase::InvocationBase; + + StringData kStatusField = "status"_sd; + + Response typedRun(OperationContext* opCtx) { + const NamespaceString& nss = ns(); + + ConfigsvrBalancerCollectionStatus configsvrRequest(nss); + configsvrRequest.setDbName(request().getDbName()); + + auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); + auto cmdResponse = uassertStatusOK(configShard->runCommandWithFixedRetryAttempts( + opCtx, + ReadPreferenceSetting(ReadPreference::PrimaryOnly), + "admin", + configsvrRequest.toBSON({}), + Shard::RetryPolicy::kIdempotent)); + + uassertStatusOK(cmdResponse.commandStatus); + + return Response::parse(IDLParserErrorContext("BalancerCollectionStatusResponse"), + cmdResponse.response); + } + + private: + NamespaceString ns() const override { + return request().getCommandParameter(); + } + + bool supportsWriteConcern() const override { + return false; + } + + void doCheckAuthorization(OperationContext* opCtx) const override { + uassert(ErrorCodes::Unauthorized, + "Unauthorized", + AuthorizationSession::get(opCtx->getClient()) + ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(), + ActionType::shardCollection)); + } + }; + + std::string help() const override { + return "command to check whether the chunks of a given collection are in a quiesced state " + "or there are any which need to be moved because of (1) draining shards, (2) zone " + "violation or (3) imbalance between shards"; + } + + bool adminOnly() const override { + return true; + } + + AllowedOnSecondary secondaryAllowed(ServiceContext*) const override { + return AllowedOnSecondary::kNever; + } + +} balancerCollectionStatusCmd; + +} // namespace +} // namespace mongo
\ No newline at end of file diff --git a/src/mongo/s/request_types/balancer_collection_status.idl b/src/mongo/s/request_types/balancer_collection_status.idl new file mode 100644 index 00000000000..f6a1dac70ee --- /dev/null +++ b/src/mongo/s/request_types/balancer_collection_status.idl @@ -0,0 +1,59 @@ +# Copyright(C) 2019 - present MongoDB, Inc. +# +# This program is free software : you can redistribute it and / or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program.If not, see +# < http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library.You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein.If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so.If you do not wish to do so, +# delete this exception statement from your version.If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +# _configsvrBalancerCollectionStatus and balancerCollectionStatus IDL File + +global: + cpp_namespace: "mongo" + +imports: + - "mongo/idl/basic_types.idl" + +structs: + BalancerCollectionStatusResponse: + description: "Response of the config server command" + strict: false + fields: + status: + type: string + description: "One of the following: balanced, draining, zoneViolation or chunksImbalance" + +commands: + balancerCollectionStatus: + cpp_name: BalancerCollectionStatus + description: "Public balancerCollectionStatus command on mongos" + strict: true + namespace: type + type: namespacestring + + _configsvrBalancerCollectionStatus: + cpp_name: ConfigsvrBalancerCollectionStatus + description: "Internal balancerCollectionStatus command on the config server" + strict: true + namespace: type + type: namespacestring diff --git a/src/mongo/shell/utils_sh.js b/src/mongo/shell/utils_sh.js index 07c3b5a14b1..d60fe31c7c2 100644 --- a/src/mongo/shell/utils_sh.js +++ b/src/mongo/shell/utils_sh.js @@ -91,6 +91,9 @@ sh.help = function() { print("\tsh.disableAutoSplit() disable autoSplit on one collection"); print("\tsh.enableAutoSplit() re-enable autoSplit on one collection"); print("\tsh.getShouldAutoSplit() returns whether autosplit is enabled"); + print( + "\tsh.balancerCollectionStatus(fullName) " + + "returns wheter the specified collection is balanced or the balancer needs to take more actions on it"); }; sh.status = function(verbose, configDB) { @@ -543,6 +546,10 @@ sh._shardingStatusStr = function(indent, s) { return indentStr(indent, s) + "\n"; }; +sh.balancerCollectionStatus = function(coll) { + return sh._adminCommand({balancerCollectionStatus: coll}, true); +}; + function printShardingStatus(configDB, verbose) { // configDB is a DB object that contains the sharding metadata of interest. // Defaults to the db named "config" on the current connection. @@ -827,4 +834,4 @@ function printShardingSizes(configDB) { }); print(raw); -} +}
\ No newline at end of file |