summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRandolph Tan <randolph@mongodb.com>2020-01-07 16:09:37 +0000
committerevergreen <evergreen@mongodb.com>2020-01-07 16:09:37 +0000
commit7a4e7bb1274a382832a6144e214e658825ffa3cd (patch)
tree1e5ee654d5ab96784d7680efc148a68073e3b4b8
parent9898ecda37f563eec41de95e7319076022ab2932 (diff)
downloadmongo-7a4e7bb1274a382832a6144e214e658825ffa3cd.tar.gz
SERVER-40435 A clearJumboChunk command to clear the jumbo flag
(cherry picked from commit 546e411b72cf6f75d24b304ce9219d1f3d3a4e4f) (cherry picked from commit 380711ea7dba41cb372a12a8d5258fdc124a0d23)
-rw-r--r--buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml2
-rw-r--r--jstests/auth/lib/commands_lib.js29
-rw-r--r--jstests/core/views/views_all_commands.js8
-rw-r--r--jstests/sharding/clear_jumbo.js108
-rw-r--r--jstests/sharding/database_and_shard_versioning_all_commands.js1
-rw-r--r--jstests/sharding/libs/last_stable_mongos_commands.js1
-rw-r--r--jstests/sharding/safe_secondary_reads_drop_recreate.js2
-rw-r--r--jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js2
-rw-r--r--jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js2
-rw-r--r--src/mongo/db/auth/action_types.txt1
-rw-r--r--src/mongo/db/auth/role_graph_builtin_roles.cpp1
-rw-r--r--src/mongo/db/s/SConscript2
-rw-r--r--src/mongo/db/s/config/configsvr_clear_jumbo_flag_command.cpp136
-rw-r--r--src/mongo/db/s/config/sharding_catalog_manager.h8
-rw-r--r--src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp105
-rw-r--r--src/mongo/db/s/config/sharding_catalog_manager_clear_jumbo_flag_test.cpp140
-rw-r--r--src/mongo/s/SConscript1
-rw-r--r--src/mongo/s/commands/SConscript1
-rw-r--r--src/mongo/s/commands/cluster_clear_jumbo_flag_cmd.cpp153
-rw-r--r--src/mongo/s/request_types/clear_jumbo_flag.idl72
20 files changed, 775 insertions, 0 deletions
diff --git a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
index 03701533b38..c1ed49a68df 100644
--- a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
@@ -58,6 +58,8 @@ selector:
# mongos in 4.0 doesn't like an aggregation explain without stages for optimized away pipelines,
# so blacklisting the test until 4.2 becomes last-stable.
- jstests/sharding/agg_write_stages_cannot_run_on_mongos.js
+ # Enable when SERVER-40435 is backported to last stable 4.0
+ - jstests/sharding/clear_jumbo.js
- jstests/sharding/explain_agg_read_pref.js
- jstests/sharding/explain_cmd.js
- jstests/sharding/failcommand_failpoint_not_parallel.js
diff --git a/jstests/auth/lib/commands_lib.js b/jstests/auth/lib/commands_lib.js
index 43fc19c9a06..979e53d780a 100644
--- a/jstests/auth/lib/commands_lib.js
+++ b/jstests/auth/lib/commands_lib.js
@@ -5648,6 +5648,35 @@ var authCommandsLib = {
db.runCommand({startRecordingTraffic: 1, filename: "notARealPath"});
}
},
+ {
+ testname: "clearJumboFlag",
+ command: {clearJumboFlag: "test.x"},
+ skipUnlessSharded: true,
+ testcases: [
+ {
+ runOnDb: adminDbName,
+ roles: roles_clusterManager,
+ privileges: [{resource: {db: "test", collection: "x"}, actions: ["clearJumboFlag"]}],
+ expectFail: true
+ },
+ {runOnDb: firstDbName, roles: {}},
+ {runOnDb: secondDbName, roles: {}}
+ ]
+ },
+ {
+ testname: "_configsvrClearJumboFlag",
+ command: {_configsvrClearJumboFlag: "x.y", epoch: ObjectId(), minKey: {x: 0}, maxKey: {x: 10}},
+ skipSharded: true,
+ expectFail: true,
+ testcases: [
+ {
+ runOnDb: adminDbName,
+ roles: {__system: 1},
+ privileges: [{resource: {cluster: true}, actions: ["internal"]}],
+ expectFail: true
+ },
+ ]
+ },
],
/************* SHARED TEST LOGIC ****************/
diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js
index 303981ac239..51a8f31e8ab 100644
--- a/jstests/core/views/views_all_commands.js
+++ b/jstests/core/views/views_all_commands.js
@@ -75,6 +75,7 @@ let viewsCommandTests = {
_configsvrBalancerStart: {skip: isAnInternalCommand},
_configsvrBalancerStatus: {skip: isAnInternalCommand},
_configsvrBalancerStop: {skip: isAnInternalCommand},
+ _configsvrClearJumboFlag: {skip: isAnInternalCommand},
_configsvrCommitChunkMerge: {skip: isAnInternalCommand},
_configsvrCommitChunkMigration: {skip: isAnInternalCommand},
_configsvrCommitChunkSplit: {skip: isAnInternalCommand},
@@ -131,6 +132,13 @@ let viewsCommandTests = {
cleanupOrphaned: {
skip: "Tested in views/views_sharded.js",
},
+ clearJumboFlag: {
+ command: {clearJumboFlag: "test.view"},
+ skipStandalone: true,
+ isAdminCommand: true,
+ expectFailure: true,
+ expectedErrorCode: ErrorCodes.NamespaceNotSharded,
+ },
clearLog: {skip: isUnrelated},
cloneCollection: {skip: "Tested in noPassthroughWithMongod/clonecollection.js"},
cloneCollectionAsCapped: {
diff --git a/jstests/sharding/clear_jumbo.js b/jstests/sharding/clear_jumbo.js
new file mode 100644
index 00000000000..39bdd79175d
--- /dev/null
+++ b/jstests/sharding/clear_jumbo.js
@@ -0,0 +1,108 @@
+(function() {
+"use strict";
+
+let st = new ShardingTest({shards: 2});
+
+assert.commandWorked(st.s.adminCommand({enableSharding: 'test'}));
+st.ensurePrimaryShard('test', st.shard0.shardName);
+assert.commandWorked(
+ st.s.adminCommand({addShardToZone: st.shard1.shardName, zone: 'finalDestination'}));
+
+////////////////////////////////////////////////////////////////////////////
+// Ranged shard key
+assert.commandWorked(st.s.adminCommand({shardCollection: 'test.range', key: {x: 1}}));
+assert.commandWorked(st.s.adminCommand({split: 'test.range', middle: {x: 0}}));
+
+let chunkColl = st.s.getDB('config').chunks;
+assert.commandWorked(chunkColl.update({ns: 'test.range', min: {x: 0}}, {$set: {jumbo: true}}));
+
+let jumboChunk = chunkColl.findOne({ns: 'test.range', min: {x: 0}});
+assert(jumboChunk.jumbo, tojson(jumboChunk));
+let jumboMajorVersionBefore = jumboChunk.lastmod.getTime();
+
+// Target non-jumbo chunk should not affect real jumbo chunk.
+assert.commandWorked(st.s.adminCommand({clearJumboFlag: 'test.range', find: {x: -1}}));
+jumboChunk = chunkColl.findOne({ns: 'test.range', min: {x: 0}});
+assert(jumboChunk.jumbo, tojson(jumboChunk));
+assert.eq(jumboMajorVersionBefore, jumboChunk.lastmod.getTime());
+
+// Target real jumbo chunk should bump version.
+assert.commandWorked(st.s.adminCommand({clearJumboFlag: 'test.range', find: {x: 1}}));
+jumboChunk = chunkColl.findOne({ns: 'test.range', min: {x: 0}});
+assert(!jumboChunk.jumbo, tojson(jumboChunk));
+assert.lt(jumboMajorVersionBefore, jumboChunk.lastmod.getTime());
+
+////////////////////////////////////////////////////////////////////////////
+// Hashed shard key
+assert.commandWorked(
+ st.s.adminCommand({shardCollection: 'test.hashed', key: {x: 'hashed'}, numInitialChunks: 2}));
+assert.commandWorked(chunkColl.update({ns: 'test.hashed', min: {x: 0}}, {$set: {jumbo: true}}));
+jumboChunk = chunkColl.findOne({ns: 'test.hashed', min: {x: 0}});
+assert(jumboChunk.jumbo, tojson(jumboChunk));
+jumboMajorVersionBefore = jumboChunk.lastmod.getTime();
+
+// Target non-jumbo chunk should not affect real jumbo chunk.
+let unrelatedChunk = chunkColl.findOne({ns: 'test.hashed', min: {x: MinKey}});
+assert.commandWorked(st.s.adminCommand(
+ {clearJumboFlag: 'test.hashed', bounds: [unrelatedChunk.min, unrelatedChunk.max]}));
+jumboChunk = chunkColl.findOne({ns: 'test.hashed', min: {x: 0}});
+assert(jumboChunk.jumbo, tojson(jumboChunk));
+assert.eq(jumboMajorVersionBefore, jumboChunk.lastmod.getTime());
+
+// Target real jumbo chunk should bump version.
+assert.commandWorked(
+ st.s.adminCommand({clearJumboFlag: 'test.hashed', bounds: [jumboChunk.min, jumboChunk.max]}));
+jumboChunk = chunkColl.findOne({ns: 'test.hashed', min: {x: 0}});
+assert(!jumboChunk.jumbo, tojson(jumboChunk));
+assert.lt(jumboMajorVersionBefore, jumboChunk.lastmod.getTime());
+
+////////////////////////////////////////////////////////////////////////////
+// Balancer with jumbo chunks behavior
+// Forces a jumbo chunk to be on a wrong zone but balancer shouldn't be able to move it until
+// jumbo flag is cleared.
+
+st.stopBalancer();
+assert.commandWorked(chunkColl.update({ns: 'test.range', min: {x: 0}}, {$set: {jumbo: true}}));
+assert.commandWorked(st.s.adminCommand(
+ {updateZoneKeyRange: 'test.range', min: {x: 0}, max: {x: MaxKey}, zone: 'finalDestination'}));
+
+let chunk = chunkColl.findOne({ns: 'test.range', min: {x: 0}});
+assert(chunk.jumbo, tojson(chunk));
+assert.eq(st.shard0.shardName, chunk.shard);
+
+st._configServers.forEach((conn) => {
+ conn.adminCommand({
+ configureFailPoint: 'overrideBalanceRoundInterval',
+ mode: 'alwaysOn',
+ data: {intervalMs: 200}
+ });
+});
+
+let waitForBalancerToRun = function() {
+ let lastRoundNumber =
+ assert.commandWorked(st.s.adminCommand({balancerStatus: 1})).numBalancerRounds;
+ st.startBalancer();
+
+ assert.soon(function() {
+ let roundNumber =
+ assert.commandWorked(st.s.adminCommand({balancerStatus: 1})).numBalancerRounds;
+ return roundNumber > lastRoundNumber;
+ });
+
+ st.stopBalancer();
+};
+
+waitForBalancerToRun();
+
+chunk = chunkColl.findOne({ns: 'test.range', min: {x: 0}});
+assert.eq(st.shard0.shardName, chunk.shard);
+
+assert.commandWorked(st.s.adminCommand({clearJumboFlag: 'test.range', find: {x: 0}}));
+
+waitForBalancerToRun();
+
+chunk = chunkColl.findOne({ns: 'test.range', min: {x: 0}});
+assert.eq(st.shard1.shardName, chunk.shard);
+
+st.stop();
+})();
diff --git a/jstests/sharding/database_and_shard_versioning_all_commands.js b/jstests/sharding/database_and_shard_versioning_all_commands.js
index 8a5d4ce6db1..779f8f3ae89 100644
--- a/jstests/sharding/database_and_shard_versioning_all_commands.js
+++ b/jstests/sharding/database_and_shard_versioning_all_commands.js
@@ -63,6 +63,7 @@ let testCases = {
balancerStatus: {skip: "not on a user database"},
balancerStop: {skip: "not on a user database"},
buildInfo: {skip: "executes locally on mongos (not sent to any remote node)"},
+ clearJumboFlag: {skip: "does not forward command to primary shard"},
clearLog: {skip: "executes locally on mongos (not sent to any remote node)"},
collMod: {
sendsDbVersion: true,
diff --git a/jstests/sharding/libs/last_stable_mongos_commands.js b/jstests/sharding/libs/last_stable_mongos_commands.js
index 67011fa9f13..92cacdb3e16 100644
--- a/jstests/sharding/libs/last_stable_mongos_commands.js
+++ b/jstests/sharding/libs/last_stable_mongos_commands.js
@@ -17,6 +17,7 @@ const commandsRemovedFromMongosIn42 = [
// being used.
const commandsAddedToMongosIn42 = [
'abortTransaction',
+ 'clearJumboFlag',
'commitTransaction',
'dropConnections',
'setIndexCommitQuorum',
diff --git a/jstests/sharding/safe_secondary_reads_drop_recreate.js b/jstests/sharding/safe_secondary_reads_drop_recreate.js
index bbf34b5a2a3..3a40c3afe62 100644
--- a/jstests/sharding/safe_secondary_reads_drop_recreate.js
+++ b/jstests/sharding/safe_secondary_reads_drop_recreate.js
@@ -44,6 +44,7 @@ let testCases = {
_configsvrBalancerStart: {skip: "primary only"},
_configsvrBalancerStatus: {skip: "primary only"},
_configsvrBalancerStop: {skip: "primary only"},
+ _configsvrClearJumboFlag: {skip: "primary only"},
_configsvrCommitChunkMerge: {skip: "primary only"},
_configsvrCommitChunkMigration: {skip: "primary only"},
_configsvrCommitChunkSplit: {skip: "primary only"},
@@ -93,6 +94,7 @@ let testCases = {
captrunc: {skip: "primary only"},
checkShardingIndex: {skip: "primary only"},
cleanupOrphaned: {skip: "primary only"},
+ clearJumboFlag: {skip: "primary only"},
clearLog: {skip: "does not return user data"},
clone: {skip: "primary only"},
cloneCollection: {skip: "primary only"},
diff --git a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
index 363c4cc06ca..e7e3c6566d4 100644
--- a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
+++ b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
@@ -51,6 +51,7 @@ let testCases = {
_configsvrBalancerStart: {skip: "primary only"},
_configsvrBalancerStatus: {skip: "primary only"},
_configsvrBalancerStop: {skip: "primary only"},
+ _configsvrClearJumboFlag: {skip: "primary only"},
_configsvrCommitChunkMerge: {skip: "primary only"},
_configsvrCommitChunkMigration: {skip: "primary only"},
_configsvrCommitChunkSplit: {skip: "primary only"},
@@ -106,6 +107,7 @@ let testCases = {
captrunc: {skip: "primary only"},
checkShardingIndex: {skip: "primary only"},
cleanupOrphaned: {skip: "primary only"},
+ clearJumboFlag: {skip: "primary only"},
clearLog: {skip: "does not return user data"},
clone: {skip: "primary only"},
cloneCollection: {skip: "primary only"},
diff --git a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
index 7b3a5be24f8..2562cd124e1 100644
--- a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
+++ b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
@@ -44,6 +44,7 @@ let testCases = {
_configsvrBalancerStart: {skip: "primary only"},
_configsvrBalancerStatus: {skip: "primary only"},
_configsvrBalancerStop: {skip: "primary only"},
+ _configsvrClearJumboFlag: {skip: "primary only"},
_configsvrCommitChunkMerge: {skip: "primary only"},
_configsvrCommitChunkMigration: {skip: "primary only"},
_configsvrCommitChunkSplit: {skip: "primary only"},
@@ -94,6 +95,7 @@ let testCases = {
captrunc: {skip: "primary only"},
checkShardingIndex: {skip: "primary only"},
cleanupOrphaned: {skip: "primary only"},
+ clearJumboFlag: {skip: "primary only"},
clearLog: {skip: "does not return user data"},
clone: {skip: "primary only"},
cloneCollection: {skip: "primary only"},
diff --git a/src/mongo/db/auth/action_types.txt b/src/mongo/db/auth/action_types.txt
index 56bbe633972..71582f7f85b 100644
--- a/src/mongo/db/auth/action_types.txt
+++ b/src/mongo/db/auth/action_types.txt
@@ -21,6 +21,7 @@
"changeStream",
"checkFreeMonitoringStatus",
"cleanupOrphaned",
+"clearJumboFlag",
"closeAllDatabases", # Deprecated, needs to stay around for backwards compatibility
"collMod",
"collStats",
diff --git a/src/mongo/db/auth/role_graph_builtin_roles.cpp b/src/mongo/db/auth/role_graph_builtin_roles.cpp
index daecf3176fc..783be516d4c 100644
--- a/src/mongo/db/auth/role_graph_builtin_roles.cpp
+++ b/src/mongo/db/auth/role_graph_builtin_roles.cpp
@@ -248,6 +248,7 @@ MONGO_INITIALIZER(AuthorizationBuiltinRoles)(InitializerContext* context) {
<< ActionType::setFreeMonitoring;
clusterManagerRoleDatabaseActions
+ << ActionType::clearJumboFlag
<< ActionType::splitChunk
<< ActionType::moveChunk
<< ActionType::enableSharding
diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript
index 6e9eda9fe4a..7b786a4d149 100644
--- a/src/mongo/db/s/SConscript
+++ b/src/mongo/db/s/SConscript
@@ -297,6 +297,7 @@ env.Library(
'clone_collection_options_from_primary_shard_cmd.cpp',
'config/configsvr_add_shard_command.cpp',
'config/configsvr_add_shard_to_zone_command.cpp',
+ 'config/configsvr_clear_jumbo_flag_command.cpp',
'config/configsvr_commit_chunk_migration_command.cpp',
'config/configsvr_commit_move_primary_command.cpp',
'config/configsvr_control_balancer_command.cpp',
@@ -451,6 +452,7 @@ env.CppUnitTest(
'config/sharding_catalog_manager_add_shard_test.cpp',
'config/sharding_catalog_manager_add_shard_to_zone_test.cpp',
'config/sharding_catalog_manager_assign_key_range_to_zone_test.cpp',
+ 'config/sharding_catalog_manager_clear_jumbo_flag_test.cpp',
'config/sharding_catalog_manager_commit_chunk_migration_test.cpp',
'config/sharding_catalog_manager_config_initialization_test.cpp',
'config/sharding_catalog_manager_create_collection_test.cpp',
diff --git a/src/mongo/db/s/config/configsvr_clear_jumbo_flag_command.cpp b/src/mongo/db/s/config/configsvr_clear_jumbo_flag_command.cpp
new file mode 100644
index 00000000000..8779bd3e48c
--- /dev/null
+++ b/src/mongo/db/s/config/configsvr_clear_jumbo_flag_command.cpp
@@ -0,0 +1,136 @@
+/**
+ * Copyright (C) 2019-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kSharding
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/audit.h"
+#include "mongo/db/auth/authorization_session.h"
+#include "mongo/db/commands.h"
+#include "mongo/db/repl/repl_client_info.h"
+#include "mongo/db/s/config/sharding_catalog_manager.h"
+#include "mongo/s/catalog/dist_lock_manager.h"
+#include "mongo/s/grid.h"
+#include "mongo/s/request_types/clear_jumbo_flag_gen.h"
+#include "mongo/util/log.h"
+
+namespace mongo {
+namespace {
+
+class ConfigsvrClearJumboFlagCommand final : public TypedCommand<ConfigsvrClearJumboFlagCommand> {
+public:
+ using Request = ConfigsvrClearJumboFlag;
+
+ class Invocation final : public InvocationBase {
+ public:
+ using InvocationBase::InvocationBase;
+
+ void typedRun(OperationContext* opCtx) {
+ const NamespaceString& nss = ns();
+
+ uassert(ErrorCodes::IllegalOperation,
+ "_configsvrClearJumboFlag can only be run on config servers",
+ serverGlobalParams.clusterRole == ClusterRole::ConfigServer);
+ uassert(ErrorCodes::InvalidOptions,
+ "_configsvrClearJumboFlag must be called with majority writeConcern",
+ opCtx->getWriteConcern().wMode == WriteConcernOptions::kMajority);
+
+ // Set the operation context read concern level to local for reads into the config
+ // database.
+ repl::ReadConcernArgs::get(opCtx) =
+ repl::ReadConcernArgs(repl::ReadConcernLevel::kLocalReadConcern);
+
+ const auto catalogClient = Grid::get(opCtx)->catalogClient();
+
+ // Acquire distlocks on the namespace's database and collection.
+ DistLockManager::ScopedDistLock dbDistLock(
+ uassertStatusOK(catalogClient->getDistLockManager()->lock(
+ opCtx, nss.db(), "clearJumboFlag", DistLockManager::kDefaultLockTimeout)));
+ DistLockManager::ScopedDistLock collDistLock(
+ uassertStatusOK(catalogClient->getDistLockManager()->lock(
+ opCtx, nss.ns(), "clearJumboFlag", DistLockManager::kDefaultLockTimeout)));
+
+ const auto collStatus =
+ catalogClient->getCollection(opCtx, nss, repl::ReadConcernLevel::kLocalReadConcern);
+
+ uassert(ErrorCodes::NamespaceNotSharded,
+ str::stream() << "clearJumboFlag namespace " << nss.toString()
+ << " is not sharded",
+ collStatus != ErrorCodes::NamespaceNotFound);
+
+ const auto collType = uassertStatusOK(collStatus).value;
+
+ uassert(ErrorCodes::StaleEpoch,
+ str::stream()
+ << "clearJumboFlag namespace " << nss.toString()
+ << " has a different epoch than mongos had in its routing table cache",
+ request().getEpoch() == collType.getEpoch());
+
+ ShardingCatalogManager::get(opCtx)->clearJumboFlag(
+ opCtx,
+ nss,
+ request().getEpoch(),
+ ChunkRange{request().getMinKey(), request().getMaxKey()});
+ }
+
+ private:
+ NamespaceString ns() const override {
+ return request().getCommandParameter();
+ }
+
+ bool supportsWriteConcern() const override {
+ return true;
+ }
+
+ void doCheckAuthorization(OperationContext* opCtx) const override {
+ uassert(ErrorCodes::Unauthorized,
+ "Unauthorized",
+ AuthorizationSession::get(opCtx->getClient())
+ ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(),
+ ActionType::internal));
+ }
+ };
+
+ std::string help() const override {
+ return "Internal command, which is exported by the sharding config server. Do not call "
+ "directly. Clears the jumbo flag of the chunk specified.";
+ }
+
+ bool adminOnly() const override {
+ return true;
+ }
+
+ AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
+ return AllowedOnSecondary::kNever;
+ }
+} configsvrRefineCollectionShardKeyCmd;
+
+} // namespace
+} // namespace mongo
diff --git a/src/mongo/db/s/config/sharding_catalog_manager.h b/src/mongo/db/s/config/sharding_catalog_manager.h
index 8283333b183..4e5f0d61241 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager.h
+++ b/src/mongo/db/s/config/sharding_catalog_manager.h
@@ -219,6 +219,14 @@ public:
const ShardId& toShard,
const boost::optional<Timestamp>& validAfter);
+ /**
+ * Removes the jumbo flag from the specified chunk.
+ */
+ void clearJumboFlag(OperationContext* opCtx,
+ const NamespaceString& nss,
+ const OID& collectionEpoch,
+ const ChunkRange& chunk);
+
//
// Database Operations
//
diff --git a/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp
index 0542d7f5843..9174a7008ef 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp
+++ b/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp
@@ -60,6 +60,8 @@ namespace {
MONGO_FAIL_POINT_DEFINE(migrationCommitVersionError);
MONGO_FAIL_POINT_DEFINE(skipExpiringOldChunkHistory);
+const WriteConcernOptions kNoWaitWriteConcern(1, WriteConcernOptions::SyncMode::UNSET, Seconds(0));
+
/**
* Append min, max and version information from chunk to the buffer for logChange purposes.
*/
@@ -855,4 +857,107 @@ StatusWith<ChunkVersion> ShardingCatalogManager::_findCollectionVersion(
return currentCollectionVersion;
}
+void ShardingCatalogManager::clearJumboFlag(OperationContext* opCtx,
+ const NamespaceString& nss,
+ const OID& collectionEpoch,
+ const ChunkRange& chunk) {
+ auto const configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
+
+ // Take _kChunkOpLock in exclusive mode to prevent concurrent chunk splits, merges, and
+ // migrations.
+ //
+ // ConfigSvrClearJumboFlag commands must be run serially because the new ChunkVersions
+ // for the modified chunks are generated within the command and must be committed to the
+ // database before another chunk operation generates new ChunkVersions in the same manner.
+ //
+ // TODO(SERVER-25359): Replace with a collection-specific lock map to allow splits/merges/
+ // move chunks on different collections to proceed in parallel.
+ // (Note: This is not needed while we have a global lock, taken here only for consistency.)
+ Lock::ExclusiveLock lk(opCtx->lockState(), _kChunkOpLock);
+
+ auto targetChunkResult = uassertStatusOK(configShard->exhaustiveFindOnConfig(
+ opCtx,
+ ReadPreferenceSetting{ReadPreference::PrimaryOnly},
+ repl::ReadConcernLevel::kLocalReadConcern,
+ ChunkType::ConfigNS,
+ BSON(ChunkType::ns(nss.ns())
+ << ChunkType::min(chunk.getMin()) << ChunkType::max(chunk.getMax())),
+ {},
+ 1));
+
+ const auto targetChunkVector = std::move(targetChunkResult.docs);
+ uassert(51262,
+ str::stream() << "Unable to locate chunk " << chunk.toString()
+ << " from ns: " << nss.ns(),
+ !targetChunkVector.empty());
+
+ const auto targetChunk = uassertStatusOK(ChunkType::fromConfigBSON(targetChunkVector.front()));
+
+ if (!targetChunk.getJumbo()) {
+ return;
+ }
+
+ // Must use local read concern because we will perform subsequent writes.
+ auto findResponse = uassertStatusOK(
+ configShard->exhaustiveFindOnConfig(opCtx,
+ ReadPreferenceSetting{ReadPreference::PrimaryOnly},
+ repl::ReadConcernLevel::kLocalReadConcern,
+ ChunkType::ConfigNS,
+ BSON(ChunkType::ns(nss.ns())),
+ BSON(ChunkType::lastmod << -1),
+ 1));
+
+ const auto chunksVector = std::move(findResponse.docs);
+ uassert(ErrorCodes::IncompatibleShardingMetadata,
+ str::stream() << "Tried to find max chunk version for collection '" << nss.ns()
+ << ", but found no chunks",
+ !chunksVector.empty());
+
+ const auto highestVersionChunk =
+ uassertStatusOK(ChunkType::fromConfigBSON(chunksVector.front()));
+ const auto currentCollectionVersion = highestVersionChunk.getVersion();
+
+ // It is possible for a migration to end up running partly without the protection of the
+ // distributed lock if the config primary stepped down since the start of the migration and
+ // failed to recover the migration. Check that the collection has not been dropped and recreated
+ // since the migration began, unbeknown to the shard when the command was sent.
+ uassert(ErrorCodes::StaleEpoch,
+ str::stream() << "The collection '" << nss.ns()
+ << "' has been dropped and recreated since the migration began."
+ " The config server's collection version epoch is now '"
+ << currentCollectionVersion.epoch().toString() << "', but the shard's is "
+ << collectionEpoch.toString() << "'. Aborting clear jumbo on chunk ("
+ << chunk.toString() << ").",
+ currentCollectionVersion.epoch() == collectionEpoch);
+
+ ChunkVersion newVersion(
+ currentCollectionVersion.majorVersion() + 1, 0, currentCollectionVersion.epoch());
+
+ BSONObj chunkQuery(BSON(ChunkType::name(targetChunk.getName())
+ << ChunkType::ns(nss.ns()) << ChunkType::epoch(collectionEpoch)
+ << ChunkType::min(chunk.getMin()) << ChunkType::max(chunk.getMax())));
+
+ BSONObjBuilder updateBuilder;
+ updateBuilder.append("$unset", BSON(ChunkType::jumbo() << ""));
+
+ BSONObjBuilder updateVersionClause(updateBuilder.subobjStart("$set"));
+ newVersion.appendLegacyWithField(&updateVersionClause, ChunkType::lastmod());
+ updateVersionClause.doneFast();
+
+ auto chunkUpdate = updateBuilder.obj();
+
+ auto didUpdate = uassertStatusOK(
+ Grid::get(opCtx)->catalogClient()->updateConfigDocument(opCtx,
+ ChunkType::ConfigNS,
+ chunkQuery,
+ chunkUpdate,
+ false /* upsert */,
+ kNoWaitWriteConcern));
+
+ uassert(51263,
+ str::stream() << "failed to clear jumbo flag due to " << chunkQuery
+ << " not matching any existing chunks",
+ didUpdate);
+}
+
} // namespace mongo
diff --git a/src/mongo/db/s/config/sharding_catalog_manager_clear_jumbo_flag_test.cpp b/src/mongo/db/s/config/sharding_catalog_manager_clear_jumbo_flag_test.cpp
new file mode 100644
index 00000000000..89441e6f64e
--- /dev/null
+++ b/src/mongo/db/s/config/sharding_catalog_manager_clear_jumbo_flag_test.cpp
@@ -0,0 +1,140 @@
+/**
+ * Copyright (C) 2019-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/bson/bsonobj.h"
+#include "mongo/bson/bsonobjbuilder.h"
+#include "mongo/client/read_preference.h"
+#include "mongo/db/namespace_string.h"
+#include "mongo/db/s/config/sharding_catalog_manager.h"
+#include "mongo/s/catalog/type_chunk.h"
+#include "mongo/s/catalog/type_shard.h"
+#include "mongo/s/client/shard_registry.h"
+#include "mongo/s/config_server_test_fixture.h"
+
+namespace mongo {
+namespace {
+
+using unittest::assertGet;
+
+
+class ClearJumboFlagTest : public ConfigServerTestFixture {
+public:
+ const NamespaceString& ns() {
+ return _namespace;
+ }
+
+ const OID& epoch() {
+ return _epoch;
+ }
+
+ ChunkRange jumboChunk() {
+ return ChunkRange(BSON("x" << MINKEY), BSON("x" << 0));
+ }
+
+ ChunkRange nonJumboChunk() {
+ return ChunkRange(BSON("x" << 0), BSON("x" << MaxKey));
+ }
+
+protected:
+ void setUp() override {
+ ConfigServerTestFixture::setUp();
+
+ ShardType shard;
+ shard.setName("shard");
+ shard.setHost("shard:12");
+
+ setupShards({shard});
+
+ CollectionType collection;
+ collection.setNs(_namespace);
+ collection.setEpoch(_epoch);
+ collection.setKeyPattern(BSON("x" << 1));
+
+ ASSERT_OK(insertToConfigCollection(
+ operationContext(), CollectionType::ConfigNS, collection.toBSON()));
+
+ ChunkType chunk;
+ chunk.setNS(_namespace);
+ chunk.setVersion({12, 7, _epoch});
+ chunk.setShard(shard.getName());
+ chunk.setMin(jumboChunk().getMin());
+ chunk.setMax(jumboChunk().getMax());
+ chunk.setJumbo(true);
+
+ ChunkType otherChunk;
+ otherChunk.setNS(_namespace);
+ otherChunk.setVersion({14, 7, _epoch});
+ otherChunk.setShard(shard.getName());
+ otherChunk.setMin(nonJumboChunk().getMin());
+ otherChunk.setMax(nonJumboChunk().getMax());
+
+ setupChunks({chunk, otherChunk});
+ }
+
+private:
+ const NamespaceString _namespace{"TestDB.TestColl"};
+ const OID _epoch{OID::gen()};
+};
+
+TEST_F(ClearJumboFlagTest, ClearJumboShouldBumpVersion) {
+ ShardingCatalogManager::get(operationContext())
+ ->clearJumboFlag(operationContext(), ns(), epoch(), jumboChunk());
+
+ auto chunkDoc = uassertStatusOK(getChunkDoc(operationContext(), jumboChunk().getMin()));
+ ASSERT_FALSE(chunkDoc.getJumbo());
+ ASSERT_EQ(ChunkVersion(15, 0, epoch()), chunkDoc.getVersion());
+}
+
+TEST_F(ClearJumboFlagTest, ClearJumboShouldNotBumpVersionIfChunkNotJumbo) {
+ ShardingCatalogManager::get(operationContext())
+ ->clearJumboFlag(operationContext(), ns(), epoch(), nonJumboChunk());
+
+ auto chunkDoc = uassertStatusOK(getChunkDoc(operationContext(), nonJumboChunk().getMin()));
+ ASSERT_FALSE(chunkDoc.getJumbo());
+ ASSERT_EQ(ChunkVersion(14, 7, epoch()), chunkDoc.getVersion());
+}
+
+TEST_F(ClearJumboFlagTest, AssertsOnEpochMismatch) {
+ ASSERT_THROWS_CODE(ShardingCatalogManager::get(operationContext())
+ ->clearJumboFlag(operationContext(), ns(), OID::gen(), jumboChunk()),
+ AssertionException,
+ ErrorCodes::StaleEpoch);
+}
+
+TEST_F(ClearJumboFlagTest, AssertsIfChunkCantBeFound) {
+ ChunkRange imaginaryChunk(BSON("x" << 0), BSON("x" << 10));
+ ASSERT_THROWS(ShardingCatalogManager::get(operationContext())
+ ->clearJumboFlag(operationContext(), ns(), OID::gen(), imaginaryChunk),
+ AssertionException);
+}
+
+} // namespace
+} // namespace mongo
diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript
index 1854d61023c..a11e34b9e74 100644
--- a/src/mongo/s/SConscript
+++ b/src/mongo/s/SConscript
@@ -177,6 +177,7 @@ env.Library(
env.Idlc('chunk_version.idl')[0],
env.Idlc('database_version.idl')[0],
env.Idlc('request_types/clone_catalog_data.idl')[0],
+ env.Idlc('request_types/clear_jumbo_flag.idl')[0],
env.Idlc('request_types/create_collection.idl')[0],
env.Idlc('request_types/create_database.idl')[0],
env.Idlc('request_types/flush_database_cache_updates.idl')[0],
diff --git a/src/mongo/s/commands/SConscript b/src/mongo/s/commands/SConscript
index c74d5396746..a135bf56d1a 100644
--- a/src/mongo/s/commands/SConscript
+++ b/src/mongo/s/commands/SConscript
@@ -27,6 +27,7 @@ env.Library(
'cluster_add_shard_to_zone_cmd.cpp',
'cluster_available_query_options_cmd.cpp',
'cluster_build_info.cpp',
+ 'cluster_clear_jumbo_flag_cmd.cpp',
'cluster_coll_stats_cmd.cpp',
'cluster_collection_mod_cmd.cpp',
'cluster_commit_transaction_cmd.cpp',
diff --git a/src/mongo/s/commands/cluster_clear_jumbo_flag_cmd.cpp b/src/mongo/s/commands/cluster_clear_jumbo_flag_cmd.cpp
new file mode 100644
index 00000000000..95b214d5bf8
--- /dev/null
+++ b/src/mongo/s/commands/cluster_clear_jumbo_flag_cmd.cpp
@@ -0,0 +1,153 @@
+/**
+ * Copyright (C) 2019-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kCommand
+
+#include "mongo/platform/basic.h"
+
+#include <string>
+#include <vector>
+
+#include "mongo/db/auth/action_set.h"
+#include "mongo/db/auth/action_type.h"
+#include "mongo/db/auth/authorization_session.h"
+#include "mongo/db/commands.h"
+#include "mongo/s/catalog_cache.h"
+#include "mongo/s/client/shard_registry.h"
+#include "mongo/s/cluster_commands_helpers.h"
+#include "mongo/s/grid.h"
+#include "mongo/s/request_types/clear_jumbo_flag_gen.h"
+#include "mongo/util/log.h"
+
+namespace mongo {
+namespace {
+
+class ClearJumboFlagCommand final : public TypedCommand<ClearJumboFlagCommand> {
+public:
+ using Request = ClearJumboFlag;
+
+ class Invocation : public MinimalInvocationBase {
+ public:
+ using MinimalInvocationBase::MinimalInvocationBase;
+
+ private:
+ bool supportsWriteConcern() const override {
+ return false;
+ }
+
+ void doCheckAuthorization(OperationContext* opCtx) const override {
+ uassert(ErrorCodes::Unauthorized,
+ "Unauthorized",
+ AuthorizationSession::get(opCtx->getClient())
+ ->isAuthorizedForActionsOnResource(ResourcePattern::forExactNamespace(ns()),
+ ActionType::clearJumboFlag));
+ }
+
+ NamespaceString ns() const override {
+ return request().getCommandParameter();
+ }
+
+ void run(OperationContext* opCtx, rpc::ReplyBuilderInterface* result) override {
+ auto routingInfo = uassertStatusOK(
+ Grid::get(opCtx)->catalogCache()->getShardedCollectionRoutingInfoWithRefresh(opCtx,
+ ns()));
+ const auto cm = routingInfo.cm();
+
+ uassert(ErrorCodes::InvalidOptions,
+ "bounds can only have exactly 2 elements",
+ !request().getBounds() || request().getBounds()->size() == 2);
+
+ uassert(ErrorCodes::InvalidOptions,
+ "cannot specify bounds and find at the same time",
+ !(request().getFind() && request().getBounds()));
+
+ uassert(ErrorCodes::InvalidOptions,
+ "need to specify find or bounds",
+ request().getFind() || request().getBounds());
+
+ boost::optional<Chunk> chunk;
+
+ if (request().getFind()) {
+ BSONObj shardKey = uassertStatusOK(
+ cm->getShardKeyPattern().extractShardKeyFromQuery(opCtx, *request().getFind()));
+ uassert(51260,
+ str::stream()
+ << "no shard key found in chunk query " << *request().getFind(),
+ !shardKey.isEmpty());
+
+ chunk.emplace(cm->findIntersectingChunkWithSimpleCollation(shardKey));
+ } else {
+ auto boundsArray = *request().getBounds();
+ BSONObj minKey = cm->getShardKeyPattern().normalizeShardKey(boundsArray.front());
+ BSONObj maxKey = cm->getShardKeyPattern().normalizeShardKey(boundsArray.back());
+
+ chunk.emplace(cm->findIntersectingChunkWithSimpleCollation(minKey));
+
+ uassert(51261,
+ str::stream() << "no chunk found with the shard key bounds "
+ << ChunkRange(minKey, maxKey).toString(),
+ chunk->getMin().woCompare(minKey) == 0 &&
+ chunk->getMax().woCompare(maxKey) == 0);
+ }
+
+ ConfigsvrClearJumboFlag configCmd(
+ ns(), cm->getVersion().epoch(), chunk->getMin(), chunk->getMax());
+ configCmd.setDbName(request().getDbName());
+
+ auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
+ auto cmdResponse = uassertStatusOK(configShard->runCommandWithFixedRetryAttempts(
+ opCtx,
+ ReadPreferenceSetting(ReadPreference::PrimaryOnly),
+ "admin",
+ CommandHelpers::appendMajorityWriteConcern(configCmd.toBSON({})),
+ Shard::RetryPolicy::kIdempotent));
+
+ uassertStatusOK(cmdResponse.commandStatus);
+ uassertStatusOK(cmdResponse.writeConcernStatus);
+ }
+ };
+
+ bool adminOnly() const override {
+ return true;
+ }
+
+ AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
+ return AllowedOnSecondary::kAlways;
+ }
+
+
+ std::string help() const override {
+ return "clears the jumbo flag of the chunk that contains the key\n"
+ " { clearJumboFlag : 'alleyinsider.blog.posts' , find : { ts : 1 } }\n";
+ }
+
+} clusterClearJumboFlag;
+
+} // namespace
+} // namespace mongo
diff --git a/src/mongo/s/request_types/clear_jumbo_flag.idl b/src/mongo/s/request_types/clear_jumbo_flag.idl
new file mode 100644
index 00000000000..fc65a878ed1
--- /dev/null
+++ b/src/mongo/s/request_types/clear_jumbo_flag.idl
@@ -0,0 +1,72 @@
+# Copyright (C) 2019-present MongoDB, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the Server Side Public License, version 1,
+# as published by MongoDB, Inc.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Server Side Public License for more details.
+#
+# You should have received a copy of the Server Side Public License
+# along with this program. If not, see
+# <http://www.mongodb.com/licensing/server-side-public-license>.
+#
+# As a special exception, the copyright holders give permission to link the
+# code of portions of this program with the OpenSSL library under certain
+# conditions as described in each individual source file and distribute
+# linked combinations including the program with the OpenSSL library. You
+# must comply with the Server Side Public License in all respects for
+# all of the code used other than as permitted herein. If you modify file(s)
+# with this exception, you may extend this exception to your version of the
+# file(s), but you are not obligated to do so. If you do not wish to do so,
+# delete this exception statement from your version. If you delete this
+# exception statement from all source files in the program, then also delete
+# it in the license file.
+#
+
+# clearJumboFlag IDL File
+
+global:
+ cpp_namespace: "mongo"
+
+imports:
+ - "mongo/idl/basic_types.idl"
+
+commands:
+ clearJumboFlag:
+ description: "clearJumboFlag command for mongos"
+ namespace: type
+ type: namespacestring
+ strict: false
+ fields:
+ bounds:
+ type: array<object>
+ description: "The exact boundaries for a single chunk."
+ optional: true
+
+ find:
+ type: object
+ description: "The shard key value that is within a chunk's boundaries.
+ Cannot be used on collections with hashed shard keys."
+ optional: true
+
+ _configsvrClearJumboFlag:
+ cpp_name: ConfigsvrClearJumboFlag
+ description: "internal clearJumboFlag command for config server"
+ namespace: type
+ type: namespacestring
+ strict: false
+ fields:
+ epoch:
+ type: objectid
+ description: "The expected epoch of the namespace provided to
+ clearJumboFlag."
+ optional: false
+ minKey:
+ type: object
+ description: "The lower bound key value of the chunk."
+ maxKey:
+ type: object
+ description: "The upper bound key value of the chunk." \ No newline at end of file