summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--jstests/sharding/refine_collection_shard_key_basic.js236
-rw-r--r--src/mongo/db/s/SConscript1
-rw-r--r--src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp45
-rw-r--r--src/mongo/db/s/config/configsvr_shard_collection_command.cpp242
-rw-r--r--src/mongo/db/s/shard_key_util.cpp231
-rw-r--r--src/mongo/db/s/shard_key_util.h91
-rw-r--r--src/mongo/s/shard_key_pattern.cpp4
-rw-r--r--src/mongo/s/shard_key_pattern.h6
8 files changed, 609 insertions, 247 deletions
diff --git a/jstests/sharding/refine_collection_shard_key_basic.js b/jstests/sharding/refine_collection_shard_key_basic.js
index c19206aa88e..708e154dc49 100644
--- a/jstests/sharding/refine_collection_shard_key_basic.js
+++ b/jstests/sharding/refine_collection_shard_key_basic.js
@@ -13,39 +13,247 @@
const kCollName = 'foo';
const kNsName = kDbName + '.' + kCollName;
- assert.commandWorked(mongos.adminCommand({enableSharding: kDbName}));
+ function enableShardingAndShardColl(keyDoc) {
+ assert.commandWorked(mongos.adminCommand({enableSharding: kDbName}));
+ assert.commandWorked(mongos.adminCommand({shardCollection: kNsName, key: keyDoc}));
+ }
- // refineCollectionShardKey should fail because namespace 'db.foo' is not sharded.
+ function dropAndRecreateColl(keyDoc) {
+ assert.commandWorked(mongos.getDB(kDbName).runCommand({drop: kCollName}));
+ assert.writeOK(mongos.getCollection(kNsName).insert(keyDoc));
+ }
+
+ function dropAndReshardColl(keyDoc) {
+ assert.commandWorked(mongos.getDB(kDbName).runCommand({drop: kCollName}));
+ assert.commandWorked(mongos.adminCommand({shardCollection: kNsName, key: keyDoc}));
+ }
+
+ function dropAndReshardCollUnique(keyDoc) {
+ assert.commandWorked(mongos.getDB(kDbName).runCommand({drop: kCollName}));
+ assert.commandWorked(
+ mongos.adminCommand({shardCollection: kNsName, key: keyDoc, unique: true}));
+ }
+
+ // ********** SIMPLE TESTS **********
+
+ // Should fail because arguments 'refineCollectionShardKey' and 'key' are invalid types.
assert.commandFailedWithCode(
- mongos.adminCommand({refineCollectionShardKey: kNsName, key: {aKey: 1}}),
+ mongos.adminCommand({refineCollectionShardKey: {_id: 1}, key: {_id: 1, aKey: 1}}),
+ ErrorCodes.TypeMismatch);
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: 'blah'}),
+ ErrorCodes.TypeMismatch);
+
+ // Should fail because refineCollectionShardKey may only be run against the admin database.
+ assert.commandFailedWithCode(mongos.getDB(kDbName).runCommand(
+ {refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
+ ErrorCodes.Unauthorized);
+
+ // Should fail because namespace 'db.foo' does not exist.
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
+ ErrorCodes.NamespaceNotFound);
+
+ assert.writeOK(mongos.getCollection(kNsName).insert({aKey: 1}));
+
+ // Should fail because namespace 'db.foo' is not sharded. NOTE: This NamespaceNotSharded error
+ // is thrown in RefineCollectionShardKeyCommand by 'getShardedCollectionRoutingInfoWithRefresh'.
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
ErrorCodes.NamespaceNotSharded);
- // refineCollectionShardKey should work because namespace 'db.foo' is sharded.
+ enableShardingAndShardColl({_id: 1});
+
+ // Should fail because shard key is invalid (i.e. bad values).
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 5}}),
+ ErrorCodes.BadValue);
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: -1}}),
+ ErrorCodes.BadValue);
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 'hashed'}}),
+ ErrorCodes.BadValue);
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {aKey: 'hahashed'}}),
+ ErrorCodes.BadValue);
+
+ // Should fail because shard key is not specified.
+ assert.commandFailedWithCode(mongos.adminCommand({refineCollectionShardKey: kNsName}), 40414);
+ assert.commandFailedWithCode(mongos.adminCommand({refineCollectionShardKey: kNsName, key: {}}),
+ ErrorCodes.BadValue);
+
+ // Should work because new shard key is already same as current shard key of namespace 'db.foo'.
+ assert.commandWorked(mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1}}));
+ dropAndReshardColl({a: 1, b: 1});
+ assert.commandWorked(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {a: 1, b: 1}}));
+ dropAndReshardColl({aKey: 'hashed'});
+ assert.commandWorked(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {aKey: 'hashed'}}));
+
+ assert.commandWorked(mongos.getDB(kDbName).dropDatabase());
+
+ // ********** NAMESPACE VALIDATION TESTS **********
+
+ enableShardingAndShardColl({_id: 1});
+
+ // Configure failpoint 'hangRefineCollectionShardKeyAfterRefresh' on staleMongos and run
+ // refineCollectionShardKey against this mongos in a parallel thread.
+ assert.commandWorked(staleMongos.adminCommand(
+ {configureFailPoint: 'hangRefineCollectionShardKeyAfterRefresh', mode: 'alwaysOn'}));
+ const awaitShellToTriggerNamespaceNotSharded = startParallelShell(() => {
+ assert.commandFailedWithCode(
+ db.adminCommand({refineCollectionShardKey: 'db.foo', key: {_id: 1, aKey: 1}}),
+ ErrorCodes.NamespaceNotSharded);
+ }, staleMongos.port);
+ waitForFailpoint('Hit hangRefineCollectionShardKeyAfterRefresh', 1);
+
+ // Drop and re-create namespace 'db.foo' without staleMongos refreshing its metadata.
+ dropAndRecreateColl({aKey: 1});
+
+ // Should fail because namespace 'db.foo' is not sharded. NOTE: This NamespaceNotSharded error
+ // is thrown in ConfigsvrRefineCollectionShardKeyCommand.
+ assert.commandWorked(staleMongos.adminCommand(
+ {configureFailPoint: 'hangRefineCollectionShardKeyAfterRefresh', mode: 'off'}));
+ awaitShellToTriggerNamespaceNotSharded();
+
assert.commandWorked(mongos.adminCommand({shardCollection: kNsName, key: {_id: 1}}));
- assert.commandWorked(mongos.adminCommand({refineCollectionShardKey: kNsName, key: {aKey: 1}}));
// Configure failpoint 'hangRefineCollectionShardKeyAfterRefresh' on staleMongos and run
// refineCollectionShardKey against this mongos in a parallel thread.
assert.commandWorked(staleMongos.adminCommand(
{configureFailPoint: 'hangRefineCollectionShardKeyAfterRefresh', mode: 'alwaysOn'}));
- const awaitShell = startParallelShell(() => {
+ const awaitShellToTriggerStaleEpoch = startParallelShell(() => {
assert.commandFailedWithCode(
- db.adminCommand({refineCollectionShardKey: 'db.foo', key: {aKey: 1}}),
+ db.adminCommand({refineCollectionShardKey: 'db.foo', key: {_id: 1, aKey: 1}}),
ErrorCodes.StaleEpoch);
}, staleMongos.port);
- waitForFailpoint('Hit hangRefineCollectionShardKeyAfterRefresh', 1);
+ waitForFailpoint('Hit hangRefineCollectionShardKeyAfterRefresh', 2);
// Drop and re-shard namespace 'db.foo' without staleMongos refreshing its metadata.
- assert.commandWorked(mongos.getDB(kDbName).runCommand({drop: kCollName}));
- assert.commandWorked(mongos.adminCommand({shardCollection: kNsName, key: {_id: 1}}));
+ dropAndReshardColl({_id: 1});
- // refineCollectionShardKey should fail because staleMongos has a stale epoch.
+ // Should fail because staleMongos has a stale epoch.
assert.commandWorked(staleMongos.adminCommand(
{configureFailPoint: 'hangRefineCollectionShardKeyAfterRefresh', mode: 'off'}));
- awaitShell();
+ awaitShellToTriggerStaleEpoch();
+
+ assert.commandWorked(mongos.getDB(kDbName).dropDatabase());
+
+ // ********** SHARD KEY VALIDATION TESTS **********
+
+ enableShardingAndShardColl({_id: 1});
+
+ // Should fail because new shard key {aKey: 1} does not extend current shard key {_id: 1} of
+ // namespace 'db.foo'.
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {aKey: 1}}),
+ ErrorCodes.InvalidOptions);
+
+ // Should fail because no index exists for new shard key {_id: 1, aKey: 1}.
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
+ ErrorCodes.InvalidOptions);
+
+ // Should fail because only a sparse index exists for new shard key {_id: 1, aKey: 1}.
+ dropAndReshardColl({_id: 1});
+ assert.commandWorked(
+ mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}, {sparse: true}));
+
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
+ ErrorCodes.InvalidOptions);
+
+ // Should fail because only a partial index exists for new shard key {_id: 1, aKey: 1}.
+ dropAndReshardColl({_id: 1});
+ assert.commandWorked(mongos.getCollection(kNsName).createIndex(
+ {_id: 1, aKey: 1}, {partialFilterExpression: {aKey: {$gt: 0}}}));
+
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
+ ErrorCodes.OperationFailed);
+
+ // Should fail because only a multikey index exists for new shard key {_id: 1, aKey: 1}.
+ dropAndReshardColl({_id: 1});
+ assert.commandWorked(mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}));
+ assert.writeOK(mongos.getCollection(kNsName).insert({aKey: [1, 2, 3, 4, 5]}));
+
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
+ ErrorCodes.OperationFailed);
+
+ // Should fail because current shard key {a: 1} is unique, new shard key is {a: 1, b: 1}, and an
+ // index only exists on {a: 1, b: 1, c: 1}.
+ dropAndReshardCollUnique({a: 1});
+ assert.commandWorked(mongos.getCollection(kNsName).createIndex({a: 1, b: 1, c: 1}));
+
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {a: 1, b: 1}}),
+ ErrorCodes.InvalidOptions);
+
+ // Should work because current shard key {_id: 1} is not unique, new shard key is {_id: 1, aKey:
+ // 1}, and an index exists on {_id: 1, aKey: 1, bKey: 1}.
+ dropAndReshardColl({_id: 1});
+ assert.commandWorked(mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1, bKey: 1}));
+
+ assert.commandWorked(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}));
+
+ // Should fail because only an index with missing or incomplete shard key entries exists for new
+ // shard key {_id: 1, aKey: 1}.
+ dropAndReshardColl({_id: 1});
+ assert.commandWorked(mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}));
+ assert.writeOK(mongos.getCollection(kNsName).insert({_id: 12345}));
+
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
+ ErrorCodes.OperationFailed);
+
+ // Should fail because new shard key {aKey: 1} is not a prefix of current shard key {_id: 1,
+ // aKey: 1}.
+ dropAndReshardColl({_id: 1, aKey: 1});
+ assert.commandWorked(mongos.getCollection(kNsName).createIndex({aKey: 1}));
+
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {aKey: 1}}),
+ ErrorCodes.InvalidOptions);
+
+ // Should fail because new shard key {aKey: 1, _id: 1} is not a prefix of current shard key
+ // {_id: 1, aKey: 1}.
+ dropAndReshardColl({_id: 1, aKey: 1});
+ assert.commandWorked(mongos.getCollection(kNsName).createIndex({aKey: 1, _id: 1}));
+
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {aKey: 1, _id: 1}}),
+ ErrorCodes.InvalidOptions);
+
+ // Should fail because new shard key {aKey: 1, _id: 1, bKey: 1} is not a prefix of current shard
+ // key {_id: 1, aKey: 1}.
+ dropAndReshardColl({_id: 1, aKey: 1});
+ assert.commandWorked(mongos.getCollection(kNsName).createIndex({aKey: 1, _id: 1, bKey: 1}));
+
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {aKey: 1, _id: 1, bKey: 1}}),
+ ErrorCodes.InvalidOptions);
+
+ // Should fail because new shard key {aKey: 1, bKey: 1} is not a prefix of current shard key
+ // {_id: 1}.
+ dropAndReshardColl({_id: 1});
+ assert.commandWorked(mongos.getCollection(kNsName).createIndex({aKey: 1, bKey: 1}));
+
+ assert.commandFailedWithCode(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {aKey: 1, bKey: 1}}),
+ ErrorCodes.InvalidOptions);
+
+ // Should work because a 'useful' index exists for new shard key {_id: 1, aKey: 1}.
+ dropAndReshardColl({_id: 1});
+ assert.commandWorked(mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}));
+
+ assert.commandWorked(
+ mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}));
- // refineCollectionShardKey should work because mongos has the current epoch.
- assert.commandWorked(mongos.adminCommand({refineCollectionShardKey: kNsName, key: {aKey: 1}}));
+ assert.commandWorked(mongos.getDB(kDbName).dropDatabase());
st.stop();
})();
diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript
index 78f1819573e..41588ca7eb8 100644
--- a/src/mongo/db/s/SConscript
+++ b/src/mongo/db/s/SConscript
@@ -57,6 +57,7 @@ env.Library(
'session_catalog_migration_source.cpp',
'shard_filtering_metadata_refresh.cpp',
'shard_identity_rollback_notifier.cpp',
+ 'shard_key_util.cpp',
'shard_metadata_util.cpp',
'shard_server_catalog_cache_loader.cpp',
'shard_server_op_observer.cpp',
diff --git a/src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp b/src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp
index 37b8d03b4e7..910fb096689 100644
--- a/src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp
+++ b/src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp
@@ -33,6 +33,8 @@
#include "mongo/db/auth/authorization_session.h"
#include "mongo/db/commands.h"
+#include "mongo/db/repl/repl_client_info.h"
+#include "mongo/db/s/shard_key_util.h"
#include "mongo/s/catalog/dist_lock_manager.h"
#include "mongo/s/grid.h"
#include "mongo/s/request_types/refine_collection_shard_key_gen.h"
@@ -79,6 +81,8 @@ public:
"refineCollectionShardKey",
DistLockManager::kDefaultLockTimeout)));
+ // Validate the given namespace is (i) sharded and (ii) has the same epoch as the router
+ // that received refineCollectionShardKey had in its routing table cache.
const auto collStatus =
catalogClient->getCollection(opCtx, nss, repl::ReadConcernLevel::kLocalReadConcern);
@@ -95,6 +99,47 @@ public:
<< nss.toString()
<< " has a different epoch than mongos had in its routing table cache",
request().getEpoch() == collType.getEpoch());
+
+ const auto oldShardKeyPattern = ShardKeyPattern(collType.getKeyPattern());
+
+ // Validate the given shard key (i) extends the current shard key, (ii) has a "useful"
+ // index, and (iii) the index in question has no null entries.
+ const auto proposedKey = request().getKey().getOwned();
+
+ if (SimpleBSONObjComparator::kInstance.evaluate(oldShardKeyPattern.toBSON() ==
+ proposedKey)) {
+ repl::ReplClientInfo::forClient(opCtx->getClient())
+ .setLastOpToSystemLastOpTime(opCtx);
+ return;
+ }
+
+ const auto newShardKeyPattern = ShardKeyPattern(proposedKey);
+
+ uassert(ErrorCodes::InvalidOptions,
+ str::stream() << "refineCollectionShardKey shard key " << proposedKey.toString()
+ << " does not extend the current shard key "
+ << collType.getKeyPattern().toString(),
+ oldShardKeyPattern.isExtendedBy(newShardKeyPattern));
+
+ const auto dbType =
+ uassertStatusOK(
+ catalogClient->getDatabase(
+ opCtx, nss.db().toString(), repl::ReadConcernArgs::get(opCtx).getLevel()))
+ .value;
+ const auto primaryShardId = dbType.getPrimary();
+ const auto primaryShard =
+ uassertStatusOK(Grid::get(opCtx)->shardRegistry()->getShard(opCtx, primaryShardId));
+
+ // Since createIndexIfPossible is false, we have no need for default collation and set
+ // it to boost::none.
+ shardkeyutil::validateShardKeyAgainstExistingIndexes(opCtx,
+ nss,
+ proposedKey,
+ newShardKeyPattern,
+ primaryShard,
+ boost::none,
+ collType.getUnique(),
+ false); // createIndexIfPossible
}
private:
diff --git a/src/mongo/db/s/config/configsvr_shard_collection_command.cpp b/src/mongo/db/s/config/configsvr_shard_collection_command.cpp
index 3cdb4d2e5d3..e53552916d8 100644
--- a/src/mongo/db/s/config/configsvr_shard_collection_command.cpp
+++ b/src/mongo/db/s/config/configsvr_shard_collection_command.cpp
@@ -31,7 +31,6 @@
#include "mongo/platform/basic.h"
-#include "mongo/bson/simple_bsonelement_comparator.h"
#include "mongo/bson/util/bson_extract.h"
#include "mongo/db/audit.h"
#include "mongo/db/auth/action_type.h"
@@ -39,8 +38,6 @@
#include "mongo/db/auth/privilege.h"
#include "mongo/db/commands.h"
#include "mongo/db/commands/test_commands_enabled.h"
-#include "mongo/db/hasher.h"
-#include "mongo/db/index/index_descriptor.h"
#include "mongo/db/namespace_string.h"
#include "mongo/db/query/collation/collator_factory_interface.h"
#include "mongo/db/repl/read_concern_args.h"
@@ -49,16 +46,14 @@
#include "mongo/db/repl/replication_coordinator.h"
#include "mongo/db/s/config/initial_split_policy.h"
#include "mongo/db/s/config/sharding_catalog_manager.h"
+#include "mongo/db/s/shard_key_util.h"
#include "mongo/s/balancer_configuration.h"
#include "mongo/s/catalog/type_database.h"
#include "mongo/s/catalog/type_shard.h"
#include "mongo/s/catalog_cache.h"
#include "mongo/s/client/shard_registry.h"
-#include "mongo/s/cluster_commands_helpers.h"
#include "mongo/s/config_server_client.h"
#include "mongo/s/grid.h"
-#include "mongo/s/request_types/shard_collection_gen.h"
-#include "mongo/s/shard_util.h"
#include "mongo/util/log.h"
#include "mongo/util/scopeguard.h"
#include "mongo/util/str.h"
@@ -71,60 +66,6 @@ using std::string;
const long long kMaxSizeMBDefault = 0;
/**
- * Constructs the BSON specification document for the given namespace, index key and options.
- */
-BSONObj makeCreateIndexesCmd(const NamespaceString& nss,
- const BSONObj& keys,
- const BSONObj& collation,
- bool unique) {
- BSONObjBuilder index;
-
- // Required fields for an index.
-
- index.append("key", keys);
-
- StringBuilder indexName;
- bool isFirstKey = true;
- for (BSONObjIterator keyIter(keys); keyIter.more();) {
- BSONElement currentKey = keyIter.next();
-
- if (isFirstKey) {
- isFirstKey = false;
- } else {
- indexName << "_";
- }
-
- indexName << currentKey.fieldName() << "_";
- if (currentKey.isNumber()) {
- indexName << currentKey.numberInt();
- } else {
- indexName << currentKey.str(); // this should match up with shell command
- }
- }
- index.append("name", indexName.str());
-
- // Index options.
-
- if (!collation.isEmpty()) {
- // Creating an index with the "collation" option requires a v=2 index.
- index.append("v", static_cast<int>(IndexDescriptor::IndexVersion::kV2));
- index.append("collation", collation);
- }
-
- if (unique && !IndexDescriptor::isIdIndexPattern(keys)) {
- index.appendBool("unique", unique);
- }
-
- // The outer createIndexes command.
-
- BSONObjBuilder createIndexes;
- createIndexes.append("createIndexes", nss.coll());
- createIndexes.append("indexes", BSON_ARRAY(index.obj()));
- createIndexes.append("writeConcern", WriteConcernOptions::Majority);
- return appendAllowImplicitCreate(createIndexes.obj(), true);
-}
-
-/**
* Validates the options specified in the request.
*
* WARNING: After validating the request's collation, replaces it with the collection default
@@ -247,177 +188,6 @@ void validateAndDeduceFullRequestOptions(OperationContext* opCtx,
}
/**
- * Compares the proposed shard key with the collection's existing indexes on the primary shard to
- * ensure they are a legal combination.
- *
- * If the collection is empty and no index on the shard key exists, creates the required index.
- */
-void validateShardKeyAgainstExistingIndexes(OperationContext* opCtx,
- const NamespaceString& nss,
- const BSONObj& proposedKey,
- const ShardKeyPattern& shardKeyPattern,
- const std::shared_ptr<Shard>& primaryShard,
- const ConfigsvrShardCollectionRequest& request) {
- // The proposed shard key must be validated against the set of existing indexes.
- // In particular, we must ensure the following constraints
- //
- // 1. All existing unique indexes, except those which start with the _id index,
- // must contain the proposed key as a prefix (uniqueness of the _id index is
- // ensured by the _id generation process or guaranteed by the user).
- //
- // 2. If the collection is not empty, there must exist at least one index that
- // is "useful" for the proposed key. A "useful" index is defined as follows
- // Useful Index:
- // i. contains proposedKey as a prefix
- // ii. is not a sparse index, partial index, or index with a non-simple collation
- // iii. contains no null values
- // iv. is not multikey (maybe lift this restriction later)
- // v. if a hashed index, has default seed (lift this restriction later)
- //
- // 3. If the proposed shard key is specified as unique, there must exist a useful,
- // unique index exactly equal to the proposedKey (not just a prefix).
- //
- // After validating these constraint:
- //
- // 4. If there is no useful index, and the collection is non-empty, we
- // must fail.
- //
- // 5. If the collection is empty, and it's still possible to create an index
- // on the proposed key, we go ahead and do so.
-
- auto listIndexesCmd = BSON("listIndexes" << nss.coll());
- auto indexesRes =
- primaryShard->runExhaustiveCursorCommand(opCtx,
- ReadPreferenceSetting(ReadPreference::PrimaryOnly),
- nss.db().toString(),
- listIndexesCmd,
- Milliseconds(-1));
- std::vector<BSONObj> indexes;
- if (indexesRes.getStatus().code() != ErrorCodes::NamespaceNotFound) {
- indexes = uassertStatusOK(indexesRes).docs;
- }
-
- // 1. Verify consistency with existing unique indexes
- for (const auto& idx : indexes) {
- BSONObj currentKey = idx["key"].embeddedObject();
- bool isUnique = idx["unique"].trueValue();
- uassert(ErrorCodes::InvalidOptions,
- str::stream() << "can't shard collection '" << nss.ns() << "' with unique index on "
- << currentKey
- << " and proposed shard key "
- << proposedKey
- << ". Uniqueness can't be maintained unless shard key is a prefix",
- !isUnique || shardKeyPattern.isUniqueIndexCompatible(currentKey));
- }
-
- // 2. Check for a useful index
- bool hasUsefulIndexForKey = false;
- for (const auto& idx : indexes) {
- BSONObj currentKey = idx["key"].embeddedObject();
- // Check 2.i. and 2.ii.
- if (!idx["sparse"].trueValue() && idx["filter"].eoo() && idx["collation"].eoo() &&
- proposedKey.isPrefixOf(currentKey, SimpleBSONElementComparator::kInstance)) {
- // We can't currently use hashed indexes with a non-default hash seed
- // Check v.
- // Note that this means that, for sharding, we only support one hashed index
- // per field per collection.
- uassert(ErrorCodes::InvalidOptions,
- str::stream() << "can't shard collection " << nss.ns()
- << " with hashed shard key "
- << proposedKey
- << " because the hashed index uses a non-default seed of "
- << idx["seed"].numberInt(),
- !shardKeyPattern.isHashedPattern() || idx["seed"].eoo() ||
- idx["seed"].numberInt() == BSONElementHasher::DEFAULT_HASH_SEED);
- hasUsefulIndexForKey = true;
- }
- }
-
- // 3. If proposed key is required to be unique, additionally check for exact match.
-
- if (hasUsefulIndexForKey && request.getUnique()) {
- BSONObj eqQuery = BSON("ns" << nss.ns() << "key" << proposedKey);
- BSONObj eqQueryResult;
-
- for (const auto& idx : indexes) {
- if (SimpleBSONObjComparator::kInstance.evaluate(idx["key"].embeddedObject() ==
- proposedKey)) {
- eqQueryResult = idx;
- break;
- }
- }
-
- if (eqQueryResult.isEmpty()) {
- // If no exact match, index not useful, but still possible to create one later
- hasUsefulIndexForKey = false;
- } else {
- bool isExplicitlyUnique = eqQueryResult["unique"].trueValue();
- BSONObj currKey = eqQueryResult["key"].embeddedObject();
- bool isCurrentID = (currKey.firstElementFieldNameStringData() == "_id");
- uassert(ErrorCodes::InvalidOptions,
- str::stream() << "can't shard collection " << nss.ns() << ", " << proposedKey
- << " index not unique, and unique index explicitly specified",
- isExplicitlyUnique || isCurrentID);
- }
- }
-
- auto countCmd = BSON("count" << nss.coll());
- auto countRes =
- uassertStatusOK(primaryShard->runCommand(opCtx,
- ReadPreferenceSetting(ReadPreference::PrimaryOnly),
- nss.db().toString(),
- countCmd,
- Shard::RetryPolicy::kIdempotent));
- const bool isEmpty = (countRes.response["n"].Int() == 0);
-
- if (hasUsefulIndexForKey) {
- // Check 2.iii and 2.iv. Make sure no null entries in the sharding index
- // and that there is a useful, non-multikey index available
- BSONObjBuilder checkShardingIndexCmd;
- checkShardingIndexCmd.append("checkShardingIndex", nss.ns());
- checkShardingIndexCmd.append("keyPattern", proposedKey);
- auto checkShardingIndexRes = uassertStatusOK(
- primaryShard->runCommand(opCtx,
- ReadPreferenceSetting(ReadPreference::PrimaryOnly),
- "admin",
- checkShardingIndexCmd.obj(),
- Shard::RetryPolicy::kIdempotent));
- uassert(ErrorCodes::OperationFailed,
- checkShardingIndexRes.response["errmsg"].str(),
- checkShardingIndexRes.commandStatus == Status::OK());
- } else if (!isEmpty) {
- // 4. if no useful index, and collection is non-empty, fail
- uasserted(ErrorCodes::InvalidOptions,
- "Please create an index that starts with the proposed shard key before "
- "sharding the collection");
- } else {
- // 5. If no useful index exists, and collection empty, create one on proposedKey.
- // Only need to call ensureIndex on primary shard, since indexes get copied to
- // receiving shard whenever a migrate occurs.
- // If the collection has a default collation, explicitly send the simple
- // collation as part of the createIndex request.
- BSONObj collation =
- !request.getCollation()->isEmpty() ? CollationSpec::kSimpleSpec : BSONObj();
- auto createIndexesCmd =
- makeCreateIndexesCmd(nss, proposedKey, collation, request.getUnique());
-
- const auto swResponse = primaryShard->runCommandWithFixedRetryAttempts(
- opCtx,
- ReadPreferenceSetting(ReadPreference::PrimaryOnly),
- nss.db().toString(),
- createIndexesCmd,
- Shard::RetryPolicy::kNoRetry);
- auto createIndexesStatus = swResponse.getStatus();
- if (createIndexesStatus.isOK()) {
- const auto response = swResponse.getValue();
- createIndexesStatus = (!response.commandStatus.isOK()) ? response.commandStatus
- : response.writeConcernStatus;
- }
- uassertStatusOK(createIndexesStatus);
- }
-}
-
-/**
* Migrates the initial "big chunks" from the primary shard to spread them evenly across the shards.
*
* If 'finalSplitPoints' is not empty, additionally splits each "big chunk" into smaller chunks
@@ -812,8 +582,14 @@ public:
}
// Step 3.
- validateShardKeyAgainstExistingIndexes(
- opCtx, nss, proposedKey, shardKeyPattern, primaryShard, request);
+ shardkeyutil::validateShardKeyAgainstExistingIndexes(opCtx,
+ nss,
+ proposedKey,
+ shardKeyPattern,
+ primaryShard,
+ request.getCollation(),
+ request.getUnique(),
+ true); // createIndexIfPossible
// Step 4.
if (request.getGetUUIDfromPrimaryShard()) {
diff --git a/src/mongo/db/s/shard_key_util.cpp b/src/mongo/db/s/shard_key_util.cpp
new file mode 100644
index 00000000000..a056fcd3232
--- /dev/null
+++ b/src/mongo/db/s/shard_key_util.cpp
@@ -0,0 +1,231 @@
+/**
+ * Copyright (C) 2019-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/bson/simple_bsonelement_comparator.h"
+#include "mongo/db/hasher.h"
+#include "mongo/db/index/index_descriptor.h"
+#include "mongo/db/query/collation/collator_factory_interface.h"
+#include "mongo/db/s/shard_key_util.h"
+#include "mongo/s/cluster_commands_helpers.h"
+
+namespace mongo {
+namespace shardkeyutil {
+
+BSONObj makeCreateIndexesCmd(const NamespaceString& nss,
+ const BSONObj& keys,
+ const BSONObj& collation,
+ const bool unique) {
+ BSONObjBuilder index;
+
+ // Required fields for an index.
+ index.append("key", keys);
+
+ StringBuilder indexName;
+ bool isFirstKey = true;
+ for (BSONObjIterator keyIter(keys); keyIter.more();) {
+ BSONElement currentKey = keyIter.next();
+
+ if (isFirstKey) {
+ isFirstKey = false;
+ } else {
+ indexName << "_";
+ }
+
+ indexName << currentKey.fieldName() << "_";
+ if (currentKey.isNumber()) {
+ indexName << currentKey.numberInt();
+ } else {
+ indexName << currentKey.str(); // This should match up with shell command.
+ }
+ }
+ index.append("name", indexName.str());
+
+ // Index options.
+ if (!collation.isEmpty()) {
+ // Creating an index with the "collation" option requires a v=2 index.
+ index.append("v", static_cast<int>(IndexDescriptor::IndexVersion::kV2));
+ index.append("collation", collation);
+ }
+
+ if (unique && !IndexDescriptor::isIdIndexPattern(keys)) {
+ index.appendBool("unique", unique);
+ }
+
+ // The outer createIndexes command.
+ BSONObjBuilder createIndexes;
+ createIndexes.append("createIndexes", nss.coll());
+ createIndexes.append("indexes", BSON_ARRAY(index.obj()));
+ createIndexes.append("writeConcern", WriteConcernOptions::Majority);
+ return appendAllowImplicitCreate(createIndexes.obj(), true);
+}
+
+void validateShardKeyAgainstExistingIndexes(OperationContext* opCtx,
+ const NamespaceString& nss,
+ const BSONObj& proposedKey,
+ const ShardKeyPattern& shardKeyPattern,
+ const std::shared_ptr<Shard>& primaryShard,
+ const boost::optional<BSONObj>& defaultCollation,
+ const bool unique,
+ const bool createIndexIfPossible) {
+ auto listIndexesCmd = BSON("listIndexes" << nss.coll());
+ auto indexesRes =
+ primaryShard->runExhaustiveCursorCommand(opCtx,
+ ReadPreferenceSetting(ReadPreference::PrimaryOnly),
+ nss.db().toString(),
+ listIndexesCmd,
+ Milliseconds(-1));
+ std::vector<BSONObj> indexes;
+ if (indexesRes.getStatus().code() != ErrorCodes::NamespaceNotFound) {
+ indexes = uassertStatusOK(indexesRes).docs;
+ }
+
+ // 1. Verify consistency with existing unique indexes
+ for (const auto& idx : indexes) {
+ BSONObj currentKey = idx["key"].embeddedObject();
+ bool isUnique = idx["unique"].trueValue();
+ uassert(ErrorCodes::InvalidOptions,
+ str::stream() << "can't shard collection '" << nss.ns() << "' with unique index on "
+ << currentKey
+ << " and proposed shard key "
+ << proposedKey
+ << ". Uniqueness can't be maintained unless shard key is a prefix",
+ !isUnique || shardKeyPattern.isUniqueIndexCompatible(currentKey));
+ }
+
+ // 2. Check for a useful index
+ bool hasUsefulIndexForKey = false;
+ for (const auto& idx : indexes) {
+ BSONObj currentKey = idx["key"].embeddedObject();
+ // Check 2.i. and 2.ii.
+ if (!idx["sparse"].trueValue() && idx["filter"].eoo() && idx["collation"].eoo() &&
+ proposedKey.isPrefixOf(currentKey, SimpleBSONElementComparator::kInstance)) {
+ // We can't currently use hashed indexes with a non-default hash seed
+ // Check v.
+ // Note that this means that, for sharding, we only support one hashed index
+ // per field per collection.
+ uassert(ErrorCodes::InvalidOptions,
+ str::stream() << "can't shard collection " << nss.ns()
+ << " with hashed shard key "
+ << proposedKey
+ << " because the hashed index uses a non-default seed of "
+ << idx["seed"].numberInt(),
+ !shardKeyPattern.isHashedPattern() || idx["seed"].eoo() ||
+ idx["seed"].numberInt() == BSONElementHasher::DEFAULT_HASH_SEED);
+ hasUsefulIndexForKey = true;
+ }
+ }
+
+ // 3. If proposed key is required to be unique, additionally check for exact match.
+ if (hasUsefulIndexForKey && unique) {
+ BSONObj eqQuery = BSON("ns" << nss.ns() << "key" << proposedKey);
+ BSONObj eqQueryResult;
+
+ for (const auto& idx : indexes) {
+ if (SimpleBSONObjComparator::kInstance.evaluate(idx["key"].embeddedObject() ==
+ proposedKey)) {
+ eqQueryResult = idx;
+ break;
+ }
+ }
+
+ if (eqQueryResult.isEmpty()) {
+ // If no exact match, index not useful, but still possible to create one later
+ hasUsefulIndexForKey = false;
+ } else {
+ bool isExplicitlyUnique = eqQueryResult["unique"].trueValue();
+ BSONObj currKey = eqQueryResult["key"].embeddedObject();
+ bool isCurrentID = (currKey.firstElementFieldNameStringData() == "_id");
+ uassert(ErrorCodes::InvalidOptions,
+ str::stream() << "can't shard collection " << nss.ns() << ", " << proposedKey
+ << " index not unique, and unique index explicitly specified",
+ isExplicitlyUnique || isCurrentID);
+ }
+ }
+
+ auto countCmd = BSON("count" << nss.coll());
+ auto countRes =
+ uassertStatusOK(primaryShard->runCommand(opCtx,
+ ReadPreferenceSetting(ReadPreference::PrimaryOnly),
+ nss.db().toString(),
+ countCmd,
+ Shard::RetryPolicy::kIdempotent));
+ const bool isEmpty = (countRes.response["n"].Int() == 0);
+
+ if (hasUsefulIndexForKey) {
+ // Check 2.iii and 2.iv. Make sure no null entries in the sharding index
+ // and that there is a useful, non-multikey index available
+ BSONObjBuilder checkShardingIndexCmd;
+ checkShardingIndexCmd.append("checkShardingIndex", nss.ns());
+ checkShardingIndexCmd.append("keyPattern", proposedKey);
+ auto checkShardingIndexRes = uassertStatusOK(
+ primaryShard->runCommand(opCtx,
+ ReadPreferenceSetting(ReadPreference::PrimaryOnly),
+ "admin",
+ checkShardingIndexCmd.obj(),
+ Shard::RetryPolicy::kIdempotent));
+ uassert(ErrorCodes::OperationFailed,
+ checkShardingIndexRes.response["errmsg"].str(),
+ checkShardingIndexRes.commandStatus == Status::OK());
+ } else if (!isEmpty || !createIndexIfPossible) {
+ // 4. If no useful index, and collection is non-empty or createIndexIfPossible is false,
+ // fail
+ uasserted(ErrorCodes::InvalidOptions,
+ "Please create an index that starts with the proposed shard key before "
+ "sharding the collection");
+ } else {
+ // 5. If no useful index exists, and collection empty and createIndexIfPossible is true,
+ // create one on proposedKey. Only need to call ensureIndex on primary shard, since
+ // indexes get copied to receiving shard whenever a migrate occurs. If the collection
+ // has a default collation, explicitly send the simple collation as part of the
+ // createIndex request.
+ invariant(createIndexIfPossible);
+
+ BSONObj collation = !defaultCollation->isEmpty() ? CollationSpec::kSimpleSpec : BSONObj();
+ auto createIndexesCmd = makeCreateIndexesCmd(nss, proposedKey, collation, unique);
+
+ const auto swResponse = primaryShard->runCommandWithFixedRetryAttempts(
+ opCtx,
+ ReadPreferenceSetting(ReadPreference::PrimaryOnly),
+ nss.db().toString(),
+ createIndexesCmd,
+ Shard::RetryPolicy::kNoRetry);
+ auto createIndexesStatus = swResponse.getStatus();
+ if (createIndexesStatus.isOK()) {
+ const auto response = swResponse.getValue();
+ createIndexesStatus = (!response.commandStatus.isOK()) ? response.commandStatus
+ : response.writeConcernStatus;
+ }
+ uassertStatusOK(createIndexesStatus);
+ }
+}
+
+} // namespace shardkeyutil
+} // namespace mongo
diff --git a/src/mongo/db/s/shard_key_util.h b/src/mongo/db/s/shard_key_util.h
new file mode 100644
index 00000000000..62e285816e1
--- /dev/null
+++ b/src/mongo/db/s/shard_key_util.h
@@ -0,0 +1,91 @@
+/**
+ * Copyright (C) 2019-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/s/request_types/shard_collection_gen.h"
+#include "mongo/s/shard_util.h"
+
+namespace mongo {
+namespace shardkeyutil {
+
+/**
+ * Constructs the BSON specification document for the create indexes command using the given
+ * namespace, index key and options.
+ */
+BSONObj makeCreateIndexesCmd(const NamespaceString& nss,
+ const BSONObj& keys,
+ const BSONObj& collation,
+ bool unique);
+
+/**
+ * Compares the proposed shard key with the collection's existing indexes on the primary shard to
+ * ensure they are a legal combination.
+ *
+ * Creates the required index if and only if (i) the collection is empty, (ii) no index on the shard
+ * key exists, and (iii) createIndexIfPossible is true.
+ *
+ * The proposed shard key must be validated against the set of existing indexes.
+ * In particular, we must ensure the following constraints:
+ *
+ * 1. All existing unique indexes, except those which start with the _id index,
+ * must contain the proposed key as a prefix (uniqueness of the _id index is
+ * ensured by the _id generation process or guaranteed by the user).
+ *
+ * 2. If the collection is not empty, there must exist at least one index that
+ * is "useful" for the proposed key. A "useful" index is defined as adhering to
+ * all of the following properties:
+ * i. contains proposedKey as a prefix
+ * ii. is not a sparse index, partial index, or index with a non-simple collation
+ * iii. contains no null values
+ * iv. is not multikey (maybe lift this restriction later)
+ * v. if a hashed index, has default seed (lift this restriction later)
+ *
+ * 3. If the proposed shard key is specified as unique, there must exist a useful,
+ * unique index exactly equal to the proposedKey (not just a prefix).
+ *
+ * After validating these constraints:
+ *
+ * 4. If there is no useful index, and the collection is non-empty or createIndexIfPossible
+ * is false, we must fail.
+ *
+ * 5. If the collection is empty and createIndexIfPossible is true, and it's still possible
+ * to create an index on the proposed key, we go ahead and do so.
+ */
+void validateShardKeyAgainstExistingIndexes(OperationContext* opCtx,
+ const NamespaceString& nss,
+ const BSONObj& proposedKey,
+ const ShardKeyPattern& shardKeyPattern,
+ const std::shared_ptr<Shard>& primaryShard,
+ const boost::optional<BSONObj>& defaultCollation,
+ const bool unique,
+ const bool createIndexIfPossible);
+
+} // namespace shardkeyutil
+} // namespace mongo
diff --git a/src/mongo/s/shard_key_pattern.cpp b/src/mongo/s/shard_key_pattern.cpp
index 395a80a9d98..ca229e13cd4 100644
--- a/src/mongo/s/shard_key_pattern.cpp
+++ b/src/mongo/s/shard_key_pattern.cpp
@@ -226,6 +226,10 @@ bool ShardKeyPattern::isShardKey(const BSONObj& shardKey) const {
return shardKey.nFields() == keyPatternBSON.nFields();
}
+bool ShardKeyPattern::isExtendedBy(const ShardKeyPattern& newShardKeyPattern) const {
+ return toBSON().isFieldNamePrefixOf(newShardKeyPattern.toBSON());
+}
+
BSONObj ShardKeyPattern::normalizeShardKey(const BSONObj& shardKey) const {
// We want to return an empty key if users pass us something that's not a shard key
if (shardKey.nFields() > _keyPattern.toBSON().nFields())
diff --git a/src/mongo/s/shard_key_pattern.h b/src/mongo/s/shard_key_pattern.h
index 42557730e0c..6d177188b13 100644
--- a/src/mongo/s/shard_key_pattern.h
+++ b/src/mongo/s/shard_key_pattern.h
@@ -113,6 +113,12 @@ public:
bool isShardKey(const BSONObj& shardKey) const;
/**
+ * Returns true if the new shard key pattern extends this shard key pattern - i.e. contains this
+ * shard key pattern as a prefix (begins with the same field names in the same order).
+ */
+ bool isExtendedBy(const ShardKeyPattern& newShardKeyPattern) const;
+
+ /**
* Given a shard key, return it in normal form where the fields are in the same order as
* the shard key pattern fields.
*
View Lorry Controller Status