SERVER-63732 Add new implicitlyCreateIndex and enforceUniqueness to shardCollection command

(cherry picked from commit 0f28913257b0d6fee3b92927d7ec16a2ad54a0a0)

9 files changed, 139 insertions, 17 deletions

diff --git a/etc/backports_required_for_multiversion_tests.yml b/etc/backports_required_for_multiversion_tests.yml
index 4b7ec4699e4..1b5de58ba6f 100644
--- a/etc/backports_required_for_multiversion_tests.yml
+++ b/etc/backports_required_for_multiversion_tests.yml
@@ -225,6 +225,8 @@ all:
     test_file: jstests/sharding/write_commands_read_concern_validation.js
   - ticket: SERVER-64741
     test_file: jstests/sharding/append_oplog_note_mongos.js
+  - ticket: SERVER-63732
+    test_file: jstests/sharding/shard_collection_basic.js
 
 suites:
 
@@ -267,3 +269,5 @@ suites:
     test_file: jstests/sharding/query/pipeline_length_limit.js
   - ticket: SERVER-56127
     test_file: jstests/sharding/retryable_writes_nested_shard_key.js
+  - ticket: SERVER-63732
+    test_file: jstests/sharding/shard_collection_basic.js
diff --git a/jstests/sharding/shard_collection_basic.js b/jstests/sharding/shard_collection_basic.js
index 877df390af6..359cb6c94f2 100644
--- a/jstests/sharding/shard_collection_basic.js
+++ b/jstests/sharding/shard_collection_basic.js
@@ -103,6 +103,12 @@ assert.commandFailed(
 assert.commandWorked(
     mongos.adminCommand({shardCollection: kDbName + '.shard_key_dotted_path', key: {'_id.a': 1}}));
 
+jsTestLog('Command should still verify index even if implicitlyCreateIndex is false.');
+assert.commandFailedWithCode(
+    mongos.adminCommand(
+        {shardCollection: kDbName + '.foo', key: {x: 1}, implicitlyCreateIndex: false}),
+    6373200);
+
 //
 // Test shardCollection's idempotency
 //
@@ -124,7 +130,31 @@ assert.commandFailed(
 
 assert.commandWorked(mongos.getDB(kDbName).dropDatabase());
 
-// Shard empty collections no index required.
+jsTestLog('Allow non-unique index if enforceUniquenessCheck is false');
+assert.commandWorked(mongos.getDB(kDbName).foo.createIndex({x: 1}));
+assert.commandWorked(mongos.adminCommand({enableSharding: kDbName}));
+assert.commandWorked(mongos.adminCommand(
+    {shardCollection: kDbName + '.foo', key: {x: 1}, unique: true, enforceUniquenessCheck: false}));
+let collDoc = mongos.getDB('config').collections.findOne({_id: `${kDbName}.foo`});
+assert(collDoc);
+assert(collDoc.unique);
+assert.commandWorked(mongos.getDB(kDbName).dropDatabase());
+
+jsTestLog('mongosync unique key pattern use case');
+assert.commandWorked(mongos.getDB(kDbName).foo.createIndex({x: 1}));
+assert.commandWorked(mongos.adminCommand({enableSharding: kDbName}));
+assert.commandWorked(mongos.adminCommand({
+    shardCollection: kDbName + '.foo',
+    key: {x: 1},
+    unique: true,
+    implicitlyCreateIndex: false,
+    enforceUniquenessCheck: false
+}));
+collDoc = mongos.getDB('config').collections.findOne({_id: `${kDbName}.foo`});
+assert(collDoc);
+assert(collDoc.unique);
+assert.commandWorked(mongos.getDB(kDbName).dropDatabase());
+
 testAndClenaupWithKeyNoIndexOK({_id: 1});
 testAndClenaupWithKeyNoIndexOK({_id: 'hashed'});
 
diff --git a/src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp b/src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp
index 511831c5cb0..5a400678daf 100644
--- a/src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp
+++ b/src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp
@@ -147,6 +147,7 @@ public:
                         newShardKeyPattern,
                         boost::none,
                         collType.getUnique(),
+                        true /* enforceUniquenessCheck */,
                         shardkeyutil::ValidationBehaviorsRefineShardKey(opCtx, nss));
                 });
 
diff --git a/src/mongo/db/s/config/configsvr_shard_collection_command.cpp b/src/mongo/db/s/config/configsvr_shard_collection_command.cpp
index 2a16c5bd554..023c189dfb5 100644
--- a/src/mongo/db/s/config/configsvr_shard_collection_command.cpp
+++ b/src/mongo/db/s/config/configsvr_shard_collection_command.cpp
@@ -346,6 +346,9 @@ public:
         shardsvrShardCollectionRequest.setCollation(request.getCollation());
         shardsvrShardCollectionRequest.setGetUUIDfromPrimaryShard(
             request.getGetUUIDfromPrimaryShard());
+        shardsvrShardCollectionRequest.setImplicitlyCreateIndex(request.getImplicitlyCreateIndex());
+        shardsvrShardCollectionRequest.setEnforceUniquenessCheck(
+            request.getEnforceUniquenessCheck());
 
         auto cmdResponse = uassertStatusOK(primaryShard->runCommandWithFixedRetryAttempts(
             opCtx,
diff --git a/src/mongo/db/s/shard_key_util.cpp b/src/mongo/db/s/shard_key_util.cpp
index 7bbeed2614d..d280273403a 100644
--- a/src/mongo/db/s/shard_key_util.cpp
+++ b/src/mongo/db/s/shard_key_util.cpp
@@ -102,13 +102,13 @@ BSONObj makeCreateIndexesCmd(const NamespaceString& nss,
 
 }  // namespace
 
-void validateShardKeyIndexExistsOrCreateIfPossible(OperationContext* opCtx,
-                                                   const NamespaceString& nss,
-                                                   const BSONObj& proposedKey,
-                                                   const ShardKeyPattern& shardKeyPattern,
-                                                   const boost::optional<BSONObj>& defaultCollation,
-                                                   bool unique,
-                                                   const ShardKeyValidationBehaviors& behaviors) {
+bool validShardKeyIndexExists(OperationContext* opCtx,
+                              const NamespaceString& nss,
+                              const BSONObj& proposedKey,
+                              const ShardKeyPattern& shardKeyPattern,
+                              const boost::optional<BSONObj>& defaultCollation,
+                              bool requiresUnique,
+                              const ShardKeyValidationBehaviors& behaviors) {
     auto indexes = behaviors.loadIndexes(nss);
 
     // 1.  Verify consistency with existing unique indexes
@@ -145,7 +145,7 @@ void validateShardKeyIndexExistsOrCreateIfPossible(OperationContext* opCtx,
     }
 
     // 3. If proposed key is required to be unique, additionally check for exact match.
-    if (hasUsefulIndexForKey && unique) {
+    if (hasUsefulIndexForKey && requiresUnique) {
         BSONObj eqQuery = BSON("ns" << nss.ns() << "key" << proposedKey);
         BSONObj eqQueryResult;
 
@@ -174,6 +174,26 @@ void validateShardKeyIndexExistsOrCreateIfPossible(OperationContext* opCtx,
     if (hasUsefulIndexForKey) {
         // Check 2.iii Make sure that there is a useful, non-multikey index available.
         behaviors.verifyUsefulNonMultiKeyIndex(nss, proposedKey);
+    }
+
+    return hasUsefulIndexForKey;
+}
+
+void validateShardKeyIndexExistsOrCreateIfPossible(OperationContext* opCtx,
+                                                   const NamespaceString& nss,
+                                                   const BSONObj& proposedKey,
+                                                   const ShardKeyPattern& shardKeyPattern,
+                                                   const boost::optional<BSONObj>& defaultCollation,
+                                                   bool unique,
+                                                   bool enforceUniquenessCheck,
+                                                   const ShardKeyValidationBehaviors& behaviors) {
+    if (validShardKeyIndexExists(opCtx,
+                                 nss,
+                                 proposedKey,
+                                 shardKeyPattern,
+                                 defaultCollation,
+                                 unique && enforceUniquenessCheck,
+                                 behaviors)) {
         return;
     }
 
diff --git a/src/mongo/db/s/shard_key_util.h b/src/mongo/db/s/shard_key_util.h
index 494ef5bce04..d12a044a3ce 100644
--- a/src/mongo/db/s/shard_key_util.h
+++ b/src/mongo/db/s/shard_key_util.h
@@ -147,7 +147,23 @@ void validateShardKeyIndexExistsOrCreateIfPossible(OperationContext* opCtx,
                                                    const ShardKeyPattern& shardKeyPattern,
                                                    const boost::optional<BSONObj>& defaultCollation,
                                                    bool unique,
+                                                   bool enforceUniquenessCheck,
                                                    const ShardKeyValidationBehaviors& behaviors);
 
+/**
+ * Compares the proposed shard key with the collection's existing indexes to ensure they are a legal
+ * combination.
+ *
+ * Returns true if the shard key is valid and already exists. Steps 1, 2 and 3 of the previous
+ * function.
+ */
+bool validShardKeyIndexExists(OperationContext* opCtx,
+                              const NamespaceString& nss,
+                              const BSONObj& proposedKey,
+                              const ShardKeyPattern& shardKeyPattern,
+                              const boost::optional<BSONObj>& defaultCollation,
+                              bool requiresUnique,
+                              const ShardKeyValidationBehaviors& behaviors);
+
 }  // namespace shardkeyutil
 }  // namespace mongo
diff --git a/src/mongo/db/s/shardsvr_shard_collection.cpp b/src/mongo/db/s/shardsvr_shard_collection.cpp
index ca403aa385f..91cf4c92371 100644
--- a/src/mongo/db/s/shardsvr_shard_collection.cpp
+++ b/src/mongo/db/s/shardsvr_shard_collection.cpp
@@ -327,14 +327,28 @@ ShardCollectionTargetState calculateTargetState(OperationContext* opCtx,
     auto proposedKey(request.getKey().getOwned());
     ShardKeyPattern shardKeyPattern(proposedKey);
 
-    shardkeyutil::validateShardKeyIndexExistsOrCreateIfPossible(
-        opCtx,
-        nss,
-        proposedKey,
-        shardKeyPattern,
-        request.getCollation(),
-        request.getUnique(),
-        shardkeyutil::ValidationBehaviorsShardCollection(opCtx));
+    if (request.getImplicitlyCreateIndex()) {
+        shardkeyutil::validateShardKeyIndexExistsOrCreateIfPossible(
+            opCtx,
+            nss,
+            proposedKey,
+            shardKeyPattern,
+            request.getCollation(),
+            request.getUnique(),
+            request.getEnforceUniquenessCheck(),
+            shardkeyutil::ValidationBehaviorsShardCollection(opCtx));
+    } else {
+        uassert(6373200,
+                "Must have an index compatible with the proposed shard key",
+                shardkeyutil::validShardKeyIndexExists(
+                    opCtx,
+                    nss,
+                    proposedKey,
+                    shardKeyPattern,
+                    request.getCollation(),
+                    request.getUnique() && request.getEnforceUniquenessCheck(),
+                    shardkeyutil::ValidationBehaviorsShardCollection(opCtx)));
+    }
 
     // Wait until the index is majority written, to prevent having the collection commited to the
     // config server, but the index creation rolled backed on stepdowns.
diff --git a/src/mongo/s/commands/cluster_shard_collection_cmd.cpp b/src/mongo/s/commands/cluster_shard_collection_cmd.cpp
index c0f311de262..e821933293e 100644
--- a/src/mongo/s/commands/cluster_shard_collection_cmd.cpp
+++ b/src/mongo/s/commands/cluster_shard_collection_cmd.cpp
@@ -117,6 +117,10 @@ public:
         configShardCollRequest.setNumInitialChunks(shardCollRequest.getNumInitialChunks());
         configShardCollRequest.setPresplitHashedZones(shardCollRequest.getPresplitHashedZones());
         configShardCollRequest.setCollation(shardCollRequest.getCollation());
+        configShardCollRequest.setImplicitlyCreateIndex(
+            shardCollRequest.getImplicitlyCreateIndex());
+        configShardCollRequest.setEnforceUniquenessCheck(
+            shardCollRequest.getEnforceUniquenessCheck());
 
         // Invalidate the routing table cache entry for this collection so that we reload the
         // collection the next time it's accessed, even if we receive a failure, e.g. NetworkError.
diff --git a/src/mongo/s/request_types/shard_collection.idl b/src/mongo/s/request_types/shard_collection.idl
index 5df76c033e7..f9591a810e0 100644
--- a/src/mongo/s/request_types/shard_collection.idl
+++ b/src/mongo/s/request_types/shard_collection.idl
@@ -66,6 +66,16 @@ structs:
                 type: object
                 description: "The collation to use for the shard key index."
                 optional: true
+            implicitlyCreateIndex:
+                description: "Creates an index on the shard key pattern if the collection is empty."
+                type: bool
+                default: true
+            enforceUniquenessCheck:
+                description: >-
+                    Controls whether this command verifies that any unique indexes are prefixed by the shard
+                    key pattern if unique is true. If true then it will verify and if false then it won't.
+                type: bool
+                default: true
 
     ConfigsvrShardCollectionRequest:
         description: "The request format of the internal shardCollection command on the config server"
@@ -102,6 +112,16 @@ structs:
                 type: bool
                 description: "Whether the collection should be created on the primary shard. This should only be false when used in mapReduce."
                 default: true
+            implicitlyCreateIndex:
+                description: "Creates an index on the shard key pattern if the collection is empty."
+                type: bool
+                default: true
+            enforceUniquenessCheck:
+                description: >-
+                    Controls whether this command verifies that any unique indexes are prefixed by the shard
+                    key pattern if unique is true. If true then it will verify and if false then it won't.
+                type: bool
+                default: true
 
     ConfigsvrShardCollectionResponse:
         description: "The response format of the internal shardCollection command on the config server"
@@ -150,6 +170,16 @@ structs:
                 type: bool
                 description: "Whether the collection should be created on the primary shard. This should only be false when used in mapReduce."
                 default: true
+            implicitlyCreateIndex:
+                description: "Creates an index on the shard key pattern if the collection is empty."
+                type: bool
+                default: true
+            enforceUniquenessCheck:
+                description: >-
+                    Controls whether this command verifies that any unique indexes are prefixed by the shard
+                    key pattern if unique is true. If true then it will verify and if false then it won't.
+                type: bool
+                default: true
 
     ShardsvrShardCollectionResponse:
         description: "The response format of the internal shardCollection command on the primary shard"