SERVER-63732 Add new implicitlyCreateIndex and enforceUniqueness to shardCollection command

(cherry picked from commit 0f28913257b0d6fee3b92927d7ec16a2ad54a0a0)

12 files changed, 123 insertions, 21 deletions

diff --git a/etc/backports_required_for_multiversion_tests.yml b/etc/backports_required_for_multiversion_tests.yml
index c88c366674e..fbfb8a2cc39 100644
--- a/etc/backports_required_for_multiversion_tests.yml
+++ b/etc/backports_required_for_multiversion_tests.yml
@@ -110,6 +110,8 @@ last-continuous:
       test_file: jstests/sharding/write_commands_read_concern_validation.js
     - ticket: SERVER-64741
       test_file: jstests/sharding/append_oplog_note_mongos.js
+    - ticket: SERVER-63732
+      test_file: jstests/sharding/shard_collection_basic.js
 
   # Tests that should only be excluded from particular suites should be listed under that suite.
   suites:
@@ -380,6 +382,8 @@ last-lts:
       test_file: jstests/sharding/write_commands_read_concern_validation.js
     - ticket: SERVER-64741
       test_file: jstests/sharding/append_oplog_note_mongos.js
+    - ticket: SERVER-63732
+      test_file: jstests/sharding/shard_collection_basic.js
 
   # Tests that should only be excluded from particular suites should be listed under that suite.
   suites:
diff --git a/jstests/sharding/shard_collection_basic.js b/jstests/sharding/shard_collection_basic.js
index d21fa1d2607..ccefece75f7 100644
--- a/jstests/sharding/shard_collection_basic.js
+++ b/jstests/sharding/shard_collection_basic.js
@@ -105,6 +105,12 @@ jsTestLog('Shard key can contain dotted path to embedded element.');
 assert.commandWorked(
     mongos.adminCommand({shardCollection: kDbName + '.shard_key_dotted_path', key: {'_id.a': 1}}));
 
+jsTestLog('Command should still verify index even if implicitlyCreateIndex is false.');
+assert.commandFailedWithCode(
+    mongos.adminCommand(
+        {shardCollection: kDbName + '.foo', key: {x: 1}, implicitlyCreateIndex: false}),
+    6373200);
+
 //
 // Test shardCollection's idempotency
 //
@@ -127,6 +133,31 @@ assert.commandFailed(
 
 assert.commandWorked(mongos.getDB(kDbName).dropDatabase());
 
+jsTestLog('Allow non-unique index if enforceUniquenessCheck is false');
+assert.commandWorked(mongos.adminCommand({enableSharding: kDbName}));
+assert.commandWorked(mongos.getDB(kDbName).foo.createIndex({x: 1}));
+assert.commandWorked(mongos.adminCommand(
+    {shardCollection: kDbName + '.foo', key: {x: 1}, unique: true, enforceUniquenessCheck: false}));
+let collDoc = mongos.getDB('config').collections.findOne({_id: `${kDbName}.foo`});
+assert(collDoc);
+assert(collDoc.unique);
+assert.commandWorked(mongos.getDB(kDbName).dropDatabase());
+
+jsTestLog('mongosync unique key pattern use case');
+assert.commandWorked(mongos.adminCommand({enableSharding: kDbName}));
+assert.commandWorked(mongos.getDB(kDbName).foo.createIndex({x: 1}));
+assert.commandWorked(mongos.adminCommand({
+    shardCollection: kDbName + '.foo',
+    key: {x: 1},
+    unique: true,
+    implicitlyCreateIndex: false,
+    enforceUniquenessCheck: false
+}));
+collDoc = mongos.getDB('config').collections.findOne({_id: `${kDbName}.foo`});
+assert(collDoc);
+assert(collDoc.unique);
+assert.commandWorked(mongos.getDB(kDbName).dropDatabase());
+
 jsTestLog('Shard empty collections no index required.');
 testAndClenaupWithKeyNoIndexOK({_id: 1});
 testAndClenaupWithKeyNoIndexOK({_id: 'hashed'});
diff --git a/src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp b/src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp
index 58674d0212b..e4be3f748a3 100644
--- a/src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp
+++ b/src/mongo/db/s/config/configsvr_refine_collection_shard_key_command.cpp
@@ -164,6 +164,7 @@ public:
                                       newShardKeyPattern,
                                       boost::none,
                                       collType.getUnique(),
+                                      true /* enforceUniquenessCheck */,
                                       shardkeyutil::ValidationBehaviorsRefineShardKey(opCtx, nss));
                               });
 
diff --git a/src/mongo/db/s/create_collection_coordinator.cpp b/src/mongo/db/s/create_collection_coordinator.cpp
index 5da200c20b6..a80bacde92f 100644
--- a/src/mongo/db/s/create_collection_coordinator.cpp
+++ b/src/mongo/db/s/create_collection_coordinator.cpp
@@ -679,13 +679,27 @@ void CreateCollectionCoordinator::_createCollectionAndIndexes(OperationContext*
         }
     }
 
-    const auto indexCreated = shardkeyutil::validateShardKeyIndexExistsOrCreateIfPossible(
-        opCtx,
-        nss(),
-        *_shardKeyPattern,
-        _collationBSON,
-        _request.getUnique().value_or(false),
-        shardkeyutil::ValidationBehaviorsShardCollection(opCtx));
+    auto indexCreated = false;
+    if (_doc.getImplicitlyCreateIndex()) {
+        indexCreated = shardkeyutil::validateShardKeyIndexExistsOrCreateIfPossible(
+            opCtx,
+            nss(),
+            *_shardKeyPattern,
+            _collationBSON,
+            _doc.getUnique().value_or(false),
+            _doc.getEnforceUniquenessCheck(),
+            shardkeyutil::ValidationBehaviorsShardCollection(opCtx));
+    } else {
+        uassert(6373200,
+                "Must have an index compatible with the proposed shard key",
+                validShardKeyIndexExists(opCtx,
+                                         nss(),
+                                         *_shardKeyPattern,
+                                         _collationBSON,
+                                         _doc.getUnique().value_or(false) &&
+                                             _doc.getEnforceUniquenessCheck(),
+                                         shardkeyutil::ValidationBehaviorsShardCollection(opCtx)));
+    }
 
     auto replClientInfo = repl::ReplClientInfo::forClient(opCtx->getClient());
 
diff --git a/src/mongo/db/s/resharding/resharding_recipient_service.cpp b/src/mongo/db/s/resharding/resharding_recipient_service.cpp
index e364494571c..ff6442a6b12 100644
--- a/src/mongo/db/s/resharding/resharding_recipient_service.cpp
+++ b/src/mongo/db/s/resharding/resharding_recipient_service.cpp
@@ -532,6 +532,7 @@ void ReshardingRecipientService::RecipientStateMachine::
                     ShardKeyPattern{_metadata.getReshardingKey()},
                     CollationSpec::kSimpleSpec,
                     false /* unique */,
+                    true /* enforceUniquenessCheck */,
                     shardkeyutil::ValidationBehaviorsShardCollection(opCtx.get()));
             });
     }
diff --git a/src/mongo/db/s/shard_collection_legacy.cpp b/src/mongo/db/s/shard_collection_legacy.cpp
index 8848ff3b5d8..6157140b802 100644
--- a/src/mongo/db/s/shard_collection_legacy.cpp
+++ b/src/mongo/db/s/shard_collection_legacy.cpp
@@ -572,14 +572,27 @@ CreateCollectionResponse shardCollection(OperationContext* opCtx,
 
         checkCollation(opCtx, request);
 
-        // Create the collection locally
-        shardkeyutil::validateShardKeyIndexExistsOrCreateIfPossible(
-            opCtx,
-            nss,
-            ShardKeyPattern(request.getKey()),
-            *request.getCollation(),
-            request.getUnique(),
-            shardkeyutil::ValidationBehaviorsShardCollection(opCtx));
+        if (request.getImplicitlyCreateIndex()) {
+            // Create the collection locally
+            shardkeyutil::validateShardKeyIndexExistsOrCreateIfPossible(
+                opCtx,
+                nss,
+                ShardKeyPattern(request.getKey()),
+                *request.getCollation(),
+                request.getUnique(),
+                request.getEnforceUniquenessCheck(),
+                shardkeyutil::ValidationBehaviorsShardCollection(opCtx));
+        } else {
+            uassert(
+                6373201,
+                "Must have an index compatible with the proposed shard key",
+                validShardKeyIndexExists(opCtx,
+                                         nss,
+                                         ShardKeyPattern(request.getKey()),
+                                         *request.getCollation(),
+                                         request.getUnique() && request.getEnforceUniquenessCheck(),
+                                         shardkeyutil::ValidationBehaviorsShardCollection(opCtx)));
+        }
 
         // Wait until the index is majority written, to prevent having the collection commited to
         // the config server, but the index creation rolled backed on stepdowns.
diff --git a/src/mongo/db/s/shard_key_util.cpp b/src/mongo/db/s/shard_key_util.cpp
index 8641111c168..f672c52e9eb 100644
--- a/src/mongo/db/s/shard_key_util.cpp
+++ b/src/mongo/db/s/shard_key_util.cpp
@@ -104,7 +104,7 @@ bool validShardKeyIndexExists(OperationContext* opCtx,
                               const NamespaceString& nss,
                               const ShardKeyPattern& shardKeyPattern,
                               const boost::optional<BSONObj>& defaultCollation,
-                              bool unique,
+                              bool requiresUnique,
                               const ShardKeyValidationBehaviors& behaviors) {
     auto indexes = behaviors.loadIndexes(nss);
 
@@ -144,7 +144,7 @@ bool validShardKeyIndexExists(OperationContext* opCtx,
     }
 
     // 3. If proposed key is required to be unique, additionally check for exact match.
-    if (hasUsefulIndexForKey && unique) {
+    if (hasUsefulIndexForKey && requiresUnique) {
         BSONObj eqQuery = BSON("ns" << nss.ns() << "key" << shardKeyPattern.toBSON());
         BSONObj eqQueryResult;
 
@@ -184,13 +184,17 @@ bool validateShardKeyIndexExistsOrCreateIfPossible(OperationContext* opCtx,
                                                    const ShardKeyPattern& shardKeyPattern,
                                                    const boost::optional<BSONObj>& defaultCollation,
                                                    bool unique,
+                                                   bool enforceUniquenessCheck,
                                                    const ShardKeyValidationBehaviors& behaviors) {
-    if (validShardKeyIndexExists(
-            opCtx, nss, shardKeyPattern, defaultCollation, unique, behaviors)) {
+    if (validShardKeyIndexExists(opCtx,
+                                 nss,
+                                 shardKeyPattern,
+                                 defaultCollation,
+                                 unique && enforceUniquenessCheck,
+                                 behaviors)) {
         return false;
     }
 
-
     // 4. If no useful index, verify we can create one.
     behaviors.verifyCanCreateShardKeyIndex(nss);
 
diff --git a/src/mongo/db/s/shard_key_util.h b/src/mongo/db/s/shard_key_util.h
index cbce71b0540..a2cd71a4bd4 100644
--- a/src/mongo/db/s/shard_key_util.h
+++ b/src/mongo/db/s/shard_key_util.h
@@ -150,6 +150,7 @@ bool validateShardKeyIndexExistsOrCreateIfPossible(OperationContext* opCtx,
                                                    const ShardKeyPattern& shardKeyPattern,
                                                    const boost::optional<BSONObj>& defaultCollation,
                                                    bool unique,
+                                                   bool enforceUniquenessCheck,
                                                    const ShardKeyValidationBehaviors& behaviors);
 /**
  * Compares the proposed shard key with the collection's existing indexes to ensure they are a legal
@@ -163,7 +164,7 @@ bool validShardKeyIndexExists(OperationContext* opCtx,
                               const NamespaceString& nss,
                               const ShardKeyPattern& shardKeyPattern,
                               const boost::optional<BSONObj>& defaultCollation,
-                              bool unique,
+                              bool requiresUnique,
                               const ShardKeyValidationBehaviors& behaviors);
 }  // namespace shardkeyutil
 }  // namespace mongo
diff --git a/src/mongo/s/commands/cluster_commands.idl b/src/mongo/s/commands/cluster_commands.idl
index 588daf66ee0..ef4c95c21b1 100644
--- a/src/mongo/s/commands/cluster_commands.idl
+++ b/src/mongo/s/commands/cluster_commands.idl
@@ -101,3 +101,13 @@ structs:
         description: "The options to create the time-series collection with."
         type: TimeseriesOptions
         optional: true
+      implicitlyCreateIndex:
+        description: "Creates an index on the shard key pattern if the collection is empty."
+        type: bool
+        default: true
+      enforceUniquenessCheck:
+        description: >-
+          Controls whether this command verifies that any unique indexes are prefixed by the shard
+          key pattern if unique is true. If true then it will verify and if false then it won't.
+        type: bool
+        default: true
diff --git a/src/mongo/s/commands/cluster_shard_collection_cmd.cpp b/src/mongo/s/commands/cluster_shard_collection_cmd.cpp
index 0a73c45f035..9f450c29a5e 100644
--- a/src/mongo/s/commands/cluster_shard_collection_cmd.cpp
+++ b/src/mongo/s/commands/cluster_shard_collection_cmd.cpp
@@ -103,6 +103,8 @@ public:
         requestParamsObj.setPresplitHashedZones(shardCollRequest.getPresplitHashedZones());
         requestParamsObj.setCollation(shardCollRequest.getCollation());
         requestParamsObj.setTimeseries(shardCollRequest.getTimeseries());
+        requestParamsObj.setImplicitlyCreateIndex(shardCollRequest.getImplicitlyCreateIndex());
+        requestParamsObj.setEnforceUniquenessCheck(shardCollRequest.getEnforceUniquenessCheck());
         shardsvrCollRequest.setCreateCollectionRequest(std::move(requestParamsObj));
         shardsvrCollRequest.setDbName(nss.db());
 
diff --git a/src/mongo/s/request_types/shard_collection.idl b/src/mongo/s/request_types/shard_collection.idl
index 6440111a61b..3c9f0124de5 100644
--- a/src/mongo/s/request_types/shard_collection.idl
+++ b/src/mongo/s/request_types/shard_collection.idl
@@ -118,6 +118,16 @@ structs:
                 type: bool
                 description: "Whether the collection should be created on the primary shard. This should only be false when used in mapReduce."
                 default: true
+            implicitlyCreateIndex:
+                description: "Creates an index on the shard key pattern if the collection is empty."
+                type: bool
+                default: true
+            enforceUniquenessCheck:
+                description: >-
+                    Controls whether this command verifies that any unique indexes are prefixed by the shard
+                    key pattern if unique is true. If true then it will verify and if false then it won't.
+                type: bool
+                default: true
 
     ShardsvrShardCollectionResponse:
         description: "The response format of the internal shardCollection command on the primary shard"
diff --git a/src/mongo/s/request_types/sharded_ddl_commands.idl b/src/mongo/s/request_types/sharded_ddl_commands.idl
index c6d74422e62..3e4ed4caefb 100644
--- a/src/mongo/s/request_types/sharded_ddl_commands.idl
+++ b/src/mongo/s/request_types/sharded_ddl_commands.idl
@@ -126,6 +126,17 @@ structs:
                 type: object_owned
                 description: "The collation to use for the shard key index."
                 optional: true
+            implicitlyCreateIndex:
+                description: "Creates an index on the shard key pattern if the collection is empty."
+                type: bool
+                default: true
+            enforceUniquenessCheck:
+                description: >-
+                    Controls whether this command verifies that any unique indexes are prefixed by
+                    the shard key pattern if unique is true. If true then it will verify and if
+                    false then it won't.
+                type: bool
+                default: true
 
     CreateCollectionResponse:
         description: "Response of the create collection command"