SERVER-63732 Add new implicitlyCreateIndex and enforceUniqueness to shardCollection command

(cherry picked from commit 411a12ad600733fc80d9b6e8da86da51dd415ab0)

7 files changed, 106 insertions, 12 deletions

diff --git a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
index 9a617e122de..e42db8a5336 100644
--- a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
@@ -189,6 +189,8 @@ selector:
   - jstests/sharding/repair_sharded_collection_history.js
   # Will fail on 4.0 because the mongos version of appendOplogNote only exists on 4.2.
   - jstests/sharding/append_oplog_note_mongos.js
+  # Remove when SERVER-63732 is backported to 4.0.
+  - jstests/sharding/shard_collection_basic.js
 
   exclude_with_any_tags:
   - multiversion_incompatible
diff --git a/etc/backports_required_for_multiversion_tests.yml b/etc/backports_required_for_multiversion_tests.yml
index 06e463e78b8..e8275f522c2 100644
--- a/etc/backports_required_for_multiversion_tests.yml
+++ b/etc/backports_required_for_multiversion_tests.yml
@@ -148,7 +148,8 @@ all:
     test_file: jstests/core/internal_apply_oplog_update.js
   - ticket: SERVER-64741
     test_file: jstests/sharding/append_oplog_note_mongos.js
-
+  - ticket: SERVER-63732
+    test_file: jstests/sharding/shard_collection_basic.js
 
 # Tests that should only be excluded from particular suites should be listed under that suite.
 suites:
@@ -188,3 +189,5 @@ suites:
   sharding_multiversion:
   - ticket: SERVER-56127
     test_file: jstests/sharding/retryable_writes_nested_shard_key.js
+  - ticket: SERVER-63732
+    test_file: jstests/sharding/shard_collection_basic.js
diff --git a/jstests/sharding/shard_collection_basic.js b/jstests/sharding/shard_collection_basic.js
index 84a2ac9b5c9..cf8ac025293 100644
--- a/jstests/sharding/shard_collection_basic.js
+++ b/jstests/sharding/shard_collection_basic.js
@@ -103,6 +103,12 @@ assert.commandFailed(
 assert.commandWorked(
     mongos.adminCommand({shardCollection: kDbName + '.shard_key_dotted_path', key: {'_id.a': 1}}));
 
+jsTestLog('Command should still verify index even if implicitlyCreateIndex is false.');
+assert.commandFailedWithCode(
+    mongos.adminCommand(
+        {shardCollection: kDbName + '.foo', key: {x: 1}, implicitlyCreateIndex: false}),
+    6373200);
+
 //
 // Test shardCollection's idempotency
 //
@@ -124,7 +130,31 @@ assert.commandFailed(
 
 assert.commandWorked(mongos.getDB(kDbName).dropDatabase());
 
-// Shard empty collections no index required.
+jsTestLog('Allow non-unique index if enforceUniquenessCheck is false');
+assert.commandWorked(mongos.getDB(kDbName).foo.createIndex({x: 1}));
+assert.commandWorked(mongos.adminCommand({enableSharding: kDbName}));
+assert.commandWorked(mongos.adminCommand(
+    {shardCollection: kDbName + '.foo', key: {x: 1}, unique: true, enforceUniquenessCheck: false}));
+let collDoc = mongos.getDB('config').collections.findOne({_id: `${kDbName}.foo`});
+assert(collDoc);
+assert(collDoc.unique);
+assert.commandWorked(mongos.getDB(kDbName).dropDatabase());
+
+jsTestLog('mongosync unique key pattern use case');
+assert.commandWorked(mongos.getDB(kDbName).foo.createIndex({x: 1}));
+assert.commandWorked(mongos.adminCommand({enableSharding: kDbName}));
+assert.commandWorked(mongos.adminCommand({
+    shardCollection: kDbName + '.foo',
+    key: {x: 1},
+    unique: true,
+    implicitlyCreateIndex: false,
+    enforceUniquenessCheck: false
+}));
+collDoc = mongos.getDB('config').collections.findOne({_id: `${kDbName}.foo`});
+assert(collDoc);
+assert(collDoc.unique);
+assert.commandWorked(mongos.getDB(kDbName).dropDatabase());
+
 testAndClenaupWithKeyNoIndexOK({_id: 1});
 testAndClenaupWithKeyNoIndexOK({_id: 'hashed'});
 
diff --git a/src/mongo/db/s/config/configsvr_shard_collection_command.cpp b/src/mongo/db/s/config/configsvr_shard_collection_command.cpp
index 0ca2d61603a..d75da55cc0b 100644
--- a/src/mongo/db/s/config/configsvr_shard_collection_command.cpp
+++ b/src/mongo/db/s/config/configsvr_shard_collection_command.cpp
@@ -731,6 +731,9 @@ public:
         shardsvrShardCollectionRequest.setCollation(request.getCollation());
         shardsvrShardCollectionRequest.setGetUUIDfromPrimaryShard(
             request.getGetUUIDfromPrimaryShard());
+        shardsvrShardCollectionRequest.setImplicitlyCreateIndex(request.getImplicitlyCreateIndex());
+        shardsvrShardCollectionRequest.setEnforceUniquenessCheck(
+            request.getEnforceUniquenessCheck());
 
         auto cmdResponse = uassertStatusOK(primaryShard->runCommandWithFixedRetryAttempts(
             opCtx,
diff --git a/src/mongo/db/s/shardsvr_shard_collection.cpp b/src/mongo/db/s/shardsvr_shard_collection.cpp
index 2727d0e192f..f0d6267c844 100644
--- a/src/mongo/db/s/shardsvr_shard_collection.cpp
+++ b/src/mongo/db/s/shardsvr_shard_collection.cpp
@@ -185,14 +185,13 @@ BSONObj makeCreateIndexesCmd(const NamespaceString& nss,
 /**
  * Compares the proposed shard key with the collection's existing indexes on the primary shard to
  * ensure they are a legal combination.
- *
- * If the collection is empty and no index on the shard key exists, creates the required index.
  */
-void createIndexesOrValidateExisting(OperationContext* opCtx,
-                                     const NamespaceString& nss,
-                                     const BSONObj& proposedKey,
-                                     const ShardKeyPattern& shardKeyPattern,
-                                     const ShardsvrShardCollection& request) {
+bool validShardKeyIndexExists(OperationContext* opCtx,
+                              const NamespaceString& nss,
+                              const BSONObj& proposedKey,
+                              const ShardKeyPattern& shardKeyPattern,
+                              const ShardsvrShardCollection& request) {
+
     // The proposed shard key must be validated against the set of existing indexes.
     // In particular, we must ensure the following constraints
     //
@@ -257,7 +256,7 @@ void createIndexesOrValidateExisting(OperationContext* opCtx,
 
     // 3. If proposed key is required to be unique, additionally check for exact match.
 
-    if (hasUsefulIndexForKey && request.getUnique()) {
+    if (hasUsefulIndexForKey && request.getUnique() && request.getEnforceUniquenessCheck()) {
         BSONObj eqQuery = BSON("ns" << nss.ns() << "key" << proposedKey);
         BSONObj eqQueryResult;
 
@@ -283,7 +282,23 @@ void createIndexesOrValidateExisting(OperationContext* opCtx,
         }
     }
 
-    if (hasUsefulIndexForKey) {
+    return hasUsefulIndexForKey;
+}
+
+/**
+ * Compares the proposed shard key with the collection's existing indexes on the primary shard to
+ * ensure they are a legal combination.
+ *
+ * If the collection is empty and no index on the shard key exists, creates the required index.
+ */
+void createIndexesOrValidateExisting(OperationContext* opCtx,
+                                     const NamespaceString& nss,
+                                     const BSONObj& proposedKey,
+                                     const ShardKeyPattern& shardKeyPattern,
+                                     const ShardsvrShardCollection& request) {
+    DBDirectClient localClient(opCtx);
+
+    if (validShardKeyIndexExists(opCtx, nss, proposedKey, shardKeyPattern, request)) {
         // Check 2.iii and 2.iv. Make sure no null entries in the sharding index
         // and that there is a useful, non-multikey index available
         BSONObjBuilder checkShardingIndexCmd;
@@ -707,7 +722,14 @@ UUID shardCollection(OperationContext* opCtx,
 
     const auto proposedKey(request.getKey().getOwned());
     const ShardKeyPattern shardKeyPattern(proposedKey);
-    createIndexesOrValidateExisting(opCtx, nss, proposedKey, shardKeyPattern, request);
+
+    if (request.getImplicitlyCreateIndex()) {
+        createIndexesOrValidateExisting(opCtx, nss, proposedKey, shardKeyPattern, request);
+    } else {
+        uassert(6373200,
+                "Must have an index compatible with the proposed shard key",
+                validShardKeyIndexExists(opCtx, nss, proposedKey, shardKeyPattern, request));
+    }
 
     {
         // From this point onward the collection can only be read, not written to, so it is safe to
diff --git a/src/mongo/s/commands/cluster_shard_collection_cmd.cpp b/src/mongo/s/commands/cluster_shard_collection_cmd.cpp
index d5c23dcacc7..9bbd90f78ef 100644
--- a/src/mongo/s/commands/cluster_shard_collection_cmd.cpp
+++ b/src/mongo/s/commands/cluster_shard_collection_cmd.cpp
@@ -117,6 +117,10 @@ public:
         configShardCollRequest.setUnique(shardCollRequest.getUnique());
         configShardCollRequest.setNumInitialChunks(shardCollRequest.getNumInitialChunks());
         configShardCollRequest.setCollation(shardCollRequest.getCollation());
+        configShardCollRequest.setImplicitlyCreateIndex(
+            shardCollRequest.getImplicitlyCreateIndex());
+        configShardCollRequest.setEnforceUniquenessCheck(
+            shardCollRequest.getEnforceUniquenessCheck());
 
         // Invalidate the routing table cache entry for this collection so that we reload the
         // collection the next time it's accessed, even if we receive a failure, e.g. NetworkError.
diff --git a/src/mongo/s/request_types/shard_collection.idl b/src/mongo/s/request_types/shard_collection.idl
index 5a1e9229d38..c35f9e1445b 100644
--- a/src/mongo/s/request_types/shard_collection.idl
+++ b/src/mongo/s/request_types/shard_collection.idl
@@ -62,6 +62,16 @@ structs:
                 type: object
                 description: "The collation to use for the shard key index."
                 optional: true
+            implicitlyCreateIndex:
+                description: "Creates an index on the shard key pattern if the collection is empty."
+                type: bool
+                default: true
+            enforceUniquenessCheck:
+                description: >-
+                    Controls whether this command verifies that any unique indexes are prefixed by the shard
+                    key pattern if unique is true. If true then it will verify and if false then it won't.
+                type: bool
+                default: true
 
     ConfigsvrShardCollectionRequest:
         description: "The request format of the internal shardCollection command on the config server"
@@ -94,6 +104,16 @@ structs:
                 type: bool
                 description: "Whether the collection should be created on the primary shard. This should only be false when used in mapReduce."
                 default: true
+            implicitlyCreateIndex:
+                description: "Creates an index on the shard key pattern if the collection is empty."
+                type: bool
+                default: true
+            enforceUniquenessCheck:
+                description: >-
+                    Controls whether this command verifies that any unique indexes are prefixed by the shard
+                    key pattern if unique is true. If true then it will verify and if false then it won't.
+                type: bool
+                default: true
 
     ConfigsvrShardCollectionResponse:
         description: "The response format of the internal shardCollection command on the config server"
@@ -138,6 +158,16 @@ structs:
                 type: bool
                 description: "Whether the collection should be created on the primary shard. This should only be false when used in mapReduce."
                 default: true
+            implicitlyCreateIndex:
+                description: "Creates an index on the shard key pattern if the collection is empty."
+                type: bool
+                default: true
+            enforceUniquenessCheck:
+                description: >-
+                    Controls whether this command verifies that any unique indexes are prefixed by the shard
+                    key pattern if unique is true. If true then it will verify and if false then it won't.
+                type: bool
+                default: true
 
     ShardsvrShardCollectionResponse:
         description: "The response format of the internal shardCollection command on the primary shard"