SERVER-61009 Make createIndex a no-op on a cluster key and allow 'clustered' on cluster key

5 files changed, 318 insertions, 20 deletions

diff --git a/jstests/core/clustered_collection_create_index_clustered.js b/jstests/core/clustered_collection_create_index_clustered.js
new file mode 100644
index 00000000000..3aefaa8821b
--- /dev/null
+++ b/jstests/core/clustered_collection_create_index_clustered.js
@@ -0,0 +1,36 @@
+/**
+ * Tests createIndexes with the 'clustered' option on a replicated collection. Note: there are
+ * different restrictions for non-replicated versus replicated clustered collections - eg replicated
+ * collections can only be created with cluster key _id whereas non-replicated collections can be
+ * created with arbitrary single field cluster keys.
+ *
+ * @tags: [
+ *   requires_fcv_52,
+ *   assumes_no_implicit_collection_creation_after_drop,
+ *   # Does not support sharding
+ *   assumes_against_mongod_not_mongos,
+ * ]
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/clustered_collections/clustered_collection_util.js");
+load("jstests/libs/clustered_collections/clustered_collection_create_index_clustered_common.js");
+
+if (!ClusteredCollectionUtil.areClusteredIndexesEnabled(db.getMongo())) {
+    jsTestLog('Skipping test because the clustered indexes feature flag is disabled');
+    return;
+}
+
+const replicatedDB = db.getSiblingDB("create_index_clustered");
+const collName = "coll";
+
+CreateIndexesClusteredTest.runBaseTests(replicatedDB, collName);
+
+// Only cluster key _id is valid for creating replicated clustered collections.
+CreateIndexesClusteredTest.assertCreateIndexesImplicitCreateFails(
+    replicatedDB,
+    collName,
+    {createIndexes: collName, indexes: [{key: {a: 1}, name: "a_1", clustered: true, unique: true}]},
+    5979701);
+})();
diff --git a/jstests/core/clustered_collection_creation.js b/jstests/core/clustered_collection_creation.js
index cfa5c326c76..7b3e02db552 100644
--- a/jstests/core/clustered_collection_creation.js
+++ b/jstests/core/clustered_collection_creation.js
@@ -6,7 +6,7 @@
  * non-replicated collections.
  *
  * @tags: [
- *   requires_fcv_51,
+ *   requires_fcv_52,
  *   assumes_against_mongod_not_mongos,
  *   assumes_no_implicit_collection_creation_after_drop,
  *   does_not_support_stepdowns,
@@ -38,17 +38,43 @@ const validateCompoundSecondaryIndexes = function(db, coll, clusterKey) {
     coll.drop();
 };
 
-// Cannot create an index with the same key as the cluster key.
-const validateClusteredIndexAlreadyExists = function(db, collName, fullCreateOptions) {
+// Tests it is legal to call createIndex on the cluster key with or without {'clustered': true} as
+// an option. Additionally, confirms it is illegal to call createIndex with the 'clustered' option
+// on a pattern that is not the cluster key.
+const validateCreateIndexOnClusterKey = function(db, collName, fullCreateOptions) {
     const clusterKey = fullCreateOptions.clusteredIndex.key;
-    const res = db[collName].createIndex(clusterKey);
-    assert.commandFailedWithCode(res, ErrorCodes.CannotCreateIndex);
-    const clusterKeyField = Object.keys(clusterKey)[0];
-    if (clusterKeyField == "_id") {
-        assert(res.errmsg.includes("cannot create the _id index on a clustered collection"));
-    } else {
-        assert(res.errmsg.includes("cannot create an index with the same key as the cluster key"));
-    }
+
+    const listIndexes0 = assert.commandWorked(db[collName].runCommand("listIndexes"));
+    const listIndexesClusteredIndex = listIndexes0.cursor.firstBatch[0];
+
+    // Expect listIndexes to append the 'clustered' field to it's clusteredIndex output.
+    assert.docEq(listIndexesClusteredIndex.key, clusterKey);
+    assert.eq(listIndexesClusteredIndex.clustered, true);
+
+    // no-op with the 'clustered' option.
+    assert.commandWorked(
+        db[collName].runCommand({createIndexes: collName, indexes: [listIndexesClusteredIndex]}));
+
+    // no-op without the 'clustered' option.
+    assert.commandWorked(db[collName].createIndex(clusterKey));
+
+    // 'clustered' is not a valid option for an index not on the cluster key.
+    assert.commandFailedWithCode(
+        db[collName].createIndex({notMyIndex: 1}, {clustered: true, unique: true}), 6100904);
+
+    assert.commandFailedWithCode(db[collName].runCommand({
+        createIndexes: collName,
+        indexes: [
+            {key: {a: 1}, name: "a_1"},
+            {key: {b: 1}, name: "b_1_clustered", clustered: true, unique: true}
+        ]
+    }),
+                                 6100904);
+
+    // The listIndexes output should be unchanged.
+    const listIndexes1 = assert.commandWorked(db[collName].runCommand("listIndexes"));
+    assert.eq(listIndexes1.cursor.firstBatch.length, 1);
+    assert.docEq(listIndexes1.cursor.firstBatch[0], listIndexes0.cursor.firstBatch[0]);
 };
 
 // It is illegal to drop the clusteredIndex. Verify that the various ways of dropping the
@@ -66,15 +92,15 @@ const validateClusteredIndexUndroppable = function(db, collName, fullCreateOptio
 };
 
 const validateCreatedCollection = function(db, collName, createOptions) {
-    // Upon creating a collection, fields absent in the user provided create options are filled in
-    // with default values. The fullCreateOptions should contain default values for the fields not
-    // specified by the user.
+    // Upon creating a collection, fields absent in the user provided create options are filled
+    // in with default values. The fullCreateOptions should contain default values for the
+    // fields not specified by the user.
     const fullCreateOptions = ClusteredCollectionUtil.constructFullCreateOptions(createOptions);
 
     ClusteredCollectionUtil.validateListCollections(db, collName, fullCreateOptions);
     ClusteredCollectionUtil.validateListIndexes(db, collName, fullCreateOptions);
 
-    validateClusteredIndexAlreadyExists(db, collName, fullCreateOptions);
+    validateCreateIndexOnClusterKey(db, collName, fullCreateOptions);
     validateClusteredIndexUndroppable(db, collName, fullCreateOptions);
 };
 
diff --git a/jstests/core/timeseries/timeseries_id_index.js b/jstests/core/timeseries/timeseries_id_index.js
index 824a0ff1550..e3bc20d6648 100644
--- a/jstests/core/timeseries/timeseries_id_index.js
+++ b/jstests/core/timeseries/timeseries_id_index.js
@@ -1,7 +1,8 @@
 /**
- * Verifies that the _id index cannot be created on a time-series collection.
+ * Verifies that the _id index can be created on a timeseries collection.
  *
  * @tags: [
+ *   requires_fcv_52,
  *   does_not_support_stepdowns,
  *   does_not_support_transactions,
  *   requires_getmore,
@@ -18,8 +19,10 @@ assert.commandWorked(db.createCollection(coll.getName(), {timeseries: {timeField
 
 const bucketsColl = db.getCollection("system.buckets." + coll.getName());
 
-const res = bucketsColl.createIndex({"_id": 1});
-assert.commandFailedWithCode(res, ErrorCodes.CannotCreateIndex);
-assert(res.errmsg.includes("cannot create the _id index on a clustered collection") ||
-       res.errmsg.includes("cannot create an _id index on a collection already clustered by _id"));
+assert.commandWorked(bucketsColl.createIndex({"_id": 1}));
+assert.commandWorked(bucketsColl.createIndex({"_id": 1}, {clustered: true, unique: true}));
+
+// Passing 'clustered' without unique, regardless of the type of clustered collection, is illegal.
+assert.commandFailedWithCode(bucketsColl.createIndex({"_id": 1}, {clustered: true}),
+                             ErrorCodes.CannotCreateIndex);
 })();
diff --git a/jstests/libs/clustered_collections/clustered_collection_create_index_clustered_common.js b/jstests/libs/clustered_collections/clustered_collection_create_index_clustered_common.js
new file mode 100644
index 00000000000..13e7f7dc666
--- /dev/null
+++ b/jstests/libs/clustered_collections/clustered_collection_create_index_clustered_common.js
@@ -0,0 +1,193 @@
+/**
+ * Encapsulates testing that verifies the behavior of createIndexes with the 'clustered' option. The
+ * 'clustered' option may be used to implicitly create a clustered collection via createIndexes.
+ */
+const CreateIndexesClusteredTest = (function() {
+    "use strict";
+
+    /**
+     * From createIndex with the 'clustered' option 'true', generates the corresponding
+     * createCollection options for the implicitly created 'coll'.
+     */
+    const getImplicitCreateOptsFromCreateIndex = function(cmd) {
+        const fullCreateOptions =
+            ClusteredCollectionUtil.constructFullCreateOptions({clusteredIndex: cmd.indexes[0]});
+        delete fullCreateOptions.clusteredIndex.clustered;
+        return fullCreateOptions;
+    };
+
+    /**
+     * Asserts that running the createIndexes 'cmd' implicitly creates a clustered collection.
+     */
+    const assertCreateIndexesImplicitCreateSucceeds = function(testDB, collName, cmd) {
+        assertDropCollection(testDB, collName);
+        assert.commandWorked(testDB.runCommand(cmd));
+
+        // From the createIndex command, generate the corresponding createCollection options.
+        const fullCreateOptions = getImplicitCreateOptsFromCreateIndex(cmd);
+
+        ClusteredCollectionUtil.validateListCollections(testDB, collName, fullCreateOptions);
+        ClusteredCollectionUtil.validateListIndexes(testDB, collName, fullCreateOptions);
+    };
+
+    /**
+     * Asserts that running createIndexes 'cmd' on a non-existant collection fails with 'errorCode'.
+     */
+    const assertCreateIndexesImplicitCreateFails = function(testDB, collName, cmd, errorCode) {
+        assertDropCollection(testDB, collName);
+        assert.commandFailedWithCode(
+            testDB.runCommand(cmd),
+            errorCode,
+            `Expected indexOpts ${tojson(cmd)} to fail with error code ${errorCode}`);
+    };
+
+    /**
+     * Tests that createIndex with the 'clustered' option fails when a collection exists and is not
+     * clustered.
+     */
+    const runNonClusteredCollectionTest = function(testDB, collName) {
+        assertDropCollection(testDB, collName);
+        const testColl = testDB[collName];
+        assert.commandWorked(testDB.createCollection(collName));
+
+        // Start with the collection empty.
+        assert.commandFailedWithCode(
+            testColl.createIndex({_id: 1}, {clustered: true, unique: true}), 6100905);
+        assert.commandFailedWithCode(testColl.createIndex({a: 1}, {clustered: true, unique: true}),
+                                     6100905);
+
+        // Insert some docs. Sometimes empty collections are treated as special when it comes to
+        // index builds.
+        const batchSize = 100;
+        const bulk = testColl.initializeUnorderedBulkOp();
+        for (let i = 0; i < batchSize; i++) {
+            bulk.insert({_id: i, a: -i});
+        }
+        assert.commandWorked(bulk.execute());
+        assert.commandFailedWithCode(
+            testColl.createIndex({_id: 1}, {clustered: true, unique: true}), 6100905);
+        assert.commandFailedWithCode(testColl.createIndex({a: 1}, {clustered: true, unique: true}),
+                                     6100905);
+    };
+
+    /**
+     * Tests running createIndex on a clustered collection
+     */
+    const runClusteredCollectionTest = function(testDB, collName) {
+        assertDropCollection(testDB, collName);
+
+        const createOptions = {
+            clusteredIndex: {key: {_id: 1}, name: "theClusterKeyName", unique: true}
+        };
+        assert.commandWorked(testDB.createCollection(collName, createOptions));
+
+        // Confirm we start out with a valid clustered collection.
+        const fullCreateOptions = ClusteredCollectionUtil.constructFullCreateOptions(createOptions);
+        ClusteredCollectionUtil.validateListCollections(testDB, collName, fullCreateOptions);
+        ClusteredCollectionUtil.validateListIndexes(testDB, collName, fullCreateOptions);
+
+        const testColl = testDB[collName];
+
+        // createIndex on the cluster key is a no-op.
+        assert.commandWorked(testColl.createIndex({_id: 1}));
+        ClusteredCollectionUtil.validateListIndexes(testDB, collName, fullCreateOptions);
+
+        // createIndex on the cluster key with the 'clustered' option is a no-op.
+        assert.commandWorked(testColl.createIndex({_id: 1}, {clustered: true, unique: true}));
+
+        // 'clustered' is not a valid option for an index not on the cluster key.
+        assert.commandFailedWithCode(
+            testColl.createIndex({notMyIndex: 1}, {clustered: true, unique: true}), 6100904);
+
+        // Insert some docs. Sometimes empty collections are treated as special when it comes to
+        // index builds.
+        const batchSize = 100;
+        const bulk = testColl.initializeUnorderedBulkOp();
+        for (let i = 0; i < batchSize; i++) {
+            bulk.insert({_id: i, a: -i});
+        }
+        assert.commandWorked(bulk.execute());
+
+        assert.commandWorked(testColl.createIndex({_id: 1}));
+        assert.commandWorked(testColl.createIndex({_id: 1}, {clustered: true, unique: true}));
+
+        // Note: this a quirk of how we handle the 'name' field for indexes of {_id: 1}. The
+        // createIndex is still a no-op, and the specified name is discarded.
+        //
+        // Only in implicit collection creation on a non-existent collection can createIndex create
+        // a clusteredIndex with a custom name.
+        assert.commandWorked(testColl.createIndex({_id: 1}, {name: "notTheClusterKeyName"}));
+
+        ClusteredCollectionUtil.validateListIndexes(testDB, collName, fullCreateOptions);
+    };
+
+    /**
+     * Runs test cases where createIndexes with 'clustered' should succeed in implicit collecton
+     * creation, regardless of whether the database is replicated.
+     */
+    const runBaseSuccessTests = function(testDB, collName) {
+        assertCreateIndexesImplicitCreateSucceeds(testDB, collName, {
+            createIndexes: collName,
+            indexes: [{key: {_id: 1}, name: "_id_", clustered: true, unique: true}]
+        });
+        assertCreateIndexesImplicitCreateSucceeds(testDB, collName, {
+            createIndexes: collName,
+            indexes: [{key: {_id: 1}, name: "_id_", clustered: true, unique: true, v: 2}]
+        });
+        assertCreateIndexesImplicitCreateSucceeds(testDB, collName, {
+            createIndexes: collName,
+            indexes: [{key: {_id: 1}, name: "uniqueIdName", clustered: true, unique: true, v: 2}]
+        });
+    };
+
+    /**
+     * Runs test cases where createIndexes with 'clustered' fails, regardless of whether the
+     * database is replciated.
+     */
+    const runBaseFailureTests = function(testDB, collName) {
+        // Missing 'unique' option.
+        assertCreateIndexesImplicitCreateFails(
+            testDB,
+            collName,
+            {createIndexes: collName, indexes: [{key: {_id: 1}, name: "_id_", clustered: true}]},
+            ErrorCodes.CannotCreateIndex);
+        // Two 'clustered' indexes.
+        assertCreateIndexesImplicitCreateFails(
+            testDB,
+            collName,
+            {
+                createIndexes: collName,
+                indexes: [
+                    {key: {_id: 1}, name: "_id_", clustered: true, unique: true},
+                    {key: {a: 1}, name: "a_1", clustered: true, unique: true}
+                ]
+            },
+            6100901);
+        assertCreateIndexesImplicitCreateFails(
+            testDB,
+            collName,
+            {
+                createIndexes: collName,
+                indexes: [
+                    {key: {_id: 1}, name: "_id_", clustered: true, unique: true, hidden: true},
+                ]
+            },
+            ErrorCodes.InvalidIndexSpecificationOption);
+    };
+
+    /**
+     * Runs test cases that are agnostic to whether the database is replicated or not.
+     */
+    const runBaseTests = function(testDB, collName) {
+        runNonClusteredCollectionTest(testDB, collName);
+        runClusteredCollectionTest(testDB, collName);
+        runBaseSuccessTests(testDB, collName);
+        runBaseFailureTests(testDB, collName);
+    };
+
+    return {
+        assertCreateIndexesImplicitCreateSucceeds: assertCreateIndexesImplicitCreateSucceeds,
+        assertCreateIndexesImplicitCreateFails: assertCreateIndexesImplicitCreateFails,
+        runBaseTests: runBaseTests,
+    };
+})();
diff --git a/jstests/noPassthrough/clustered_collection_create_index_clustered_nonreplicated.js b/jstests/noPassthrough/clustered_collection_create_index_clustered_nonreplicated.js
new file mode 100644
index 00000000000..cbd1186b4d6
--- /dev/null
+++ b/jstests/noPassthrough/clustered_collection_create_index_clustered_nonreplicated.js
@@ -0,0 +1,40 @@
+/**
+ * Tests createIndexes with the 'clustered' option on a replicated collection. Note: there are
+ * different restrictions for non-replicated versus replicated clustered collections - eg replicated
+ * collections can only be created with cluster key _id whereas non-replicated collections can be
+ * created with arbitrary single field cluster keys.
+ *
+ * @tags: [
+ *   requires_fcv_52,
+ *   # Does not support sharding
+ *   assumes_against_mongod_not_mongos,
+ *   assumes_unsharded_collection,
+ * ]
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/clustered_collections/clustered_collection_util.js");
+load("jstests/libs/clustered_collections/clustered_collection_create_index_clustered_common.js");
+
+const conn = MongoRunner.runMongod();
+
+if (ClusteredCollectionUtil.areClusteredIndexesEnabled(conn) == false) {
+    jsTestLog('Skipping test because the clustered indexes feature flag is disabled');
+    MongoRunner.stopMongod(conn);
+    return;
+}
+
+const nonReplicatedDB = conn.getDB("local");
+const collName = "coll";
+const nonReplicatedColl = nonReplicatedDB[collName];
+
+CreateIndexesClusteredTest.runBaseTests(nonReplicatedDB, collName);
+
+CreateIndexesClusteredTest.assertCreateIndexesImplicitCreateSucceeds(nonReplicatedDB, collName, {
+    createIndexes: collName,
+    indexes: [{key: {a: 1}, name: "clusterKeyYay", clustered: true, unique: true}]
+});
+
+MongoRunner.stopMongod(conn);
+})();