SERVER-58825 Make pre-images collection clustered

6 files changed, 79 insertions, 58 deletions

diff --git a/jstests/change_streams/lookup_pit_pre_and_post_image.js b/jstests/change_streams/lookup_pit_pre_and_post_image.js
index b10ed83a125..69153720d61 100644
--- a/jstests/change_streams/lookup_pit_pre_and_post_image.js
+++ b/jstests/change_streams/lookup_pit_pre_and_post_image.js
@@ -14,7 +14,11 @@ load("jstests/libs/change_stream_util.js");        // For isChangeStreamPreAndPo
 const testDB = db.getSiblingDB(jsTestName());
 const collName = "test";
 
-if (!isChangeStreamPreAndPostImagesEnabled(db)) {
+const clusteredIndexesEnabled =
+    assert.commandWorked(testDB.adminCommand({getParameter: 1, featureFlagClusteredIndexes: 1}))
+        .featureFlagClusteredIndexes.value;
+
+if (!(isChangeStreamPreAndPostImagesEnabled(db) && clusteredIndexesEnabled)) {
     const coll = assertDropAndRecreateCollection(testDB, collName);
 
     // If feature flag is off, creating changeStream with new fullDocument arguments should throw.
@@ -23,7 +27,7 @@ if (!isChangeStreamPreAndPostImagesEnabled(db)) {
     assert.throwsWithCode(() => coll.watch([], {fullDocument: 'required'}), ErrorCodes.BadValue);
 
     jsTestLog(
-        'Skipping test because featureFlagChangeStreamPreAndPostImages feature flag is not enabled');
+        'Skipping test because featureFlagChangeStreamPreAndPostImages or featureFlagClusteredIndexes feature flag is not enabled');
     return;
 }
 
diff --git a/jstests/change_streams/write_pit_preimage.js b/jstests/change_streams/write_pit_preimage.js
index acb175036d8..f19f992643f 100644
--- a/jstests/change_streams/write_pit_preimage.js
+++ b/jstests/change_streams/write_pit_preimage.js
@@ -3,6 +3,8 @@
 // @tags: [
 //  requires_fcv_51,
 //  featureFlagChangeStreamPreAndPostImages,
+//  # Clustered index support is required for change stream pre-images collection.
+//  featureFlagClusteredIndexes,
 //  assumes_against_mongod_not_mongos,
 //  change_stream_does_not_expect_txns,
 //  multiversion_incompatible,
diff --git a/jstests/multiVersion/change_streams_pre_and_post_images_upgrade_downgrade.js b/jstests/multiVersion/change_streams_pre_and_post_images_upgrade_downgrade.js
index be15e6a89f2..60aaf51f050 100644
--- a/jstests/multiVersion/change_streams_pre_and_post_images_upgrade_downgrade.js
+++ b/jstests/multiVersion/change_streams_pre_and_post_images_upgrade_downgrade.js
@@ -1,7 +1,11 @@
 /**
  * Verifies that it is possible to upgrade a replica set with collections with 'recordPreImages'
  * option to use 'changeStreamPreAndPostImages' option, and to do a corresponding downgrade.
- * @tags: [requires_fcv_51, featureFlagChangeStreamPreAndPostImages]
+ * @tags: [requires_fcv_51,
+ * featureFlagChangeStreamPreAndPostImages,
+ * # Clustered index support is required for change stream pre-images collection.
+ * featureFlagClusteredIndexes,
+ * ]
  */
 (function() {
 'use strict';
@@ -35,6 +39,16 @@ function runTest(downgradeVersion) {
     rst.upgradeSet({binVersion: "latest"});
     testDB = rst.getPrimary().getDB(jsTestName());
 
+    // Verify that an attempt to set 'changeStreamPreAndPostImages' option fails for the downgraded
+    // FCV version.
+    assert.commandFailedWithCode(
+        testDB.createCollection("anotherTestCollection",
+                                {"changeStreamPreAndPostImages": {enabled: false}}),
+        5846900);
+    assert.commandFailedWithCode(
+        testDB.runCommand({"collMod": collName, "changeStreamPreAndPostImages": {enabled: false}}),
+        5846901);
+
     // Set the FCV to the latest.
     testDB.adminCommand({setFeatureCompatibilityVersion: latestFCV});
 
@@ -62,47 +76,8 @@ function runTest(downgradeVersion) {
     // Downgrade the FCV.
     testDB.adminCommand({setFeatureCompatibilityVersion: downgradeFCV});
 
-    // Verify that an attempt to set 'changeStreamPreAndPostImages' option fails for the downgrade
-    // version.
-    assert.commandFailedWithCode(
-        testDB.createCollection(collName, {"changeStreamPreAndPostImages": {enabled: false}}),
-        5846900);
-    assert.commandFailedWithCode(
-        testDB.runCommand({"collMod": collName, "changeStreamPreAndPostImages": {enabled: false}}),
-        5846901);
-
-    // Downgrade the cluster.
-    rst.upgradeSet({binVersion: downgradeVersion});
-
-    // Reset the db reference.
-    testDB = rst.getPrimary().getDB(jsTestName());
-
-    // 'changeStreamPreAndPostImages' option must be absent and 'recordPreImages' should be set to
-    // true.
-    assertCollectionOptionIsEnabled(testDB, collName, "recordPreImages");
-    assertChangeStreamPreAndPostImagesCollectionOptionIsAbsent(testDB, collName);
-
-    // Upgrade the replica set.
-    rst.upgradeSet({binVersion: "latest"});
-    testDB = rst.getPrimary().getDB(jsTestName());
-
-    // Set the FCV to the latest.
-    testDB.adminCommand({setFeatureCompatibilityVersion: latestFCV});
-
-    // Enable pre-/post-images for the collection with "recordPreImages" enabled.
-    assert.commandWorked(
-        testDB.runCommand({"collMod": collName, "changeStreamPreAndPostImages": {enabled: true}}));
-
-    // 'changeStreamPreAndPostImages' option must be enabled and 'recordPreImages' option must be
-    // absent.
-    assertChangeStreamPreAndPostImagesCollectionOptionIsEnabled(testDB, collName);
-    assertCollectionOptionIsAbsent(testDB, collName, "recordPreImages");
-
-    // Downgrade the FCV.
-    testDB.adminCommand({setFeatureCompatibilityVersion: downgradeFCV});
-
-    // Downgrading the cluster should fail, since unsupported option 'changeStreamPreAndPostImages'
-    // is enabled for the collection.
+    // Downgrading the cluster should fail, since the pre-images collection is clustered which is
+    // not supported by the downgraded binary.
     try {
         rst.upgradeSet({binVersion: downgradeVersion});
         assert(false);
diff --git a/jstests/noPassthrough/change_streams_pre_and_post_images_in_create_and_collmod.js b/jstests/noPassthrough/change_streams_pre_and_post_images_in_create_and_collmod.js
index 71c90843d3d..8f27c8476d8 100644
--- a/jstests/noPassthrough/change_streams_pre_and_post_images_in_create_and_collmod.js
+++ b/jstests/noPassthrough/change_streams_pre_and_post_images_in_create_and_collmod.js
@@ -1,8 +1,14 @@
 /*
  * Tests that the 'changeStreamPreAndPostImages' option is settable via the collMod and create
  * commands. Also tests that this option cannot be set on collections in the 'local', 'admin',
- * 'config' databases as well as timeseries and view collections.
- * @tags: [requires_fcv_51, featureFlagChangeStreamPreAndPostImages]
+ * 'config' databases as well as timeseries and view collections. Verifies that the pre-images
+ * collection is clustered.
+ * @tags: [
+ * requires_fcv_51,
+ * featureFlagChangeStreamPreAndPostImages,
+ * # Clustered index support is required for change stream pre-images collection.
+ * featureFlagClusteredIndexes,
+ * ]
  */
 (function() {
 'use strict';
@@ -39,16 +45,34 @@ const localDB = primary.getDB("local");
 const configDB = primary.getDB("config");
 const testDB = primary.getDB(dbName);
 
-function assertPreimagesCollectionIsAbsent() {
-    const result = localDB.runCommand("listCollections", {filter: {name: preimagesCollName}});
+function findPreImagesCollectionDescriptions() {
+    return localDB.runCommand("listCollections", {filter: {name: preimagesCollName}});
+}
+
+function assertPreImagesCollectionIsAbsent() {
+    const result = findPreImagesCollectionDescriptions();
     assert.eq(result.cursor.firstBatch.length, 0);
 }
 
-function assertPreimagesCollectionExists() {
-    const result = localDB.runCommand("listCollections", {filter: {name: preimagesCollName}});
+function assertPreImagesCollectionExists() {
+    const result = findPreImagesCollectionDescriptions();
     assert.eq(result.cursor.firstBatch[0].name, preimagesCollName);
 }
 
+// Verifies that the pre-images collection is clustered by _id.
+function assertPreImagesCollectionIsClustered() {
+    const collectionInfos = findPreImagesCollectionDescriptions();
+    assert.eq(collectionInfos.cursor.firstBatch.length, 1, collectionInfos);
+    const preImagesCollectionDescription = collectionInfos.cursor.firstBatch[0];
+    assert(preImagesCollectionDescription.hasOwnProperty("options"),
+           preImagesCollectionDescription);
+    assert(preImagesCollectionDescription.options.hasOwnProperty("clusteredIndex"),
+           preImagesCollectionDescription);
+    const clusteredIndexDescription = preImagesCollectionDescription.options.clusteredIndex;
+    assert.eq(clusteredIndexDescription.unique, true, preImagesCollectionDescription);
+    assert.eq(clusteredIndexDescription.key, {_id: 1}, preImagesCollectionDescription);
+}
+
 // Check that we cannot set 'changeStreamPreAndPostImages' on the local, admin and config databases.
 for (const db of [localDB, adminDB, configDB]) {
     assert.commandFailedWithCode(
@@ -63,7 +87,7 @@ for (const db of [localDB, adminDB, configDB]) {
 
 // Drop the pre-images collection.
 assertDropCollection(localDB, preimagesCollName);
-assertPreimagesCollectionIsAbsent();
+assertPreImagesCollectionIsAbsent();
 
 // Drop all collections that are used during the test.
 for (const collectionToDelete
@@ -75,23 +99,24 @@ for (const collectionToDelete
 assert.commandWorked(
     testDB.runCommand({create: collName, changeStreamPreAndPostImages: {enabled: true}}));
 assertChangeStreamPreAndPostImagesCollectionOptionIsEnabled(testDB, collName);
-assertPreimagesCollectionExists();
+assertPreImagesCollectionExists();
+assertPreImagesCollectionIsClustered();
 
 // Drop the pre-images collection.
 assertDropCollection(localDB, preimagesCollName);
-assertPreimagesCollectionIsAbsent();
+assertPreImagesCollectionIsAbsent();
 
 assert.commandWorked(testDB.runCommand({create: collName2}));
 assert.commandWorked(
     testDB.runCommand({collMod: collName2, changeStreamPreAndPostImages: {enabled: true}}));
 assertChangeStreamPreAndPostImagesCollectionOptionIsEnabled(testDB, collName2);
-assertPreimagesCollectionExists();
+assertPreImagesCollectionExists();
 
 // Verify that setting collection options with 'collMod' command does not affect
 // 'changeStreamPreAndPostImages' option.
 assert.commandWorked(testDB.runCommand({"collMod": collName2, validationLevel: "off"}));
 assertChangeStreamPreAndPostImagesCollectionOptionIsEnabled(testDB, collName2);
-assertPreimagesCollectionExists();
+assertPreImagesCollectionExists();
 
 // Should successfully disable 'changeStreamPreAndPostImages' using the 'collMod' command.
 assert.commandWorked(
@@ -100,7 +125,7 @@ assertChangeStreamPreAndPostImagesCollectionOptionIsAbsent(testDB, collName2);
 
 // Should not remove the pre-images collection on disabling 'changeStreamPreAndPostImages'
 // option.
-assertPreimagesCollectionExists();
+assertPreImagesCollectionExists();
 
 // Both 'recordPreImages' and 'changeStreamPreAndPostImages' may not be enabled at the same
 // time.
diff --git a/src/mongo/db/catalog/create_collection.cpp b/src/mongo/db/catalog/create_collection.cpp
index 0c71a02e4cf..a238779222c 100644
--- a/src/mongo/db/catalog/create_collection.cpp
+++ b/src/mongo/db/catalog/create_collection.cpp
@@ -620,11 +620,22 @@ void createChangeStreamPreImagesCollection(OperationContext* opCtx) {
     uassert(5868501,
             "Failpoint failPreimagesCollectionCreation enabled. Throwing exception",
             !MONGO_unlikely(failPreimagesCollectionCreation.shouldFail()));
+    tassert(5882500,
+            "Failed to create the pre-images collection: clustered indexes feature is not enabled",
+            feature_flags::gClusteredIndexes.isEnabled(serverGlobalParams.featureCompatibility));
 
     const auto nss = NamespaceString::kChangeStreamPreImagesNamespace;
-    const auto status = _createCollection(opCtx, nss, CollectionOptions(), BSONObj());
+    CollectionOptions preImagesCollectionOptions;
+
+    // Make the collection clustered by _id.
+    auto preImagesCollectionKey = BSON("_id" << 1);
+    preImagesCollectionOptions.clusteredIndex.emplace(
+        ClusteredIndexSpec{preImagesCollectionKey, true /*unique*/}, false /*legacyFormat*/);
+    const auto status =
+        _createCollection(opCtx, nss, std::move(preImagesCollectionOptions), BSONObj());
     uassert(status.code(),
-            str::stream() << "Failed to create the pre-images collection: " << nss.coll(),
+            str::stream() << "Failed to create the pre-images collection: " << nss.coll()
+                          << causedBy(status.reason()),
             status.isOK() || status.code() == ErrorCodes::NamespaceExists);
 }
 
diff --git a/src/mongo/db/catalog/create_collection.h b/src/mongo/db/catalog/create_collection.h
index 3b68acbfe2a..24d41eb3fbd 100644
--- a/src/mongo/db/catalog/create_collection.h
+++ b/src/mongo/db/catalog/create_collection.h
@@ -55,6 +55,10 @@ Status createCollection(OperationContext* opCtx,
                         const NamespaceString& ns,
                         const CreateCommand& cmd);
 
+/**
+ * Creates the change stream pre-images collection. The collection is clustered by the primary key,
+ * _id.
+ */
 void createChangeStreamPreImagesCollection(OperationContext* opCtx);
 
 /**