SERVER-59706 Extend tests for change stream pre-images retrieval capability controlled by option recordPreImages

3 files changed, 206 insertions, 144 deletions

diff --git a/jstests/core/timeseries/timeseries_collmod.js b/jstests/core/timeseries/timeseries_collmod.js
index 4108ce9c74a..b1cc445e9b1 100644
--- a/jstests/core/timeseries/timeseries_collmod.js
+++ b/jstests/core/timeseries/timeseries_collmod.js
@@ -51,6 +51,14 @@ assert.commandFailedWithCode(db.runCommand({"collMod": collName, "viewOn": "foo"
 assert.commandFailedWithCode(db.runCommand({"collMod": collName, "recordPreImages": true}),
                              ErrorCodes.InvalidOptions);
 
+// Tries to set 'changeStreamPreAndPostImages' for a time-series collection.
+assert.commandFailedWithCode(
+    db.runCommand({"collMod": collName, "changeStreamPreAndPostImages": {enabled: true}}), [
+        ErrorCodes.InvalidOptions,
+        // TODO SERVER-52282: remove the error code.
+        5846901
+    ]);
+
 // Successfully sets 'expireAfterSeconds' for a time-series collection.
 assert.commandWorked(db.runCommand({"collMod": collName, "expireAfterSeconds": 60}));
 
diff --git a/jstests/noPassthrough/change_stream_preimages_standalone_mode.js b/jstests/noPassthrough/change_stream_preimages_standalone_mode.js
new file mode 100644
index 00000000000..1ac6e9c807a
--- /dev/null
+++ b/jstests/noPassthrough/change_stream_preimages_standalone_mode.js
@@ -0,0 +1,198 @@
+/**
+ * Test that nodes are able to startup with 'recordPreImages' and 'changeStreamPreAndPostImages'
+ * options set in collection metadata and no pre-images are recorded while being in standalone mode.
+ *
+ * @tags: [
+ *   # Servers are restarted in this test and the data must be retained.
+ *   requires_persistence,
+ *   # This test uses a replica set and must avoid replica set incompatible test suites, like the
+ *   # test suite that turns journaling off.
+ *   requires_replication,
+ *   requires_fcv_52,
+ *   featureFlagChangeStreamPreAndPostImages,
+ *   # Clustered index support is required for change stream pre-images collection.
+ *   featureFlagClusteredIndexes,
+ *   # TODO SERVER-58694: remove this tag.
+ *   change_stream_does_not_expect_txns,
+ *   # TODO SERVER-60238: remove this tag.
+ *   assumes_read_preference_unchanged
+ * ]
+ */
+
+(function() {
+'use strict';
+
+load("jstests/libs/collection_drop_recreate.js");  // For assertDropAndRecreateCollection.
+load(
+    "jstests/libs/change_stream_util.js");  // For
+                                            // assertChangeStreamPreAndPostImagesCollectionOptionIsEnabled.
+
+// Fetches the collection with name 'collName' from database 'nodeDB'. Expects the collection to
+// exist.
+const findCollectionInfo = function(nodeDB, collName) {
+    const collInfos = nodeDB.getCollectionInfos();
+    assert.gt(collInfos.length, 0, "The database is empty");
+
+    const collInfo = collInfos.filter(collInfo => collInfo.name == collName);
+    assert.eq(collInfo.length, 1);
+    return collInfo[0];
+};
+
+// Returns the oplog entries written while performing the write operations.
+function oplogEntriesForOps(db, writeOps) {
+    const oplogColl = db.getSiblingDB('local').oplog.rs;
+    const numOplogEntriesBefore = oplogColl.find().itcount();
+
+    // Perform the write operations.
+    writeOps();
+
+    // Ensure that the last write with j:true write concern has reached the disk, and now fsync will
+    // checkpoint that data.
+    assert.commandWorked(db.adminCommand({fsync: 1}));
+
+    // Check the number of oplog entries written.
+    const numOplogEntriesAfter = oplogColl.find().itcount();
+    const numberOfNewOplogEntries = numOplogEntriesAfter - numOplogEntriesBefore;
+    if (numberOfNewOplogEntries == 0) {
+        return [];
+    }
+    return oplogColl.find().sort({ts: -1}).limit(numberOfNewOplogEntries).toArray();
+}
+
+// Returns the pre-images written while performing the write operations.
+function preImagesForOps(db, writeOps) {
+    const preImagesColl = db.getSiblingDB('config').getCollection("system.preimages");
+    const numberOfPreImagesBefore = preImagesColl.find().itcount();
+
+    // Perform the write operations.
+    writeOps();
+
+    // Ensure that the last write with j:true write concern has reached the disk, and now fsync will
+    // checkpoint that data.
+    assert.commandWorked(db.adminCommand({fsync: 1}));
+
+    // Check the number of pre-images written.
+    const numberOfPreImagesAfter = preImagesColl.find().itcount();
+    const numberOfNewPreImages = numberOfPreImagesAfter - numberOfPreImagesBefore;
+    if (numberOfNewPreImages == 0) {
+        return [];
+    }
+    return preImagesColl.find().sort({operationTime: -1}).limit(numberOfNewPreImages).toArray();
+}
+
+/**
+ * Tests the pre-image recording behavior when the server transitions to and from the stand-alone
+ * mode for a collection with specified 'collectionOptions'.
+ *
+ * @param {function} assertPreImagesRecordingEnabledFunc - asserts that pre-images
+ *     recording collection option is enabled for the specified collection.
+ * @param {function} assertPreImagesRecordedFunc - asserts that pre-images are recorded while
+ *     executing the 'writeOps' (update/replace/delete operations) on the collection.
+ * @param {function} assertNoPreImagesRecordedFunc - asserts that no pre-images are recorded while
+ *     executing the 'writeOps' (update/replace/delete operations) on the collection.
+ */
+function testStandaloneMode({
+    collectionOptions = {},
+    assertPreImagesRecordingEnabledFunc = (db, collName) => {},
+    assertPreImagesRecordedFunc = (db, writeOps) => {},
+    assertNoPreImagesRecordedFunc = (db, writeOps) => {}
+}) {
+    const rst = new ReplSetTest({nodes: 1});
+    rst.startSet();
+    rst.initiate();
+
+    const collName = "coll";
+    const dbName = jsTestName();
+    let primary = rst.getPrimary();
+    let testDB = primary.getDB(dbName);
+
+    // Create a collection with pre-images recording collection option enabled.
+    let testColl = assertDropAndRecreateCollection(testDB, collName, collectionOptions);
+    assertPreImagesRecordingEnabledFunc(testDB, collName);
+
+    // Verify that pre-images are recorded for the specified operations.
+    const writeOpsForReplSetMode = () => {
+        assert.commandWorked(testColl.insert({a: 1, b: 1}));
+        assert.commandWorked(testColl.update({a: 1}, {a: 2, b: 2}));
+        assert.commandWorked(
+            testColl.update({a: 2}, {a: 3, b: 3}, {writeConcern: {w: 1, j: true}}));
+    };
+    assertPreImagesRecordedFunc(testDB, writeOpsForReplSetMode);
+
+    // Restart the replica set member as a standalone node.
+    const replicaSetNodeId = rst.getNodeId(primary);
+    const replicaSetNodeDBPath = primary.dbpath;
+    rst.stop(replicaSetNodeId);
+
+    const standaloneConn = MongoRunner.runMongod({
+        dbpath: replicaSetNodeDBPath,
+        noCleanData: true,
+    });
+    const standaloneDB = standaloneConn.getDB(dbName);
+    const standaloneColl = standaloneDB.getCollection(collName);
+
+    // The collection must have pre-images recording option enabled even when running in standalone
+    // mode.
+    assertPreImagesRecordingEnabledFunc(standaloneDB, collName);
+
+    // Verify that no pre-images are recorded while running in standalone mode.
+    const writeOpsForStandaloneMode = () => {
+        assert.commandWorked(standaloneColl.insert({c: 1, d: 1}));
+        assert.commandWorked(standaloneColl.update({c: 1}, {c: 2, d: 2}));
+        assert.commandWorked(standaloneColl.update({c: 2}, {c: 3, d: 3}));
+        assert.commandWorked(standaloneColl.insert({c: 1, d: 1}));
+        assert.commandWorked(standaloneColl.remove({c: 1, d: 1}));
+    };
+    assertNoPreImagesRecordedFunc(standaloneDB, writeOpsForStandaloneMode);
+
+    // Shut down standalone server.
+    MongoRunner.stopMongod(standaloneConn);
+
+    // Restart the node as a replica set member.
+    rst.start(replicaSetNodeId, {}, true /*restart*/);
+    primary = rst.getPrimary();
+    testDB = primary.getDB(dbName);
+    testColl = testDB.getCollection(collName);
+
+    // Check that everything is still working properly after being in standalone mode.
+    assertPreImagesRecordingEnabledFunc(testDB, collName);
+    const writeOpsForReplSetModeAfterStandalone = () => {
+        assert.commandWorked(testColl.update({a: 3}, {a: 4, b: 4}));
+        assert.commandWorked(testColl.update({a: 4}, {a: 5, b: 5}));
+    };
+    assertPreImagesRecordedFunc(testDB, writeOpsForReplSetModeAfterStandalone);
+
+    rst.stopSet();
+}
+
+// Run the test for 'recordPreImages' option.
+testStandaloneMode({
+    collectionOptions: {recordPreImages: true},
+    assertPreImagesRecordingEnabledFunc: (db, collName) => {
+        assert.eq(findCollectionInfo(db, collName).options.recordPreImages, true);
+    },
+    assertPreImagesRecordedFunc: (db, writerOps) => {
+        const writtenOplogEntries = oplogEntriesForOps(db, writerOps);
+        assert.gt(writtenOplogEntries.length, 0, writtenOplogEntries);
+    },
+    assertNoPreImagesRecordedFunc: (db, writerOps) => {
+        const writtenOplogEntries = oplogEntriesForOps(db, writerOps);
+        assert.eq(writtenOplogEntries.length, 0, writtenOplogEntries);
+    }
+});
+
+// Run the test for 'changeStreamPreAndPostImages' option.
+testStandaloneMode({
+    collectionOptions: {changeStreamPreAndPostImages: {enabled: true}},
+    assertPreImagesRecordingEnabledFunc:
+        assertChangeStreamPreAndPostImagesCollectionOptionIsEnabled,
+    assertPreImagesRecordedFunc: (db, writerOps) => {
+        const writtenPreImages = preImagesForOps(db, writerOps);
+        assert.gt(writtenPreImages.length, 0, writtenPreImages);
+    },
+    assertNoPreImagesRecordedFunc: (db, writerOps) => {
+        const writtenPreImages = preImagesForOps(db, writerOps);
+        assert.eq(writtenPreImages.length, 0, writtenPreImages);
+    }
+});
+}());
diff --git a/jstests/noPassthrough/recordPreImages_standalone_mode.js b/jstests/noPassthrough/recordPreImages_standalone_mode.js
deleted file mode 100644
index 08ca547f507..00000000000
--- a/jstests/noPassthrough/recordPreImages_standalone_mode.js
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Test that standalones are able to startup with 'recordPreImages' set in collection metadata; and
- * that 'recordPreImages' is inactive in standalone mode.
- *
- * @tags: [
- *   # Servers are restarted in this test and the data must be retained.
- *   requires_persistence,
- *   # This test uses a replica set and must avoid replica set incompatible test suites, like the
- *   # test suite that turns journaling off.
- *   requires_replication,
- * ]
- */
-
-(function() {
-'use strict';
-
-/**
- * Fetch the collection information on database 'nodeDB' for collection 'collName'. Expects the
- * collection to exist.
- */
-const findCollectionInfo = function(nodeDB, collName) {
-    let collInfos = nodeDB.getCollectionInfos();
-    assert.gt(collInfos.length, 0, "The database is empty");
-
-    let collInfo = collInfos.filter(function(z) {
-        return z.name == collName;
-    });
-
-    assert.eq(collInfo.length, 1);
-    return collInfo[0];
-};
-
-/**
- * Prints out all of the oplog collection entries on 'node'.
- */
-function printOplog(node) {
-    let cursor = node.getDB('local').oplog.rs.find();
-    while (cursor.hasNext()) {
-        jsTest.log("Oplog entry: " + tojson(cursor.next()));
-    }
-}
-
-/**
- * A --replSet server should be able to set 'recordPreImages' on a collection and then be restarted
- * in standalone mode successfully.
- */
-
-jsTest.log("Starting up a 1-node replica set");
-
-var rst = new ReplSetTest({nodes: 1});
-rst.startSet();
-rst.initiate();
-
-const dbName = 'record_preimage_standalone_mode_test_db';
-const collName = 'testColl';
-let primary = rst.getPrimary();
-let testDB = primary.getDB(dbName);
-let testColl = testDB.getCollection(collName);
-
-jsTest.log("Creating a collection with 'recordPreImages' set to true and adding some data");
-
-assert.commandWorked(testDB.runCommand({create: collName, recordPreImages: true}));
-assert.eq(findCollectionInfo(testDB, collName).options.recordPreImages, true);
-
-assert.commandWorked(testColl.insert({a: 1, b: 1}));
-assert.commandWorked(testColl.update({a: 1}, {a: 2, b: 2}));
-// Ensure all of the writes make it to disk before checkpointing below.
-assert.commandWorked(testColl.update({a: 2}, {a: 3, b: 3}, {writeConcern: {w: 1, j: true}}));
-
-jsTest.log("Forcing a checkpoint to be taken");
-
-// Ensure that the standalone can recover all of the writes from the last checkpoint because
-// standalone mode does not run recovery from the oplog. The last write with j:true write concern
-// ensured that the data reached disk, and now fsync will checkpoint that data.
-assert.commandWorked(primary.adminCommand({fsync: 1}));
-
-jsTest.log("Restarting the replica set member as a standalone node");
-
-printOplog(primary);  // Debugging aid.
-
-let replicaSetNodeId = rst.getNodeId(primary);
-let replicaSetNodeDbpath = primary.dbpath;
-jsTest.log("replicaSetNodeId: " + replicaSetNodeId +
-           ", replicaSetNodeDbpath: " + replicaSetNodeDbpath);
-
-rst.stop(replicaSetNodeId);
-
-let standaloneConn = MongoRunner.runMongod({
-    dbpath: replicaSetNodeDbpath,
-    noCleanData: true,
-});
-
-let standaloneDB = standaloneConn.getDB(dbName);
-let standaloneColl = standaloneDB.getCollection(collName);
-let standaloneOplogColl = standaloneConn.getDB('local').oplog.rs;
-
-assert.eq(findCollectionInfo(standaloneDB, collName).options.recordPreImages, true);
-
-/**
- * A standalone mode server should be able to do writes without triggering the 'recordPreImages'
- * feature because oplog entries are not written in standalone mode: the 'recordPreImages' setting
- * causes additional oplog entries to be written.
- */
-
-const numOplogEntriesBefore = standaloneOplogColl.find().itcount();
-
-jsTest.log(
-    "Updating some data in the collection with 'recordPreImages' set to check that nothing " +
-    "happens in standalone mode");
-
-assert.commandWorked(standaloneColl.insert({c: 1, d: 1}));
-assert.commandWorked(standaloneColl.update({c: 1}, {c: 2, d: 2}));
-assert.commandWorked(standaloneColl.update({c: 2}, {c: 3, d: 3}));
-
-jsTest.log(
-    "Checking that no oplog entries were produced for 'recordPreImages': the feature is inactive");
-
-printOplog(standaloneConn);  // Debugging aid.
-
-const numOplogEntriesAfter = standaloneOplogColl.find().itcount();
-assert.eq(numOplogEntriesBefore, numOplogEntriesAfter);
-
-jsTest.log("Shutting down standalone");
-
-MongoRunner.stopMongod(standaloneConn);
-
-jsTest.log("Restarting the node as a replica set member again and doing some writes");
-
-rst.start(replicaSetNodeId, {}, true /*restart*/);
-
-primary = rst.getPrimary();
-testDB = primary.getDB(dbName);
-testColl = testDB.getCollection(collName);
-
-// Check that everything is still working properly after being in standalone mode.
-assert.eq(findCollectionInfo(testDB, collName).options.recordPreImages, true);
-
-assert.commandWorked(testColl.update({a: 3}, {a: 4, b: 4}));
-assert.commandWorked(testColl.update({a: 4}, {a: 5, b: 5}));
-
-jsTest.log("Shutting down replica set");
-
-rst.stopSet();
-}());