Merge branch 'master' into jenniferpeshansky/SERVER-70854jenniferpeshansky/SERVER-70854

8 files changed, 1297 insertions, 0 deletions

diff --git a/jstests/sharding/analyze_shard_key/libs/query_sampling_util.js b/jstests/sharding/analyze_shard_key/libs/query_sampling_util.js
new file mode 100644
index 00000000000..9f3e07d8e50
--- /dev/null
+++ b/jstests/sharding/analyze_shard_key/libs/query_sampling_util.js
@@ -0,0 +1,120 @@
+/**
+ * Utilities for testing query sampling.
+ */
+var QuerySamplingUtil = (function() {
+    load("jstests/libs/uuid_util.js");
+    load("jstests/sharding/analyze_shard_key/libs/analyze_shard_key_util.js");
+
+    function getCollectionUuid(db, collName) {
+        const listCollectionRes =
+            assert.commandWorked(db.runCommand({listCollections: 1, filter: {name: collName}}));
+        return listCollectionRes.cursor.firstBatch[0].info.uuid;
+    }
+
+    function generateRandomString(length = 5) {
+        return extractUUIDFromObject(UUID()).substring(0, length);
+    }
+
+    function generateRandomCollation() {
+        return {locale: "en_US", strength: AnalyzeShardKeyUtil.getRandInteger(1, 5)};
+    }
+
+    function makeCmdObjIgnoreSessionInfo(originalCmdObj) {
+        const modifiedCmdObj = Object.extend({}, originalCmdObj);
+        delete modifiedCmdObj["lsid"];
+        delete modifiedCmdObj["txnNumber"];
+        return modifiedCmdObj;
+    }
+
+    /**
+     * Waits for the config.sampledQueries collection to have 'expectedSampledQueryDocs.length'
+     * number of documents for the collection 'ns'. For every (sampleId, cmdName, cmdObj) in
+     * 'expectedSampledQueryDocs', asserts that there is a config.sampledQueries document with _id
+     * equal to sampleId and that it has the given fields.
+     */
+    function assertSoonSampledQueryDocuments(conn, ns, collectionUuid, expectedSampledQueryDocs) {
+        const coll = conn.getCollection("config.sampledQueries");
+
+        let actualSampledQueryDocs;
+        assert.soon(() => {
+            actualSampledQueryDocs = coll.find({ns}).toArray();
+            return actualSampledQueryDocs.length >= expectedSampledQueryDocs.length;
+        }, "timed out waiting for sampled query documents");
+        assert.eq(actualSampledQueryDocs.length,
+                  expectedSampledQueryDocs.length,
+                  {actualSampledQueryDocs, expectedSampledQueryDocs});
+
+        for (let {sampleId, cmdName, cmdObj} of expectedSampledQueryDocs) {
+            const doc = coll.findOne({_id: sampleId});
+
+            assert.neq(doc, null);
+            assert.eq(doc.ns, ns, doc);
+            assert.eq(doc.collectionUuid, collectionUuid, doc);
+            assert.eq(doc.cmdName, cmdName, doc);
+
+            for (let key in cmdObj) {
+                const value = cmdObj[key];
+                if (typeof value === 'object') {
+                    for (let subKey in value) {
+                        assert.eq(doc.cmd[key][subKey],
+                                  cmdObj[key][subKey],
+                                  {subKey, actual: doc.cmd, expected: cmdObj});
+                    }
+                } else {
+                    assert.eq(doc.cmd[key], cmdObj[key], {key, actual: doc.cmd, expected: cmdObj});
+                }
+            }
+        }
+    }
+
+    function assertNoSampledQueryDocuments(conn, ns) {
+        const coll = conn.getCollection("config.sampledQueries");
+        assert.eq(coll.find({ns}).itcount(), 0);
+    }
+
+    /**
+     * Waits for the config.sampledQueriesDiff collection to have a document with _id equal to
+     * sampleId, and then asserts that the diff in that document matches one of the diffs in
+     * 'expectedSampledDiffs'.
+     */
+    function assertSoonSingleSampledDiffDocument(
+        conn, sampleId, ns, collectionUuid, expectedSampledDiffs) {
+        const coll = conn.getCollection("config.sampledQueriesDiff");
+
+        assert.soon(() => {
+            const doc = coll.findOne({_id: sampleId});
+            if (!doc) {
+                return false;
+            }
+            assert.eq(doc.ns, ns, doc);
+            assert.eq(doc.collectionUuid, collectionUuid, doc);
+            assert(expectedSampledDiffs.some(diff => {
+                return bsonUnorderedFieldsCompare(doc.diff, diff) === 0;
+            }),
+                   doc);
+            return true;
+        });
+    }
+
+    function assertNoSampledDiffDocuments(conn, ns) {
+        const coll = conn.getCollection("config.sampledQueriesDiff");
+        assert.eq(coll.find({ns: ns}).itcount(), 0);
+    }
+
+    function clearSampledDiffCollection(primary) {
+        const coll = primary.getCollection("config.sampledQueriesDiff");
+        assert.commandWorked(coll.remove({}));
+    }
+
+    return {
+        getCollectionUuid,
+        generateRandomString,
+        generateRandomCollation,
+        makeCmdObjIgnoreSessionInfo,
+        assertSoonSampledQueryDocuments,
+        assertNoSampledQueryDocuments,
+        assertSoonSingleSampledDiffDocument,
+        assertNoSampledDiffDocuments,
+        clearSampledDiffCollection
+    };
+})();
diff --git a/jstests/sharding/analyze_shard_key/persist_sampled_diffs.js b/jstests/sharding/analyze_shard_key/persist_sampled_diffs.js
new file mode 100644
index 00000000000..77b34f52295
--- /dev/null
+++ b/jstests/sharding/analyze_shard_key/persist_sampled_diffs.js
@@ -0,0 +1,200 @@
+/**
+ * Tests that shardsvr mongods support persisting diff for sampled write queries and non-shardsvr
+ * mongods don't support that. Specifically, tests that each write query on a shardsvr mongod
+ * generates at most one document regardless of the number of documents that it modifies.
+ *
+ * @tags: [requires_fcv_62, featureFlagAnalyzeShardKey]
+ */
+(function() {
+"use strict";
+
+load("jstests/sharding/analyze_shard_key/libs/query_sampling_util.js");
+
+const testCases = [];
+
+// multi=false update.
+for (const updateType of ["modifier", "replacement", "pipeline"]) {
+    const preImageDocs = [{a: 1}];
+    const postImageDocs = [{a: 2, b: 0}];
+    const updateOp = (() => {
+        switch (updateType) {
+            case "modifier":
+                return {$mul: {a: 2}, $set: {b: 0}};
+            case "replacement":
+                return {a: 2, b: 0};
+            case "pipeline":
+                return [{$set: {a: 2}}, {$set: {b: 0}}];
+            default:
+                throw "Unexpected update type";
+        }
+    })();
+    const makeCmdObjFuncs = [
+        (collName) => {
+            const sampleId = UUID();
+            const cmdObj = {findAndModify: collName, query: {a: 1}, update: updateOp, sampleId};
+            return {sampleId, cmdObj};
+        },
+        (collName) => {
+            const sampleId = UUID();
+            const cmdObj = {
+                update: collName,
+                updates: [{q: {a: 1}, u: updateOp, multi: false, sampleId}]
+            };
+            return {sampleId, cmdObj};
+        }
+    ];
+    const expectedDiffs = [{a: 'u', b: 'i'}];
+
+    testCases.push({preImageDocs, postImageDocs, updateType, makeCmdObjFuncs, expectedDiffs});
+}
+
+// multi=true update.
+for (const updateType of ["modifier", "pipeline"]) {
+    const preImageDocs = [{a: 0}, {a: 1}];
+    const postImageDocs = [{a: 1, b: 0}, {a: 1, b: 0}];
+    const updateOp = (() => {
+        switch (updateType) {
+            case "modifier":
+                return {$set: {a: 1, b: 0}};
+            case "pipeline":
+                return [{$set: {a: 1}}, {$set: {b: 0}}];
+            default:
+                throw "Unexpected update type";
+        }
+    })();
+    const makeCmdObjFuncs = [(collName) => {
+        const sampleId = UUID();
+        const cmdObj = {
+            update: collName,
+            updates: [{q: {a: {$gte: 0}}, u: updateOp, multi: true, sampleId}]
+        };
+        return {sampleId, cmdObj};
+    }];
+    const expectedDiffs = [{a: 'u', b: 'i'}, {b: 'i'}];
+
+    testCases.push({preImageDocs, postImageDocs, updateType, makeCmdObjFuncs, expectedDiffs});
+}
+
+// no diff.
+for (const updateType of ["modifier", "replacement", "pipeline"]) {
+    const preImageDocs = [{a: 0}];
+    const postImageDocs = [{a: 0}];
+    const updateOp = (() => {
+        switch (updateType) {
+            case "modifier":
+                return {$mul: {a: 0}};
+            case "replacement":
+                return {a: 0};
+            case "pipeline":
+                return [{$set: {a: 0}}];
+            default:
+                throw "Unexpected update type";
+        }
+    })();
+    const makeCmdObjFuncs = [(collName) => {
+        const sampleId = UUID();
+        const cmdObj = {
+            update: collName,
+            updates: [{q: {a: 0}, u: updateOp, multi: false, sampleId}]
+        };
+        return {sampleId, cmdObj};
+    }];
+    const expectedDiffs = [];
+
+    testCases.push({preImageDocs, postImageDocs, updateType, makeCmdObjFuncs, expectedDiffs});
+}
+
+// Make the periodic job for writing sampled queries have a period of 1 second to speed up the test.
+const queryAnalysisWriterIntervalSecs = 1;
+
+function testDiffs(rst, testCase, expectSampling) {
+    // If running on the config server, use "config" as the database name since it is illegal to
+    // create a user database on the config server.
+    const dbName = rst.isConfigRS ? "config" : "testDb";
+    const collName = "testColl-" + QuerySamplingUtil.generateRandomString();
+    const ns = dbName + "." + collName;
+
+    const primary = rst.getPrimary();
+    const db = primary.getDB(dbName);
+    const coll = db.getCollection(collName);
+    assert.commandWorked(db.createCollection(collName));
+    const collectionUuid = QuerySamplingUtil.getCollectionUuid(db, collName);
+
+    for (const makeCmdObjFunc of testCase.makeCmdObjFuncs) {
+        assert.commandWorked(coll.insert(testCase.preImageDocs));
+
+        const {sampleId, cmdObj} = makeCmdObjFunc(collName);
+
+        jsTest.log(`Testing test case ${tojson({
+            dbName,
+            collName,
+            preImageDocs: testCase.preImageDocs,
+            postImageDocs: testCase.postImageDocs,
+            updateType: testCase.updateType,
+            cmdObj
+        })}`);
+        const res = assert.commandWorked(db.runCommand(cmdObj));
+
+        const cmdName = Object.keys(cmdObj)[0];
+        if (cmdName == "update") {
+            assert.eq(res.n, testCase.postImageDocs.length, res);
+        } else if (cmdName == "findAndModify") {
+            assert.eq(res.lastErrorObject.n, testCase.postImageDocs.length, res);
+        } else {
+            throw Error("Unknown command " + tojson(cmdObj));
+        }
+        for (const postImageDoc of testCase.postImageDocs) {
+            assert.neq(coll.findOne(postImageDoc), null, coll.find().toArray());
+        }
+
+        if (expectSampling && testCase.expectedDiffs.length > 0) {
+            QuerySamplingUtil.assertSoonSingleSampledDiffDocument(
+                primary, sampleId, ns, collectionUuid, testCase.expectedDiffs);
+        } else {
+            // Wait for one interval before asserting to verify that the writes did not occur.
+            sleep(queryAnalysisWriterIntervalSecs * 1000);
+            QuerySamplingUtil.assertNoSampledDiffDocuments(primary, ns);
+        }
+
+        assert.commandWorked(coll.remove({}));
+        QuerySamplingUtil.clearSampledDiffCollection(primary);
+    }
+}
+
+{
+    const st = new ShardingTest({
+        shards: 1,
+        rs: {nodes: 2, setParameter: {queryAnalysisWriterIntervalSecs}},
+        // There is no periodic job for writing sample queries on the non-shardsvr mongods but set
+        // it anyway to verify that no queries are sampled.
+        other: {configOptions: {setParameter: {queryAnalysisWriterIntervalSecs}}},
+    });
+    // It is illegal to create a user database on the config server. Set 'isConfigRS' to true to
+    // allow the test helper to know if it should use "config" as the name for the test database.
+    st.configRS.isConfigRS = true;
+
+    for (const testCase of testCases) {
+        testDiffs(st.rs0, testCase, true /* expectSampling */);
+        testDiffs(st.configRS, testCase, false /* expectSampling */);
+    }
+
+    st.stop();
+}
+
+{
+    const rst = new ReplSetTest({
+        nodes: 2,
+        // There is no periodic job for writing sample queries on the non-shardsvr mongods but set
+        // it anyway to verify that no queries are sampled.
+        setParameter: {queryAnalysisWriterIntervalSecs}
+    });
+    rst.startSet();
+    rst.initiate();
+
+    for (const testCase of testCases) {
+        testDiffs(rst, testCase, false /* expectSampling */);
+    }
+
+    rst.stopSet();
+}
+})();
diff --git a/jstests/sharding/analyze_shard_key/persist_sampled_queries_failover.js b/jstests/sharding/analyze_shard_key/persist_sampled_queries_failover.js
new file mode 100644
index 00000000000..79f1aceb949
--- /dev/null
+++ b/jstests/sharding/analyze_shard_key/persist_sampled_queries_failover.js
@@ -0,0 +1,105 @@
+/**
+ * Tests that the periodic job for persisting sampled queries on shardsvr mongods can handle
+ * failover.
+ *
+ * @tags: [requires_fcv_62, featureFlagAnalyzeShardKey]
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/fail_point_util.js");
+load("jstests/sharding/analyze_shard_key/libs/query_sampling_util.js");
+
+function testStepDown(rst) {
+    const dbName = "testDb";
+    const collName = "testCollStepDown";
+    const ns = dbName + "." + collName;
+
+    let primary = rst.getPrimary();
+    let primaryDB = primary.getDB(dbName);
+
+    assert.commandWorked(primaryDB.getCollection(collName).insert({a: 0}));
+    const collectionUuid = QuerySamplingUtil.getCollectionUuid(primaryDB, collName);
+
+    const localWriteFp =
+        configureFailPoint(primary, "hangQueryAnalysisWriterBeforeWritingLocally", {}, {times: 1});
+
+    const originalCmdObj =
+        {findAndModify: collName, query: {a: 0}, update: {a: 1}, sampleId: UUID()};
+    const expectedSampledQueryDocs =
+        [{sampleId: originalCmdObj.sampleId, cmdName: "findAndModify", cmdObj: originalCmdObj}];
+    const expectedDiff = {a: "u"};
+
+    assert.commandWorked(primaryDB.getCollection(collName).runCommand(originalCmdObj));
+
+    localWriteFp.wait();
+
+    assert.commandWorked(
+        primary.adminCommand({replSetStepDown: ReplSetTest.kForeverSecs, force: true}));
+    primary = rst.getPrimary();
+    primaryDB = primary.getDB(dbName);
+
+    localWriteFp.off();
+
+    // Verify that the sampled query above did not go missing because of the retryable error caused
+    // by stepdown.
+    QuerySamplingUtil.assertSoonSampledQueryDocuments(
+        primary, ns, collectionUuid, expectedSampledQueryDocs);
+    QuerySamplingUtil.assertSoonSingleSampledDiffDocument(
+        primary, originalCmdObj.sampleId, ns, collectionUuid, [expectedDiff]);
+}
+
+function testStepUp(rst) {
+    const dbName = "testDb";
+    const collName = "testCollStepUp";
+    const ns = dbName + "." + collName;
+
+    let primary = rst.getPrimary();
+    const secondary = rst.getSecondary();
+    const primaryDB = primary.getDB(dbName);
+    const secondaryTestDB = secondary.getDB(dbName);
+
+    assert.commandWorked(primaryDB.createCollection(collName));
+    const collectionUuid = QuerySamplingUtil.getCollectionUuid(primaryDB, collName);
+    // Wait for the collection to also exist on secondaries.
+    rst.awaitReplication();
+
+    const originalCmdObj = {count: collName, query: {a: 2}, sampleId: UUID()};
+    const expectedSampledQueryDocs = [{
+        sampleId: originalCmdObj.sampleId,
+        cmdName: "count",
+        cmdObj: {filter: originalCmdObj.query}
+    }];
+
+    const remoteWriteFp =
+        configureFailPoint(secondary, "hangQueryAnalysisWriterBeforeWritingRemotely");
+    assert.commandWorked(secondaryTestDB.getCollection(collName).runCommand(originalCmdObj));
+
+    remoteWriteFp.wait();
+    assert.commandWorked(secondary.adminCommand({replSetFreeze: 0}));
+    assert.commandWorked(
+        primary.adminCommand({replSetStepDown: ReplSetTest.kForeverSecs, force: true}));
+    primary = rst.getPrimary();
+
+    remoteWriteFp.off();
+
+    // Verify that the sampled query above did not go missing because the node stepped up.
+    QuerySamplingUtil.assertSoonSampledQueryDocuments(
+        primary, ns, collectionUuid, expectedSampledQueryDocs);
+}
+
+const st = new ShardingTest({
+    shards: 1,
+    rs: {
+        nodes: 2,
+        // Make the periodic job for writing sampled queries have a period of 1 second to speed up
+        // the test.
+        setParameter: {queryAnalysisWriterIntervalSecs: 1}
+    }
+});
+
+testStepDown(st.rs0);
+testStepUp(st.rs0);
+
+st.stop();
+})();
diff --git a/jstests/sharding/analyze_shard_key/persist_sampled_read_queries.js b/jstests/sharding/analyze_shard_key/persist_sampled_read_queries.js
new file mode 100644
index 00000000000..5d23bc19461
--- /dev/null
+++ b/jstests/sharding/analyze_shard_key/persist_sampled_read_queries.js
@@ -0,0 +1,209 @@
+/**
+ * Tests that shardsvr mongods (both primary and secondary) support persisting sampled read queries
+ * and that non-shardsvr mongods don't support that.
+ *
+ * @tags: [requires_fcv_62, featureFlagAnalyzeShardKey]
+ */
+(function() {
+"use strict";
+
+load("jstests/sharding/analyze_shard_key/libs/query_sampling_util.js");
+
+const shardsvrTestCases = [
+    {collectionExists: true, markForSampling: true, expectSampling: true},
+    {collectionExists: true, markForSampling: false, expectSampling: false},
+    {collectionExists: false, markForSampling: true, expectSampling: false},
+];
+
+const nonShardsvrTestCases = [
+    {collectionExists: true, markForSampling: true, expectSampling: false},
+];
+
+// Test with empty, non-empty and missing filter and/or collation to verify that query sampling
+// doesn't require filter or collation to be non-empty.
+const filterCollationTestCases = [
+    {filter: {a: 0}, collation: QuerySamplingUtil.generateRandomCollation()},
+    {filter: {a: 1}, collation: {}},
+    {filter: {a: 2}},
+    {collation: QuerySamplingUtil.generateRandomCollation()},
+    {filter: {}, collation: QuerySamplingUtil.generateRandomCollation()},
+    {filter: {}, collation: {}},
+];
+
+// Make the periodic job for writing sampled queries have a period of 1 second to speed up the test.
+const queryAnalysisWriterIntervalSecs = 1;
+
+function testReadCmd(rst, cmdOpts, testCase) {
+    // If running on the config server, use "config" as the database name since it is illegal to
+    // create a user database on the config server.
+    const dbName = rst.isConfigRS ? "config" : "testDb";
+    const collName = "testColl-" + cmdOpts.cmdName + "-" + QuerySamplingUtil.generateRandomString();
+    const ns = dbName + "." + collName;
+
+    const primary = rst.getPrimary();
+    const secondary = rst.getSecondary();
+    const primaryDB = primary.getDB(dbName);
+    const secondaryDB = secondary.getDB(dbName);
+
+    let collectionUuid;
+    if (testCase.collectionExists) {
+        assert.commandWorked(primaryDB.createCollection(collName));
+        collectionUuid = QuerySamplingUtil.getCollectionUuid(primaryDB, collName);
+        // Wait for the collection to also exist on secondaries since some of the sampled queries
+        // below may be sent to a secondary.
+        rst.awaitReplication();
+    }
+
+    const expectedSampledQueryDocs = [];
+    for (let {filter, collation} of filterCollationTestCases) {
+        if (!filter && !cmdOpts.isFilterOptional) {
+            continue;
+        }
+
+        const originalCmdObj = cmdOpts.makeCmdObjFunc(collName, filter);
+        if (collation !== undefined) {
+            originalCmdObj.collation = collation;
+        }
+        if (testCase.markForSampling) {
+            originalCmdObj.sampleId = UUID();
+
+            if (testCase.expectSampling) {
+                expectedSampledQueryDocs.push({
+                    sampleId: originalCmdObj.sampleId,
+                    cmdName: cmdOpts.cmdName,
+                    cmdObj: {
+                        filter: filter ? filter : {},
+                        collation: collation ? collation : cmdOpts.defaultCollation
+                    }
+                });
+            }
+        }
+
+        const db = Math.random() < 0.5 ? primaryDB : secondaryDB;
+        jsTest.log(`Testing test case ${tojson(testCase)} with ${
+            tojson({dbName, collName, originalCmdObj, host: db.getMongo().host})}`);
+        assert.commandWorked(db.runCommand(originalCmdObj));
+    }
+
+    if (testCase.expectSampling) {
+        QuerySamplingUtil.assertSoonSampledQueryDocuments(
+            primary, ns, collectionUuid, expectedSampledQueryDocs);
+    } else {
+        // To verify that no writes occurred, wait for one interval before asserting.
+        sleep(queryAnalysisWriterIntervalSecs * 1000);
+        QuerySamplingUtil.assertNoSampledQueryDocuments(primary, ns);
+    }
+}
+
+function testFindCmd(rst, testCases) {
+    const cmdName = "find";
+    const isFilterOptional = false;
+    const defaultCollation = {};
+    const makeCmdObjFunc = (collName, filter) => {
+        return {find: collName, filter};
+    };
+
+    const cmdOpts = {cmdName, isFilterOptional, defaultCollation, makeCmdObjFunc};
+    for (let testCase of testCases) {
+        testReadCmd(rst, cmdOpts, testCase);
+    }
+}
+
+function testCountCmd(rst, testCases) {
+    const cmdName = "count";
+    const isFilterOptional = false;
+    const defaultCollation = {};
+    const makeCmdObjFunc = (collName, filter) => {
+        return {count: collName, query: filter};
+    };
+
+    const cmdOpts = {cmdName, isFilterOptional, defaultCollation, makeCmdObjFunc};
+    for (let testCase of testCases) {
+        testReadCmd(rst, cmdOpts, testCase);
+    }
+}
+
+function testDistinctCmd(rst, testCases) {
+    const cmdName = "distinct";
+    const isFilterOptional = true;
+    const defaultCollation = {};
+    const makeCmdObjFunc = (collName, filter) => {
+        const originalCmdObj = {distinct: collName, key: "a"};
+        if (filter !== undefined) {
+            originalCmdObj.query = filter;
+        }
+        return originalCmdObj;
+    };
+
+    const cmdOpts = {cmdName, isFilterOptional, defaultCollation, makeCmdObjFunc};
+    for (let testCase of testCases) {
+        testReadCmd(rst, cmdOpts, testCase);
+    }
+}
+
+function testAggregateCmd(rst, testCases) {
+    const cmdName = "aggregate";
+    const isFilterOptional = true;
+    // When the collation is unspecified, the aggregate command explicity sets it to the simple
+    // collation.
+    const defaultCollation = {locale: "simple"};
+    const makeCmdObjFunc = (collName, filter) => {
+        if (filter == undefined) {
+            return {
+                aggregate: collName,
+                pipeline: [{$group: {_id: "$a", count: {$sum: 1}}}],
+                cursor: {}
+            };
+        }
+        return {aggregate: collName, pipeline: [{$match: filter}], cursor: {}};
+    };
+
+    const cmdOpts = {cmdName, isFilterOptional, defaultCollation, makeCmdObjFunc};
+    for (let testCase of testCases) {
+        testReadCmd(rst, cmdOpts, testCase);
+    }
+}
+
+{
+    const st = new ShardingTest({
+        shards: 1,
+        rs: {nodes: 2, setParameter: {queryAnalysisWriterIntervalSecs}},
+        // There is no periodic job for writing sample queries on the non-shardsvr mongods but set
+        // it anyway to verify that no queries are sampled.
+        other: {configOptions: {setParameter: {queryAnalysisWriterIntervalSecs}}},
+    });
+    // It is illegal to create a user database on the config server. Set 'isConfigRS' to true to
+    // allow the test helper to know if it should use "config" as the name for the test database.
+    st.configRS.isConfigRS = true;
+
+    testFindCmd(st.rs0, shardsvrTestCases);
+    testCountCmd(st.rs0, shardsvrTestCases);
+    testDistinctCmd(st.rs0, shardsvrTestCases);
+    testAggregateCmd(st.rs0, shardsvrTestCases);
+
+    testFindCmd(st.configRS, nonShardsvrTestCases);
+    testCountCmd(st.configRS, nonShardsvrTestCases);
+    testDistinctCmd(st.configRS, nonShardsvrTestCases);
+    testAggregateCmd(st.configRS, nonShardsvrTestCases);
+
+    st.stop();
+}
+
+{
+    const rst = new ReplSetTest({
+        nodes: 2,
+        // There is no periodic job for writing sample queries on the non-shardsvr mongods but set
+        // it anyway to verify that no queries are sampled.
+        setParameter: {queryAnalysisWriterIntervalSecs}
+    });
+    rst.startSet();
+    rst.initiate();
+
+    testFindCmd(rst, nonShardsvrTestCases);
+    testCountCmd(rst, nonShardsvrTestCases);
+    testDistinctCmd(rst, nonShardsvrTestCases);
+    testAggregateCmd(rst, nonShardsvrTestCases);
+
+    rst.stopSet();
+}
+})();
diff --git a/jstests/sharding/analyze_shard_key/persist_sampled_retryable_delete_queries.js b/jstests/sharding/analyze_shard_key/persist_sampled_retryable_delete_queries.js
new file mode 100644
index 00000000000..786220227f0
--- /dev/null
+++ b/jstests/sharding/analyze_shard_key/persist_sampled_retryable_delete_queries.js
@@ -0,0 +1,136 @@
+/**
+ * Tests that retrying a retryable delete doesn't cause it to have multiple sampled query documents.
+ *
+ * @tags: [requires_fcv_62, featureFlagAnalyzeShardKey]
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/fail_point_util.js");
+load("jstests/sharding/analyze_shard_key/libs/query_sampling_util.js");
+
+// Make the periodic job for writing sampled queries have a period of 1 second to speed up the test.
+const queryAnalysisWriterIntervalSecs = 1;
+
+function testRetryExecutedWrite(rst) {
+    const dbName = "testDb";
+    const collName = "testCollExecutedWrite";
+    const ns = dbName + "." + collName;
+
+    const lsid = {id: UUID()};
+    const txnNumber = NumberLong(1);
+
+    const primary = rst.getPrimary();
+    const db = primary.getDB(dbName);
+    const coll = db.getCollection(collName);
+    assert.commandWorked(coll.insert([{a: -1}, {a: 0}]));
+    const collectionUuid = QuerySamplingUtil.getCollectionUuid(db, collName);
+
+    const deleteOp0 = {q: {a: 0}, limit: 1, sampleId: UUID()};
+    const deleteOp1 = {q: {a: {$lt: 1}}, limit: 1, sampleId: UUID()};
+
+    const originalCmdObj = {delete: collName, deletes: [deleteOp0], lsid, txnNumber};
+    const expectedSampledQueryDocs = [{
+        sampleId: deleteOp0.sampleId,
+        cmdName: "delete",
+        cmdObj: QuerySamplingUtil.makeCmdObjIgnoreSessionInfo(originalCmdObj)
+    }];
+
+    const originalRes = assert.commandWorked(db.runCommand(originalCmdObj));
+    assert.eq(originalRes.n, 1, originalRes);
+
+    QuerySamplingUtil.assertSoonSampledQueryDocuments(
+        primary, ns, collectionUuid, expectedSampledQueryDocs);
+
+    // Retry deleteOp0 with the same sampleId but batched with the new deleteOp1.
+    const retryCmdObj0 = Object.assign({}, originalCmdObj);
+    retryCmdObj0.deletes = [deleteOp0, deleteOp1];
+    expectedSampledQueryDocs.push({
+        sampleId: deleteOp1.sampleId,
+        cmdName: "delete",
+        cmdObj: Object.assign(QuerySamplingUtil.makeCmdObjIgnoreSessionInfo(retryCmdObj0),
+                              {deletes: [deleteOp1]})
+    });
+
+    const retryRes0 = assert.commandWorked(db.runCommand(retryCmdObj0));
+    assert.eq(retryRes0.n, 2, retryRes0);
+
+    QuerySamplingUtil.assertSoonSampledQueryDocuments(
+        primary, ns, collectionUuid, expectedSampledQueryDocs);
+
+    // Retry both deleteOp0 and deleteOp1 different sampleIds.
+    const retryCmdObj1 = Object.assign({}, retryCmdObj0);
+    retryCmdObj1.deletes = [
+        Object.assign({}, deleteOp0, {sampleId: UUID()}),
+        Object.assign({}, deleteOp1, {sampleId: UUID()})
+    ];
+
+    const retryRes1 = assert.commandWorked(db.runCommand(retryCmdObj1));
+    assert.eq(retryRes1.n, 2, retryRes1);
+
+    // Wait for one interval to verify that no writes occurred as a result of the retry.
+    sleep(queryAnalysisWriterIntervalSecs * 1000);
+
+    QuerySamplingUtil.assertSoonSampledQueryDocuments(
+        primary, ns, collectionUuid, expectedSampledQueryDocs);
+}
+
+function testRetryUnExecutedWrite(rst) {
+    const dbName = "testDb";
+    const collName = "testCollUnExecutedWrite";
+    const ns = dbName + "." + collName;
+
+    const lsid = {id: UUID()};
+    const txnNumber = NumberLong(1);
+
+    const primary = rst.getPrimary();
+    const db = primary.getDB(dbName);
+    const coll = db.getCollection(collName);
+    assert.commandWorked(coll.insert({a: 0}));
+    const collectionUuid = QuerySamplingUtil.getCollectionUuid(db, collName);
+
+    const deleteOp0 = {q: {a: 0}, limit: 1, sampleId: UUID()};
+    const originalCmdObj = {delete: collName, deletes: [deleteOp0], lsid, txnNumber};
+    const expectedSampledQueryDocs = [{
+        sampleId: deleteOp0.sampleId,
+        cmdName: "delete",
+        cmdObj: QuerySamplingUtil.makeCmdObjIgnoreSessionInfo(originalCmdObj)
+    }];
+
+    const fp = configureFailPoint(primary, "failAllRemoves");
+
+    // The delete fails after it has been added to the sample buffer.
+    assert.commandFailedWithCode(db.runCommand(originalCmdObj), ErrorCodes.InternalError);
+
+    QuerySamplingUtil.assertSoonSampledQueryDocuments(
+        primary, ns, collectionUuid, expectedSampledQueryDocs);
+
+    fp.off();
+
+    // Retry with the same sampleId.
+    const retryCmdObj = originalCmdObj;
+    const retryRes = assert.commandWorked(db.runCommand(retryCmdObj));
+    assert.eq(retryRes.n, 1, retryRes);
+
+    // Wait for one interval to verify that no writes occurred as a result of the retry.
+    sleep(queryAnalysisWriterIntervalSecs * 1000);
+
+    QuerySamplingUtil.assertSoonSampledQueryDocuments(
+        primary, ns, collectionUuid, expectedSampledQueryDocs);
+}
+
+const st = new ShardingTest({
+    shards: 1,
+    rs: {
+        nodes: 2,
+        // Make the periodic job for writing sampled queries have a period of 1 second to speed up
+        // the test.
+        setParameter: {queryAnalysisWriterIntervalSecs}
+    }
+});
+
+testRetryExecutedWrite(st.rs0);
+testRetryUnExecutedWrite(st.rs0);
+
+st.stop();
+})();
diff --git a/jstests/sharding/analyze_shard_key/persist_sampled_retryable_findAndModify_queries.js b/jstests/sharding/analyze_shard_key/persist_sampled_retryable_findAndModify_queries.js
new file mode 100644
index 00000000000..0cbcd378f93
--- /dev/null
+++ b/jstests/sharding/analyze_shard_key/persist_sampled_retryable_findAndModify_queries.js
@@ -0,0 +1,138 @@
+/**
+ * Tests that retrying a retryable findAndModify doesn't cause it to have multiple sampled query
+ * documents.
+ *
+ * @tags: [requires_fcv_62, featureFlagAnalyzeShardKey]
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/fail_point_util.js");
+load("jstests/sharding/analyze_shard_key/libs/query_sampling_util.js");
+
+// Make the periodic job for writing sampled queries have a period of 1 second to speed up the test.
+const queryAnalysisWriterIntervalSecs = 1;
+
+function testRetryExecutedWrite(rst) {
+    const dbName = "testDb";
+    const collName = "testCollExecutedWrite";
+    const ns = dbName + "." + collName;
+
+    const lsid = {id: UUID()};
+    const txnNumber = NumberLong(1);
+
+    const primary = rst.getPrimary();
+    const db = primary.getDB(dbName);
+    const coll = db.getCollection(collName);
+    assert.commandWorked(coll.insert({a: 0}));
+    const collectionUuid = QuerySamplingUtil.getCollectionUuid(db, collName);
+
+    const originalCmdObj = {
+        findAndModify: collName,
+        query: {a: 0},
+        update: {$inc: {a: 1}},
+        sampleId: UUID(),
+        lsid,
+        txnNumber
+    };
+    const expectedSampledQueryDocs = [{
+        sampleId: originalCmdObj.sampleId,
+        cmdName: "findAndModify",
+        cmdObj: QuerySamplingUtil.makeCmdObjIgnoreSessionInfo(originalCmdObj)
+    }];
+
+    const originalRes = assert.commandWorked(db.runCommand(originalCmdObj));
+    assert.eq(originalRes.lastErrorObject.n, 1, originalRes);
+
+    QuerySamplingUtil.assertSoonSampledQueryDocuments(
+        primary, ns, collectionUuid, expectedSampledQueryDocs);
+
+    // Retry with the same sampleId.
+    const retryCmdObj0 = originalCmdObj;
+    const retryRes0 = assert.commandWorked(db.runCommand(retryCmdObj0));
+    assert.eq(retryRes0.lastErrorObject.n, 1, retryRes0);
+
+    // Wait for one interval to verify that no writes occurred as a result of the retry.
+    sleep(queryAnalysisWriterIntervalSecs * 1000);
+
+    QuerySamplingUtil.assertSoonSampledQueryDocuments(
+        primary, ns, collectionUuid, expectedSampledQueryDocs);
+
+    // Retry with a different sampleId.
+    const retryCmdObj1 = Object.assign({}, originalCmdObj);
+    retryCmdObj1.sampleId = UUID();
+    const retryRes1 = assert.commandWorked(db.runCommand(retryCmdObj1));
+    assert.eq(retryRes1.lastErrorObject.n, 1, retryRes1);
+
+    // Wait for one interval to verify that no writes occurred as a result of the retry.
+    sleep(queryAnalysisWriterIntervalSecs * 1000);
+
+    QuerySamplingUtil.assertSoonSampledQueryDocuments(
+        primary, ns, collectionUuid, expectedSampledQueryDocs);
+}
+
+function testRetryUnExecutedWrite(rst) {
+    const dbName = "testDb";
+    const collName = "testCollUnExecutedWrite";
+    const ns = dbName + "." + collName;
+
+    const lsid = {id: UUID()};
+    const txnNumber = NumberLong(1);
+
+    const primary = rst.getPrimary();
+    const db = primary.getDB(dbName);
+    const coll = db.getCollection(collName);
+    assert.commandWorked(coll.insert({a: 0}));
+    const collectionUuid = QuerySamplingUtil.getCollectionUuid(db, collName);
+
+    const originalCmdObj = {
+        findAndModify: collName,
+        query: {a: 0},
+        update: {$inc: {a: 1}},
+        sampleId: UUID(),
+        lsid,
+        txnNumber
+    };
+    const expectedSampledQueryDocs = [{
+        sampleId: originalCmdObj.sampleId,
+        cmdName: "findAndModify",
+        cmdObj: QuerySamplingUtil.makeCmdObjIgnoreSessionInfo(originalCmdObj)
+    }];
+
+    const fp = configureFailPoint(primary, "failAllFindAndModify");
+
+    // The findAndModify fails after it has been added to the sample buffer.
+    assert.commandFailedWithCode(db.runCommand(originalCmdObj), ErrorCodes.InternalError);
+
+    QuerySamplingUtil.assertSoonSampledQueryDocuments(
+        primary, ns, collectionUuid, expectedSampledQueryDocs);
+
+    fp.off();
+
+    // Retry with the same sampleId.
+    const retryCmdObj = originalCmdObj;
+    const retryRes = assert.commandWorked(db.runCommand(retryCmdObj));
+    assert.eq(retryRes.lastErrorObject.n, 1, retryRes);
+
+    // Wait for one interval to verify that no writes occurred as a result of the retry.
+    sleep(queryAnalysisWriterIntervalSecs * 1000);
+
+    QuerySamplingUtil.assertSoonSampledQueryDocuments(
+        primary, ns, collectionUuid, expectedSampledQueryDocs);
+}
+
+const st = new ShardingTest({
+    shards: 1,
+    rs: {
+        nodes: 2,
+        // Make the periodic job for writing sampled queries have a period of 1 second to speed up
+        // the test.
+        setParameter: {queryAnalysisWriterIntervalSecs}
+    }
+});
+
+testRetryExecutedWrite(st.rs0);
+testRetryUnExecutedWrite(st.rs0);
+
+st.stop();
+})();
diff --git a/jstests/sharding/analyze_shard_key/persist_sampled_retryable_update_queries.js b/jstests/sharding/analyze_shard_key/persist_sampled_retryable_update_queries.js
new file mode 100644
index 00000000000..f1733c96ffd
--- /dev/null
+++ b/jstests/sharding/analyze_shard_key/persist_sampled_retryable_update_queries.js
@@ -0,0 +1,137 @@
+/**
+ * Tests that retrying a retryable update doesn't cause it to have multiple sampled query documents.
+ *
+ * @tags: [requires_fcv_62, featureFlagAnalyzeShardKey]
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/fail_point_util.js");
+load("jstests/sharding/analyze_shard_key/libs/query_sampling_util.js");
+
+// Make the periodic job for writing sampled queries have a period of 1 second to speed up the test.
+const queryAnalysisWriterIntervalSecs = 1;
+
+function testRetryExecutedWrite(rst) {
+    const dbName = "testDb";
+    const collName = "testCollExecutedWrite";
+    const ns = dbName + "." + collName;
+
+    const lsid = {id: UUID()};
+    const txnNumber = NumberLong(1);
+
+    const primary = rst.getPrimary();
+    const db = primary.getDB(dbName);
+    const coll = db.getCollection(collName);
+    assert.commandWorked(coll.insert({a: 0}));
+    const collectionUuid = QuerySamplingUtil.getCollectionUuid(db, collName);
+
+    const updateOp0 = {q: {a: 0}, u: {$set: {b: 0}}, multi: false, upsert: false, sampleId: UUID()};
+    const updateOp1 =
+        {q: {a: {$lt: 1}}, u: {$set: {b: "$x"}}, multi: false, upsert: true, sampleId: UUID()};
+
+    const originalCmdObj = {update: collName, updates: [updateOp0], lsid, txnNumber};
+    const expectedSampledQueryDocs = [{
+        sampleId: updateOp0.sampleId,
+        cmdName: "update",
+        cmdObj: QuerySamplingUtil.makeCmdObjIgnoreSessionInfo(originalCmdObj)
+    }];
+
+    const originalRes = assert.commandWorked(db.runCommand(originalCmdObj));
+    assert.eq(originalRes.nModified, 1, originalRes);
+
+    QuerySamplingUtil.assertSoonSampledQueryDocuments(
+        primary, ns, collectionUuid, expectedSampledQueryDocs);
+
+    // Retry updateOp0 with the same sampleId but batched with the new updateOp1.
+    const retryCmdObj0 = Object.assign({}, originalCmdObj);
+    retryCmdObj0.updates = [updateOp0, updateOp1];
+    expectedSampledQueryDocs.push({
+        sampleId: updateOp1.sampleId,
+        cmdName: "update",
+        cmdObj: Object.assign(QuerySamplingUtil.makeCmdObjIgnoreSessionInfo(retryCmdObj0),
+                              {updates: [updateOp1]})
+    });
+
+    const retryRes0 = assert.commandWorked(db.runCommand(retryCmdObj0));
+    assert.eq(retryRes0.nModified, 2, retryRes0);
+
+    QuerySamplingUtil.assertSoonSampledQueryDocuments(
+        primary, ns, collectionUuid, expectedSampledQueryDocs);
+
+    // Retry both updateOp0 and updateOp1 different sampleIds.
+    const retryCmdObj1 = Object.assign({}, retryCmdObj0);
+    retryCmdObj1.updates = [
+        Object.assign({}, updateOp0, {sampleId: UUID()}),
+        Object.assign({}, updateOp1, {sampleId: UUID()})
+    ];
+
+    const retryRes1 = assert.commandWorked(db.runCommand(retryCmdObj1));
+    assert.eq(retryRes1.nModified, 2, retryRes1);
+
+    // Wait for one interval to verify that no writes occurred as a result of the retry.
+    sleep(queryAnalysisWriterIntervalSecs * 1000);
+
+    QuerySamplingUtil.assertSoonSampledQueryDocuments(
+        primary, ns, collectionUuid, expectedSampledQueryDocs);
+}
+
+function testRetryUnExecutedWrite(rst) {
+    const dbName = "testDb";
+    const collName = "testCollUnExecutedWrite";
+    const ns = dbName + "." + collName;
+
+    const lsid = {id: UUID()};
+    const txnNumber = NumberLong(1);
+
+    const primary = rst.getPrimary();
+    const db = primary.getDB(dbName);
+    const coll = db.getCollection(collName);
+    assert.commandWorked(coll.insert({a: 0}));
+    const collectionUuid = QuerySamplingUtil.getCollectionUuid(db, collName);
+
+    const updateOp0 = {q: {a: 0}, u: {$set: {b: 0}}, multi: false, upsert: false, sampleId: UUID()};
+    const originalCmdObj = {update: collName, updates: [updateOp0], lsid, txnNumber};
+    const expectedSampledQueryDocs = [{
+        sampleId: updateOp0.sampleId,
+        cmdName: "update",
+        cmdObj: QuerySamplingUtil.makeCmdObjIgnoreSessionInfo(originalCmdObj)
+    }];
+
+    const fp = configureFailPoint(primary, "failAllUpdates");
+
+    // The update fails after it has been added to the sample buffer.
+    assert.commandFailedWithCode(db.runCommand(originalCmdObj), ErrorCodes.InternalError);
+
+    QuerySamplingUtil.assertSoonSampledQueryDocuments(
+        primary, ns, collectionUuid, expectedSampledQueryDocs);
+
+    fp.off();
+
+    // Retry with the same sampleId.
+    const retryCmdObj = originalCmdObj;
+    const retryRes = assert.commandWorked(db.runCommand(retryCmdObj));
+    assert.eq(retryRes.nModified, 1, retryRes);
+
+    // Wait for one interval to verify that no writes occurred as a result of the retry.
+    sleep(queryAnalysisWriterIntervalSecs * 1000);
+
+    QuerySamplingUtil.assertSoonSampledQueryDocuments(
+        primary, ns, collectionUuid, expectedSampledQueryDocs);
+}
+
+const st = new ShardingTest({
+    shards: 1,
+    rs: {
+        nodes: 2,
+        // Make the periodic job for writing sampled queries have a period of 1 second to speed up
+        // the test.
+        setParameter: {queryAnalysisWriterIntervalSecs}
+    }
+});
+
+testRetryExecutedWrite(st.rs0);
+testRetryUnExecutedWrite(st.rs0);
+
+st.stop();
+})();
diff --git a/jstests/sharding/analyze_shard_key/persist_sampled_write_queries.js b/jstests/sharding/analyze_shard_key/persist_sampled_write_queries.js
new file mode 100644
index 00000000000..b88d02d05f0
--- /dev/null
+++ b/jstests/sharding/analyze_shard_key/persist_sampled_write_queries.js
@@ -0,0 +1,252 @@
+/**
+ * Tests that shardsvr mongods support persisting sampled write queries and that non-shardsvr
+ * mongods don't support that.
+ *
+ * @tags: [requires_fcv_62, featureFlagAnalyzeShardKey]
+ */
+(function() {
+"use strict";
+
+load("jstests/sharding/analyze_shard_key/libs/query_sampling_util.js");
+
+const supportedTestCases = [
+    {collectionExists: true, markForSampling: true, expectSampling: true},
+    {collectionExists: true, markForSampling: false, expectSampling: false},
+    {collectionExists: false, markForSampling: true, expectSampling: false},
+];
+
+const unsupportedTestCases = [
+    {collectionExists: true, markForSampling: true, expectSampling: false},
+];
+
+// Make the periodic job for writing sampled queries have a period of 1 second to speed up the test.
+const queryAnalysisWriterIntervalSecs = 1;
+
+function testWriteCmd(rst, cmdOpts, testCase) {
+    // If running on the config server, use "config" as the database name since it is illegal to
+    // create a user database on the config server.
+    const dbName = rst.isConfigRS ? "config" : "testDb";
+    const collName = "testColl-" + cmdOpts.cmdName + "-" + QuerySamplingUtil.generateRandomString();
+    const ns = dbName + "." + collName;
+
+    const primary = rst.getPrimary();
+    const primaryDB = primary.getDB(dbName);
+
+    let collectionUuid;
+    if (testCase.collectionExists) {
+        assert.commandWorked(primaryDB.createCollection(collName));
+        collectionUuid = QuerySamplingUtil.getCollectionUuid(primaryDB, collName);
+    }
+
+    const {originalCmdObj, expectedSampledQueryDocs} =
+        cmdOpts.makeCmdObjFunc(collName, testCase.markForSampling, testCase.expectSampling);
+
+    jsTest.log(
+        `Testing test case ${tojson(testCase)} with ${tojson({dbName, collName, originalCmdObj})}`);
+    assert.commandWorked(primaryDB.runCommand(originalCmdObj));
+
+    if (testCase.expectSampling) {
+        QuerySamplingUtil.assertSoonSampledQueryDocuments(
+            primary, ns, collectionUuid, expectedSampledQueryDocs);
+    } else {
+        // To verify that no writes occurred, wait for one interval before asserting.
+        sleep(queryAnalysisWriterIntervalSecs * 1000);
+        QuerySamplingUtil.assertNoSampledQueryDocuments(primary, ns);
+    }
+}
+
+function testUpdateCmd(rst, testCases) {
+    const cmdName = "update";
+    const makeCmdObjFunc = (collName, markForSampling, expectSampling) => {
+        const updateOp0 = {
+            q: {a: 0},
+            u: {$set: {"b.$[element]": 0}},
+            arrayFilters: [{"element": {$gt: 10}}],
+            multi: false,
+            upsert: false,
+            collation: QuerySamplingUtil.generateRandomCollation()
+        };
+        const updateOp1 = {
+            q: {a: {$lt: 1}},
+            u: [{$set: {b: "$x", c: "$y"}}],
+            c: {x: 1},
+            multi: true,
+            upsert: false,
+        };
+        const updateOp2 = {
+            q: {a: {$gte: 2}},
+            u: {$set: {b: 1}},
+            multi: true,
+            upsert: false,
+            collation: QuerySamplingUtil.generateRandomCollation()
+        };
+        const originalCmdObj = {
+            update: collName,
+            updates: [updateOp0, updateOp1, updateOp2],
+            let : {y: 1},
+        };
+
+        const expectedSampledQueryDocs = [];
+        if (markForSampling) {
+            updateOp0.sampleId = UUID();
+            updateOp1.sampleId = UUID();
+
+            if (expectSampling) {
+                expectedSampledQueryDocs.push({
+                    sampleId: updateOp0.sampleId,
+                    cmdName: cmdName,
+                    cmdObj: Object.assign({}, originalCmdObj, {updates: [updateOp0]})
+                });
+                expectedSampledQueryDocs.push({
+                    sampleId: updateOp1.sampleId,
+                    cmdName: cmdName,
+                    cmdObj: Object.assign({}, originalCmdObj, {updates: [updateOp1]})
+                });
+            }
+        }
+
+        return {originalCmdObj, expectedSampledQueryDocs};
+    };
+    const cmdOpts = {cmdName, makeCmdObjFunc};
+    for (let testCase of testCases) {
+        testWriteCmd(rst, cmdOpts, testCase);
+    }
+}
+
+function testDeleteCmd(rst, testCases) {
+    const cmdName = "delete";
+    const makeCmdObjFunc = (collName, markForSampling, expectSampling) => {
+        const deleteOp0 = {
+            q: {a: 0},
+            limit: 0,
+            collation: QuerySamplingUtil.generateRandomCollation()
+        };
+        const deleteOp1 = {q: {a: {$lt: 1}}, limit: 0};
+        const deleteOp2 = {
+            q: {a: {$gte: 2}},
+            limit: 1,
+            collation: QuerySamplingUtil.generateRandomCollation()
+        };
+        const originalCmdObj = {
+            delete: collName,
+            deletes: [deleteOp0, deleteOp1, deleteOp2],
+
+        };
+
+        const expectedSampledQueryDocs = [];
+        if (markForSampling) {
+            deleteOp0.sampleId = UUID();
+            deleteOp2.sampleId = UUID();
+
+            if (expectSampling) {
+                expectedSampledQueryDocs.push({
+                    sampleId: deleteOp0.sampleId,
+                    cmdName: cmdName,
+                    cmdObj: Object.assign({}, originalCmdObj, {deletes: [deleteOp0]})
+                });
+                expectedSampledQueryDocs.push({
+                    sampleId: deleteOp2.sampleId,
+                    cmdName: cmdName,
+                    cmdObj: Object.assign({}, originalCmdObj, {deletes: [deleteOp2]})
+                });
+            }
+        }
+
+        return {originalCmdObj, expectedSampledQueryDocs};
+    };
+    const cmdOpts = {cmdName, makeCmdObjFunc};
+    for (let testCase of testCases) {
+        testWriteCmd(rst, cmdOpts, testCase);
+    }
+}
+
+function testFindAndModifyCmd(rst, testCases) {
+    const cmdName = "findAndModify";
+    const makeCmdObjFunc = (collName, markForSampling, expectSampling) => {
+        const originalCmdObj = {
+            findAndModify: collName,
+            query: {a: 0},
+            update: {$set: {"b.$[element]": 0}},
+            arrayFilters: [{"element": {$gt: 10}}],
+            sort: {_id: 1},
+            collation: QuerySamplingUtil.generateRandomCollation(),
+            new: true,
+            upsert: false,
+            let : {x: 1},
+        };
+
+        const expectedSampledQueryDocs = [];
+        if (markForSampling) {
+            originalCmdObj.sampleId = UUID();
+
+            if (expectSampling) {
+                expectedSampledQueryDocs.push({
+                    sampleId: originalCmdObj.sampleId,
+                    cmdName: cmdName,
+                    cmdObj: Object.assign({}, originalCmdObj)
+                });
+            }
+        }
+
+        return {originalCmdObj, expectedSampledQueryDocs};
+    };
+    const cmdOpts = {cmdName, makeCmdObjFunc};
+    for (let testCase of testCases) {
+        testWriteCmd(rst, cmdOpts, testCase);
+    }
+}
+
+function testInsertCmd(rst) {
+    const dbName = "testDb";
+    const collName = "testColl-insert-" + QuerySamplingUtil.generateRandomString();
+    const primary = rst.getPrimary();
+    const db = primary.getDB(dbName);
+    // Verify that no mongods support persisting sampled insert queries. Specifically, "sampleId"
+    // is an unknown field for insert commands.
+    assert.commandFailedWithCode(
+        db.runCommand({insert: collName, documents: [{a: 0}], sampleId: UUID()}), 40415);
+}
+
+{
+    const st = new ShardingTest({
+        shards: 1,
+        rs: {nodes: 2, setParameter: {queryAnalysisWriterIntervalSecs}},
+        // There is no periodic job for writing sample queries on the non-shardsvr mongods but set
+        // it anyway to verify that no queries are sampled.
+        other: {configOptions: {setParameter: {queryAnalysisWriterIntervalSecs}}},
+    });
+    // It is illegal to create a user database on the config server. Set 'isConfigRS' to true to
+    // allow the test helper to know if it should use "config" as the name for the test database.
+    st.configRS.isConfigRS = true;
+
+    testUpdateCmd(st.rs0, supportedTestCases);
+    testDeleteCmd(st.rs0, supportedTestCases);
+    testFindAndModifyCmd(st.rs0, supportedTestCases);
+    testInsertCmd(st.rs0);
+
+    testUpdateCmd(st.configRS, unsupportedTestCases);
+    testDeleteCmd(st.configRS, unsupportedTestCases);
+    testFindAndModifyCmd(st.configRS, unsupportedTestCases);
+    testInsertCmd(st.configRS);
+
+    st.stop();
+}
+
+{
+    const rst = new ReplSetTest({
+        nodes: 2,
+        // There is no periodic job for writing sample queries on the non-shardsvr mongods but set
+        // it anyway to verify that no queries are sampled.
+        setParameter: {queryAnalysisWriterIntervalSecs}
+    });
+    rst.startSet();
+    rst.initiate();
+
+    testUpdateCmd(rst, unsupportedTestCases);
+    testDeleteCmd(rst, unsupportedTestCases);
+    testFindAndModifyCmd(rst, unsupportedTestCases);
+    testInsertCmd(rst);
+
+    rst.stopSet();
+}
+})();