SERVER-57736 Add timeseries delete operation for metaField only queries

6 files changed, 265 insertions, 50 deletions

diff --git a/jstests/core/timeseries/libs/timeseries.js b/jstests/core/timeseries/libs/timeseries.js
index e8fec9adfb9..b7ca3b7e155 100644
--- a/jstests/core/timeseries/libs/timeseries.js
+++ b/jstests/core/timeseries/libs/timeseries.js
@@ -10,6 +10,16 @@ var TimeseriesTest = class {
             .featureFlagTimeseriesCollection.value;
     }
 
+    /**
+     * Returns whether time-series updates and deletes are supported.
+     */
+    static timeseriesUpdatesAndDeletesEnabled(conn) {
+        return assert
+            .commandWorked(
+                conn.adminCommand({getParameter: 1, featureFlagTimeseriesUpdatesAndDeletes: 1}))
+            .featureFlagTimeseriesUpdatesAndDeletes.value;
+    }
+
     static shardedtimeseriesCollectionsEnabled(conn) {
         return assert
             .commandWorked(conn.adminCommand({getParameter: 1, featureFlagShardedTimeSeries: 1}))
diff --git a/jstests/core/timeseries/timeseries_delete.js b/jstests/core/timeseries/timeseries_delete.js
index b90413ac4e4..87e05df35e1 100644
--- a/jstests/core/timeseries/timeseries_delete.js
+++ b/jstests/core/timeseries/timeseries_delete.js
@@ -13,55 +13,163 @@
 
 load("jstests/core/timeseries/libs/timeseries.js");
 
+if (!TimeseriesTest.timeseriesUpdatesAndDeletesEnabled(db.getMongo())) {
+    jsTestLog("Skipping test because the time-series updates and deletes feature flag is disabled");
+    return;
+}
+
+const testDB = db.getSiblingDB(jsTestName());
+assert.commandWorked(testDB.dropDatabase());
+const coll = testDB.getCollection('t');
+const timeFieldName = "time";
+const metaFieldName = "tag";
+
 TimeseriesTest.run((insert) => {
-    const testDB = db.getSiblingDB(jsTestName());
-    assert.commandWorked(testDB.dropDatabase());
-    const coll = testDB.getCollection('t');
-    const timeFieldName = "time";
-    const metaFieldName = "tag";
-    assert.commandWorked(testDB.createCollection(
-        coll.getName(), {timeseries: {timeField: timeFieldName, metaField: metaFieldName}}));
-    assert.commandWorked(insert(coll, {[timeFieldName]: ISODate(), [metaFieldName]: "A"}));
+    const testDelete = function(
+        docsToInsert,
+        expectedRemainingDocs,
+        expectedNRemoved,
+        deleteQuery,
+        {expectedErrorCode = null, ordered = true, includeMetaField = true} = {}) {
+        assert.commandWorked(testDB.createCollection(coll.getName(), {
+            timeseries: {
+                timeField: timeFieldName,
+                metaField: (includeMetaField ? metaFieldName : undefined)
+            }
+        }));
 
-    assert.commandFailedWithCode(coll.remove({tag: "A"}), ErrorCodes.IllegalOperation);
+        docsToInsert.forEach(doc => {
+            assert.commandWorked(insert(coll, doc));
+        });
+        const res = expectedErrorCode
+            ? assert.commandFailedWithCode(
+                  testDB.runCommand({delete: coll.getName(), deletes: deleteQuery, ordered}),
+                  expectedErrorCode)
+            : assert.commandWorked(
+                  testDB.runCommand({delete: coll.getName(), deletes: deleteQuery, ordered}));
+        const docs = coll.find({}, {_id: 0}).toArray();
+        assert.eq(res["n"], expectedNRemoved);
+        assert.docEq(docs, expectedRemainingDocs);
+        assert(coll.drop());
+    };
 
+    /******************** Tests deleting from a collection with a metaField **********************/
     // Query on a single field that is the metaField.
-    assert.commandFailedWithCode(coll.remove({[metaFieldName]: {b: "B"}}),
-                                 ErrorCodes.IllegalOperation);
+    testDelete([{[timeFieldName]: ISODate(), [metaFieldName]: "A"}],
+               [],
+               1,
+               [{q: {[metaFieldName]: "A"}, limit: 0}]);
+
+    const objA =
+        {[timeFieldName]: ISODate(), "measurement": {"A": "cpu"}, [metaFieldName]: {a: "A"}};
 
     // Query on a single field that is not the metaField.
-    assert.commandFailedWithCode(coll.remove({measurement: "cpu"}), ErrorCodes.InvalidOptions);
+    testDelete([objA], [objA], 0, [{q: {measurement: "cpu"}, limit: 0}], {
+        expectedErrorCode: ErrorCodes.InvalidOptions
+    });
 
     // Query on a single field that is the metaField using dot notation.
-    assert.commandFailedWithCode(coll.remove({[metaFieldName + ".a"]: "A"}),
-                                 ErrorCodes.IllegalOperation);
+    testDelete([objA], [], 1, [{q: {[metaFieldName + ".a"]: "A"}, limit: 0}]);
 
-    // Query on a single field that is not the metaField using dot notation.
-    assert.commandFailedWithCode(coll.remove({"measurement.A": "cpu"}), ErrorCodes.InvalidOptions);
+    // Compound query on a single field that is the metaField using dot notation.
+    testDelete([{[timeFieldName]: ISODate(), [metaFieldName]: {"a": "A", "b": "B"}}], [], 1, [
+        {q: {"$and": [{[metaFieldName + ".a"]: "A"}, {[metaFieldName + ".b"]: "B"}]}, limit: 0}
+    ]);
 
-    // Compound query on the metaField using dot notation.
-    assert.commandFailedWithCode(
-        coll.remove({"$and": [{[metaFieldName + ".b"]: "B"}, {[metaFieldName + ".a"]: "A"}]}),
-        ErrorCodes.IllegalOperation);
+    const objB =
+        {[timeFieldName]: ISODate(), "measurement": {"A": "cpu"}, [metaFieldName]: {b: "B"}};
+    const objC =
+        {[timeFieldName]: ISODate(), "measurement": {"A": "cpu"}, [metaFieldName]: {d: "D"}};
+
+    // Query on a single field that is not the metaField using dot notation.
+    testDelete([objA, objB, objC],
+               [objA, objB, objC],
+               0,
+               [{q: {"measurement.A": "cpu"}, limit: 0}],
+               {expectedErrorCode: ErrorCodes.InvalidOptions});
 
     // Multiple queries on a single field that is the metaField.
-    assert.commandFailedWithCode(testDB.runCommand({
-        delete: coll.getName(),
-        deletes: [
-            {q: {[metaFieldName]: {a: "A", b: "B"}}, limit: 0},
-            {q: {"$and": [{[metaFieldName]: {b: "B"}}, {[metaFieldName]: {a: "A"}}]}, limit: 0}
-        ]
-    }),
-                                 ErrorCodes.IllegalOperation);
+    testDelete([objA, objB, objC], [objB], 2, [
+        {q: {[metaFieldName]: {a: "A"}}, limit: 0},
+        {q: {"$or": [{[metaFieldName]: {d: "D"}}, {[metaFieldName]: {c: "C"}}]}, limit: 0}
+    ]);
+
+    // Multiple queries on both the metaField and a field that is not the metaField.
+    testDelete([objB],
+               [],
+               1,
+               [
+                   {q: {[metaFieldName]: {b: "B"}}, limit: 0},
+                   {q: {measurement: "cpu", [metaFieldName]: {b: "B"}}, limit: 0}
+               ],
+               {expectedErrorCode: ErrorCodes.InvalidOptions});
+
+    // Multiple queries on a field that is not the metaField.
+    testDelete([objA, objB, objC],
+               [objA, objB, objC],
+               0,
+               [{q: {measurement: "cpu"}, limit: 0}, {q: {measurement: "cpu-1"}, limit: 0}],
+               {expectedErrorCode: ErrorCodes.InvalidOptions});
 
     // Multiple queries on both the metaField and a field that is not the metaField.
-    assert.commandFailedWithCode(testDB.runCommand({
-        delete: coll.getName(),
-        deletes: [
-            {q: {[metaFieldName]: {a: "A", b: "B"}}, limit: 0},
-            {q: {measurement: "cpu", [metaFieldName]: {b: "B"}}, limit: 0}
-        ],
-    }),
-                                 ErrorCodes.InvalidOptions);
+    testDelete([objA, objB, objC],
+               [],
+               3,
+               [
+                   {q: {[metaFieldName]: {b: "B"}}, limit: 0},
+                   {q: {[metaFieldName]: {a: "A"}}, limit: 0},
+                   {q: {[metaFieldName]: {d: "D"}}, limit: 0},
+                   {q: {measurement: "cpu", [metaFieldName]: {b: "B"}}, limit: 0}
+               ],
+               {expectedErrorCode: ErrorCodes.InvalidOptions});
+
+    // Query on a single field that is the metaField using limit: 1.
+    testDelete([objA, objB, objC],
+               [objA, objB, objC],
+               0,
+               [{q: {[metaFieldName]: {a: "A"}}, limit: 1}],
+               {expectedErrorCode: ErrorCodes.IllegalOperation});
+
+    // Multiple unordered queries on both the metaField and a field that is not the metaField.
+    testDelete([objA, objB, objC],
+               [],
+               3,
+               [
+                   {q: {measurement: "cpu", [metaFieldName]: {b: "B"}}, limit: 0},
+                   {q: {[metaFieldName]: {b: "B"}}, limit: 0},
+                   {q: {[metaFieldName]: {a: "A"}}, limit: 0},
+                   {q: {[metaFieldName]: {d: "D"}}, limit: 0}
+               ],
+               {expectedErrorCode: ErrorCodes.InvalidOptions, ordered: false});
+
+    const nestedObjA =
+        {[timeFieldName]: ISODate(), "measurement": {"A": "cpu"}, [metaFieldName]: {a: {b: "B"}}};
+    const nestedObjB =
+        {[timeFieldName]: ISODate(), "measurement": {"A": "cpu"}, [metaFieldName]: {b: {a: "A"}}};
+    const nestedObjC =
+        {[timeFieldName]: ISODate(), "measurement": {"A": "cpu"}, [metaFieldName]: {d: "D"}};
+
+    // Query on a single nested field that is the metaField.
+    testDelete([nestedObjA, nestedObjB, nestedObjC],
+               [nestedObjB, nestedObjC],
+               1,
+               [{q: {[metaFieldName]: {a: {b: "B"}}}, limit: 0}]);
+
+    // Query on a single nested field that is the metaField using dot notation.
+    testDelete([nestedObjB, nestedObjC],
+               [nestedObjC],
+               1,
+               [{q: {[metaFieldName + ".b.a"]: "A"}, limit: 0}]);
+
+    // Query on a  field that is the prefix of the metaField.
+    testDelete([objA], [objA], 0, [{q: {[metaFieldName + "b"]: "A"}, limit: 0}], {
+        expectedErrorCode: ErrorCodes.InvalidOptions
+    });
+
+    /******************* Tests deleting from a collection without a metaField ********************/
+    // Remove all documents.
+    testDelete([{[timeFieldName]: ISODate(), "meta": "A"}], [], 1, [{q: {}, limit: 0}], {
+        includeMetaField: false
+    });
 });
 })();
diff --git a/jstests/core/timeseries/timeseries_update.js b/jstests/core/timeseries/timeseries_update.js
index 980302ce72f..c777ab238a0 100644
--- a/jstests/core/timeseries/timeseries_update.js
+++ b/jstests/core/timeseries/timeseries_update.js
@@ -13,6 +13,11 @@
 
 load("jstests/core/timeseries/libs/timeseries.js");
 
+if (!TimeseriesTest.timeseriesUpdatesAndDeletesEnabled(db.getMongo())) {
+    jsTestLog("Skipping test because the time-series updates and deletes feature flag is disabled");
+    return;
+}
+
 TimeseriesTest.run((insert) => {
     const testDB = db.getSiblingDB(jsTestName());
     assert.commandWorked(testDB.dropDatabase());
diff --git a/src/mongo/db/commands/write_commands.cpp b/src/mongo/db/commands/write_commands.cpp
index 39fb9252592..548e267b2f9 100644
--- a/src/mongo/db/commands/write_commands.cpp
+++ b/src/mongo/db/commands/write_commands.cpp
@@ -61,6 +61,7 @@
 #include "mongo/db/retryable_writes_stats.h"
 #include "mongo/db/stats/counters.h"
 #include "mongo/db/storage/duplicate_key_error_info.h"
+#include "mongo/db/storage/storage_parameters_gen.h"
 #include "mongo/db/timeseries/bucket_catalog.h"
 #include "mongo/db/transaction_participant.h"
 #include "mongo/db/views/view_catalog.h"
@@ -125,6 +126,28 @@ bool isMetaFieldFirstElementOfDottedPathField(StringData metaField, StringData f
     return field.substr(0, field.find('.')) == metaField;
 }
 
+// Recurses through the mutablebson element query and replaces any occurences of the
+// metaField with "meta" accounting for queries that may be in dot notation.
+void replaceTimeseriesQueryMetaFieldName(mutablebson::Element elem, const StringData& metaField) {
+    if (metaField.empty()) {
+        return;
+    }
+    if (isMetaFieldFirstElementOfDottedPathField(metaField, elem.getFieldName())) {
+        size_t dotIndex = elem.getFieldName().find('.');
+        dotIndex >= elem.getFieldName().size()
+            ? invariantStatusOK(elem.rename("meta"))
+            : invariantStatusOK(elem.rename(
+                  "meta" +
+                  elem.getFieldName().substr(dotIndex, elem.getFieldName().size() - dotIndex)));
+    }
+    if (elem.hasChildren()) {
+        for (size_t i = 0; i < elem.countChildren(); ++i) {
+            replaceTimeseriesQueryMetaFieldName(elem.findNthChild(i), metaField);
+        }
+    }
+}
+
+// TODO: SERVER-57735: remove this method.
 template <typename OpEntry>
 bool queryOnlyDependsOnTimeseriesMetaField(OperationContext* opCtx,
                                            const NamespaceString& ns,
@@ -151,9 +174,9 @@ bool queryOnlyDependsOnTimeseriesMetaField(OperationContext* opCtx,
                        [&timeseriesOptions](const auto& dependency) {
                            StringData queryField(dependency);
                            return isMetaFieldFirstElementOfDottedPathField(
-                               timeseriesOptions->getMetaField().get(), queryField);
+                               *timeseriesOptions->getMetaField(), queryField);
                        });
-}  // namespace
+}
 
 /**
  * Returns true if the given update request only modifies the time-series collection's metaField,
@@ -173,7 +196,7 @@ bool updateOnlyModifiesTimeseriesMetaField(OperationContext* opCtx,
             "Time-series buckets collection is missing time-series options",
             timeseriesOptions);
 
-    auto metaField = timeseriesOptions->getMetaField().get();
+    auto metaField = *timeseriesOptions->getMetaField();
 
     for (auto&& update : request.getUpdates()) {
         auto& updateMod = update.getU();
@@ -1363,6 +1386,10 @@ public:
 
         void _performTimeseriesUpdates(OperationContext* opCtx,
                                        write_ops::UpdateCommandReply* updateReply) const {
+            uassert(ErrorCodes::InvalidOptions,
+                    "Time-series updates are not enabled",
+                    feature_flags::gTimeseriesUpdatesAndDeletes.isEnabled(
+                        serverGlobalParams.featureCompatibility));
 
             uassert(ErrorCodes::InvalidOptions,
                     str::stream() << "Cannot perform an update on a time-series collection "
@@ -1508,16 +1535,47 @@ public:
                                    &bodyBuilder);
         }
 
-        void _performTimeseriesDeletes(OperationContext* opCtx,
-                                       write_ops::DeleteCommandReply* deleteReply) const {
+        write_ops::DeleteCommandReply* _performTimeseriesDeletes(
+            OperationContext* opCtx, write_ops::DeleteCommandReply* deleteReply) {
+
             uassert(ErrorCodes::InvalidOptions,
-                    str::stream() << "Cannot perform a delete on a time-series collection "
-                                     "when querying on a field that is not the metaField: "
-                                  << ns(),
-                    queryOnlyDependsOnTimeseriesMetaField(opCtx, ns(), request().getDeletes()));
-            uasserted(ErrorCodes::IllegalOperation,
-                      str::stream()
-                          << "Cannot perform a delete on a time-series collection: " << ns());
+                    "Time-series deletes are not enabled",
+                    feature_flags::gTimeseriesUpdatesAndDeletes.isEnabled(
+                        serverGlobalParams.featureCompatibility));
+
+            auto collection = CollectionCatalog::get(opCtx)->lookupCollectionByNamespaceForRead(
+                opCtx, ns().makeTimeseriesBucketsNamespace());
+            uassert(ErrorCodes::NamespaceNotFound,
+                    "Could not find time-series buckets collection for write",
+                    collection);
+            auto timeseriesOptions = collection->getTimeseriesOptions();
+            uassert(ErrorCodes::InvalidOptions,
+                    "Time-series buckets collection is missing time-series options",
+                    timeseriesOptions);
+            StringData metaField =
+                timeseriesOptions->getMetaField() ? *timeseriesOptions->getMetaField() : "";
+
+            std::vector<mongo::write_ops::DeleteOpEntry> timeseriesDeletes;
+            timeseriesDeletes.reserve(request().getDeletes().size());
+            for (auto& deleteEntry : request().getDeletes()) {
+                mutablebson::Document doc(deleteEntry.getQ());
+                replaceTimeseriesQueryMetaFieldName(doc.root(), metaField);
+                timeseriesDeletes.emplace_back(doc.getObject(), deleteEntry.getMulti());
+            }
+
+            write_ops::DeleteCommandRequest timeseriesDeleteReq(
+                ns().makeTimeseriesBucketsNamespace(), timeseriesDeletes);
+            timeseriesDeleteReq.setWriteCommandRequestBase(request().getWriteCommandRequestBase());
+
+            auto reply = write_ops_exec::performDeletes(
+                opCtx, timeseriesDeleteReq, OperationSource::kTimeseries);
+            populateReply(opCtx,
+                          !timeseriesDeleteReq.getWriteCommandRequestBase().getOrdered(),
+                          timeseriesDeleteReq.getDeletes().size(),
+                          std::move(reply),
+                          deleteReply);
+
+            return deleteReply;
         }
 
         const BSONObj& _commandObj;
diff --git a/src/mongo/db/ops/write_ops_exec.cpp b/src/mongo/db/ops/write_ops_exec.cpp
index 6e336d862b9..84e94ada3fb 100644
--- a/src/mongo/db/ops/write_ops_exec.cpp
+++ b/src/mongo/db/ops/write_ops_exec.cpp
@@ -570,6 +570,23 @@ SingleWriteResult makeWriteResultForInsertOrDeleteRetry() {
     return res;
 }
 
+// TODO: SERVER-58382 Handle time-series collections without a metaField.
+template <typename OpEntry>
+bool isTimeseriesMetaFieldOnlyQuery(OperationContext* opCtx,
+                                    const NamespaceString& ns,
+                                    OpEntry& opEntry) {
+
+    boost::intrusive_ptr<ExpressionContext> expCtx(new ExpressionContext(opCtx, nullptr, ns));
+
+    std::vector<BSONObj> rawPipeline{BSON("$match" << opEntry.getQ())};
+    DepsTracker dependencies = Pipeline::parse(rawPipeline, expCtx)->getDependencies({});
+    return std::all_of(
+        dependencies.fields.begin(), dependencies.fields.end(), [](const auto& dependency) {
+            StringData queryField(dependency);
+            return queryField.substr(0, queryField.find('.')) == "meta";
+        });
+}
+
 }  // namespace
 
 WriteResult performInserts(OperationContext* opCtx,
@@ -1072,7 +1089,8 @@ static SingleWriteResult performSingleDeleteOp(OperationContext* opCtx,
 }
 
 WriteResult performDeletes(OperationContext* opCtx,
-                           const write_ops::DeleteCommandRequest& wholeOp) {
+                           const write_ops::DeleteCommandRequest& wholeOp,
+                           const OperationSource& source) {
     // Delete performs its own retries, so we should not be in a WriteUnitOfWork unless we are in a
     // transaction.
     auto txnParticipant = TransactionParticipant::get(opCtx);
@@ -1130,6 +1148,20 @@ WriteResult performDeletes(OperationContext* opCtx,
         });
         try {
             lastOpFixer.startingOp();
+
+            if (source == OperationSource::kTimeseries) {
+                uassert(ErrorCodes::InvalidOptions,
+                        str::stream() << "Cannot perform a delete on a time-series collection "
+                                         "when querying on a field that is not the metaField: "
+                                      << wholeOp.getNamespace(),
+                        isTimeseriesMetaFieldOnlyQuery(opCtx, wholeOp.getNamespace(), singleOp));
+                uassert(ErrorCodes::IllegalOperation,
+                        str::stream() << "Cannot perform a delete with limit: 1 on a "
+                                         "time-series collection: "
+                                      << wholeOp.getNamespace(),
+                        singleOp.getMulti());
+            }
+
             out.results.emplace_back(performSingleDeleteOp(opCtx,
                                                            wholeOp.getNamespace(),
                                                            stmtId,
diff --git a/src/mongo/db/ops/write_ops_exec.h b/src/mongo/db/ops/write_ops_exec.h
index 9c9990306d8..acbeae17ced 100644
--- a/src/mongo/db/ops/write_ops_exec.h
+++ b/src/mongo/db/ops/write_ops_exec.h
@@ -80,7 +80,9 @@ WriteResult performInserts(OperationContext* opCtx,
 WriteResult performUpdates(OperationContext* opCtx,
                            const write_ops::UpdateCommandRequest& op,
                            const OperationSource& source = OperationSource::kStandard);
-WriteResult performDeletes(OperationContext* opCtx, const write_ops::DeleteCommandRequest& op);
+WriteResult performDeletes(OperationContext* opCtx,
+                           const write_ops::DeleteCommandRequest& op,
+                           const OperationSource& source = OperationSource::kStandard);
 
 Status performAtomicTimeseriesWrites(OperationContext* opCtx,
                                      const std::vector<write_ops::InsertCommandRequest>& insertOps,