SERVER-58779 Store the original user index definition on the transformed index definition on the buckets collection

10 files changed, 105 insertions, 48 deletions

diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp
index 95d1ab98095..ba8addcc4bb 100644
--- a/src/mongo/db/catalog/collection_impl.cpp
+++ b/src/mongo/db/catalog/collection_impl.cpp
@@ -1631,7 +1631,7 @@ StatusWith<std::vector<BSONObj>> CollectionImpl::addCollationDefaultsToIndexSpec
                 << "failed to add collation information to index spec for index creation: "
                 << originalIndexSpec);
         }
-        const auto& newIndexSpec = validateResult.getValue();
+        BSONObj newIndexSpec = validateResult.getValue();
 
         auto keyPattern = newIndexSpec[IndexDescriptor::kKeyPatternFieldName].Obj();
         if (IndexDescriptor::isIdIndexPattern(keyPattern)) {
@@ -1656,6 +1656,18 @@ StatusWith<std::vector<BSONObj>> CollectionImpl::addCollationDefaultsToIndexSpec
             }
         }
 
+        if (originalIndexSpec.hasField(IndexDescriptor::kOriginalSpecFieldName)) {
+            // Validation was already performed above.
+            BSONObj newOriginalIndexSpec = invariant(index_key_validate::validateIndexSpecCollation(
+                opCtx,
+                originalIndexSpec.getObjectField(IndexDescriptor::kOriginalSpecFieldName),
+                collator));
+
+            BSONObj specToAdd =
+                BSON(IndexDescriptor::kOriginalSpecFieldName << newOriginalIndexSpec);
+            newIndexSpec = newIndexSpec.addField(specToAdd.firstElement());
+        }
+
         newIndexSpecs.push_back(newIndexSpec);
     }
 
diff --git a/src/mongo/db/catalog/index_key_validate.cpp b/src/mongo/db/catalog/index_key_validate.cpp
index 5ea274d5360..b4169492b51 100644
--- a/src/mongo/db/catalog/index_key_validate.cpp
+++ b/src/mongo/db/catalog/index_key_validate.cpp
@@ -93,6 +93,7 @@ static std::set<StringData> allowedFieldNames = {
     IndexDescriptor::kTextVersionFieldName,
     IndexDescriptor::kUniqueFieldName,
     IndexDescriptor::kWeightsFieldName,
+    IndexDescriptor::kOriginalSpecFieldName,
     // Index creation under legacy writeMode can result in an index spec with an _id field.
     "_id"};
 
@@ -277,6 +278,7 @@ StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& in
     bool hasVersionField = false;
     bool hasCollationField = false;
     bool hasWeightsField = false;
+    bool hasOriginalSpecField = false;
     bool apiStrict = opCtx && APIParameters::get(opCtx).getAPIStrict().value_or(false);
 
     auto fieldNamesValidStatus = validateIndexSpecFieldNames(indexSpec);
@@ -378,6 +380,21 @@ StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& in
 
             hasVersionField = true;
             resolvedIndexVersion = requestedIndexVersion;
+        } else if (IndexDescriptor::kOriginalSpecFieldName == indexSpecElemFieldName) {
+            if (indexSpecElem.type() != BSONType::Object) {
+                return {ErrorCodes::TypeMismatch,
+                        str::stream()
+                            << "The field '" << IndexDescriptor::kOriginalSpecFieldName
+                            << "' must be an object, but got " << typeName(indexSpecElem.type())};
+            }
+
+            if (indexSpecElem.Obj().isEmpty()) {
+                return {ErrorCodes::BadValue,
+                        str::stream() << "The field '" << IndexDescriptor::kOriginalSpecFieldName
+                                      << "' cannot be an empty object."};
+            }
+
+            hasOriginalSpecField = true;
         } else if (IndexDescriptor::kCollationFieldName == indexSpecElemFieldName) {
             if (indexSpecElem.type() != BSONType::Object) {
                 return {ErrorCodes::TypeMismatch,
@@ -549,6 +566,18 @@ StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& in
         modifiedSpec = modifiedSpec.addField(versionObj.firstElement());
     }
 
+    if (hasOriginalSpecField) {
+        StatusWith<BSONObj> modifiedOriginalSpec = validateIndexSpec(
+            opCtx, indexSpec.getObjectField(IndexDescriptor::kOriginalSpecFieldName));
+        if (!modifiedOriginalSpec.isOK()) {
+            return modifiedOriginalSpec.getStatus();
+        }
+
+        BSONObj specToAdd =
+            BSON(IndexDescriptor::kOriginalSpecFieldName << modifiedOriginalSpec.getValue());
+        modifiedSpec = modifiedSpec.addField(specToAdd.firstElement());
+    }
+
     return modifiedSpec;
 }
 
diff --git a/src/mongo/db/commands/create_indexes.cpp b/src/mongo/db/commands/create_indexes.cpp
index 70ba477b2db..72c904254f6 100644
--- a/src/mongo/db/commands/create_indexes.cpp
+++ b/src/mongo/db/commands/create_indexes.cpp
@@ -59,6 +59,7 @@
 #include "mongo/db/s/database_sharding_state.h"
 #include "mongo/db/s/operation_sharding_state.h"
 #include "mongo/db/s/sharding_state.h"
+#include "mongo/db/storage/storage_parameters_gen.h"
 #include "mongo/db/storage/two_phase_index_build_knobs_gen.h"
 #include "mongo/db/timeseries/timeseries_index_schema_conversion_functions.h"
 #include "mongo/db/timeseries/timeseries_options.h"
@@ -665,6 +666,13 @@ std::unique_ptr<CreateIndexesCommand> makeTimeseriesCreateIndexesCommand(
                           "TTL indexes are not supported on time-series collections");
             }
         }
+
+        if (feature_flags::gTimeseriesMetricIndexes.isEnabledAndIgnoreFCV()) {
+            // Store the original user index definition on the transformed index definition for the
+            // time-series buckets collection.
+            builder.appendObject(IndexDescriptor::kOriginalSpecFieldName, origIndex.objdata());
+        }
+
         indexes.push_back(builder.obj());
     }
 
diff --git a/src/mongo/db/index/index_descriptor.h b/src/mongo/db/index/index_descriptor.h
index 6c7ca2029c1..76e5dce4c91 100644
--- a/src/mongo/db/index/index_descriptor.h
+++ b/src/mongo/db/index/index_descriptor.h
@@ -87,6 +87,7 @@ public:
     static constexpr StringData kTextVersionFieldName = "textIndexVersion"_sd;
     static constexpr StringData kUniqueFieldName = "unique"_sd;
     static constexpr StringData kWeightsFieldName = "weights"_sd;
+    static constexpr StringData kOriginalSpecFieldName = "originalSpec"_sd;
 
     /**
      * infoObj is a copy of the index-describing BSONObj contained in the catalog.
diff --git a/src/mongo/db/list_indexes.idl b/src/mongo/db/list_indexes.idl
index 684321a91e2..48900398be6 100644
--- a/src/mongo/db/list_indexes.idl
+++ b/src/mongo/db/list_indexes.idl
@@ -123,6 +123,9 @@ structs:
             dropDups:
                 type: safeBool
                 optional: true
+            originalSpec:
+                type: object_owned
+                optional: true
             #
             # An index build in progress appears with these two fields. They're required, but marked
             # optional to permit the built index format using the fields above instead.
diff --git a/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.cpp b/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.cpp
index 5e08827daef..be9784f5346 100644
--- a/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.cpp
+++ b/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.cpp
@@ -79,20 +79,19 @@ BSONObj makeTimeseriesIndexStats(const TimeseriesConversionOptions& bucketSpec,
     BSONObjBuilder builder;
     for (const auto& elem : bucketsIndexStatsBSON) {
         if (elem.fieldNameStringData() == ListIndexesReplyItem::kKeyFieldName) {
-            auto timeseriesKey = timeseries::createTimeseriesIndexSpecFromBucketsIndexSpec(
-                timeseriesOptions, elem.Obj());
-            if (!timeseriesKey) {
-                return {};
-            }
-            builder.append(ListIndexesReplyItem::kKeyFieldName, *timeseriesKey);
+            // This field is appended below.
             continue;
         }
         if (elem.fieldNameStringData() == ListIndexesReplyItem::kSpecFieldName) {
-            auto timeseriesSpec = makeTimeseriesIndexStats(bucketSpec, elem.Obj());
-            if (timeseriesSpec.isEmpty()) {
+            auto timeseriesSpec =
+                timeseries::createTimeseriesIndexFromBucketsIndex(timeseriesOptions, elem.Obj());
+            if (!timeseriesSpec) {
                 return {};
             }
-            builder.append("spec", timeseriesSpec);
+
+            builder.append(ListIndexesReplyItem::kSpecFieldName, *timeseriesSpec);
+            builder.append(ListIndexesReplyItem::kKeyFieldName,
+                           timeseriesSpec->getObjectField(IndexDescriptor::kKeyPatternFieldName));
             continue;
         }
         builder.append(elem);
diff --git a/src/mongo/db/timeseries/timeseries_constants.h b/src/mongo/db/timeseries/timeseries_constants.h
index d8564fe7c6e..0380dbd1e06 100644
--- a/src/mongo/db/timeseries/timeseries_constants.h
+++ b/src/mongo/db/timeseries/timeseries_constants.h
@@ -52,6 +52,7 @@ static constexpr StringData kMetaFieldName = "metaField"_sd;
 
 // These are hard-coded field names in index specs.
 static constexpr StringData kKeyFieldName = "key"_sd;
+static constexpr StringData kOriginalSpecFieldName = "originalSpec"_sd;
 
 static const StringDataSet kAllowedCollectionCreationOptions{
     CreateCommand::kStorageEngineFieldName,
diff --git a/src/mongo/db/timeseries/timeseries_index_schema_conversion_functions.cpp b/src/mongo/db/timeseries/timeseries_index_schema_conversion_functions.cpp
index a39b2fa7dfb..babf19a6480 100644
--- a/src/mongo/db/timeseries/timeseries_index_schema_conversion_functions.cpp
+++ b/src/mongo/db/timeseries/timeseries_index_schema_conversion_functions.cpp
@@ -182,19 +182,29 @@ StatusWith<BSONObj> createBucketsSpecFromTimeseriesSpec(const TimeseriesOptions&
     }
 
     return builder.obj();
-}  // namespace
-}  // namespace
-
-StatusWith<BSONObj> createBucketsIndexSpecFromTimeseriesIndexSpec(
-    const TimeseriesOptions& timeseriesOptions, const BSONObj& timeseriesIndexSpecBSON) {
-    return createBucketsSpecFromTimeseriesSpec(timeseriesOptions, timeseriesIndexSpecBSON, false);
-}
-
-StatusWith<BSONObj> createBucketsShardKeySpecFromTimeseriesShardKeySpec(
-    const TimeseriesOptions& timeseriesOptions, const BSONObj& timeseriesShardKeySpecBSON) {
-    return createBucketsSpecFromTimeseriesSpec(timeseriesOptions, timeseriesShardKeySpecBSON, true);
 }
 
+/**
+ * Maps the buckets collection index spec 'bucketsIndexSpecBSON' to the index schema of the
+ * time-series collection using the information provided in 'timeseriesOptions'.
+ *
+ * If 'bucketsIndexSpecBSON' does not match a valid time-series index format, then boost::none is
+ * returned.
+ *
+ * Conversion Example:
+ * On a time-series collection with 'tm' time field and 'mm' metadata field,
+ * we may see a compound index on the underlying bucket collection mapped from:
+ * {
+ *     'meta.tag1': 1,
+ *     'control.min.tm': 1,
+ *     'control.max.tm': 1
+ * }
+ * to an index on the time-series collection:
+ * {
+ *     'mm.tag1': 1,
+ *     'tm': 1
+ * }
+ */
 boost::optional<BSONObj> createTimeseriesIndexSpecFromBucketsIndexSpec(
     const TimeseriesOptions& timeseriesOptions, const BSONObj& bucketsIndexSpecBSON) {
     auto timeField = timeseriesOptions.getTimeField();
@@ -320,8 +330,24 @@ boost::optional<BSONObj> createTimeseriesIndexSpecFromBucketsIndexSpec(
     return builder.obj();
 }
 
+}  // namespace
+
+StatusWith<BSONObj> createBucketsIndexSpecFromTimeseriesIndexSpec(
+    const TimeseriesOptions& timeseriesOptions, const BSONObj& timeseriesIndexSpecBSON) {
+    return createBucketsSpecFromTimeseriesSpec(timeseriesOptions, timeseriesIndexSpecBSON, false);
+}
+
+StatusWith<BSONObj> createBucketsShardKeySpecFromTimeseriesShardKeySpec(
+    const TimeseriesOptions& timeseriesOptions, const BSONObj& timeseriesShardKeySpecBSON) {
+    return createBucketsSpecFromTimeseriesSpec(timeseriesOptions, timeseriesShardKeySpecBSON, true);
+}
+
 boost::optional<BSONObj> createTimeseriesIndexFromBucketsIndex(
     const TimeseriesOptions& timeseriesOptions, const BSONObj& bucketsIndex) {
+    if (bucketsIndex.hasField(kOriginalSpecFieldName)) {
+        // This buckets index has the original user index definition available, return it.
+        return bucketsIndex.getObjectField(kOriginalSpecFieldName);
+    }
     if (bucketsIndex.hasField(kKeyFieldName)) {
         auto timeseriesKeyValue = createTimeseriesIndexSpecFromBucketsIndexSpec(
             timeseriesOptions, bucketsIndex.getField(kKeyFieldName).Obj());
diff --git a/src/mongo/db/timeseries/timeseries_index_schema_conversion_functions.h b/src/mongo/db/timeseries/timeseries_index_schema_conversion_functions.h
index 3fa88e09a58..318aae82f80 100644
--- a/src/mongo/db/timeseries/timeseries_index_schema_conversion_functions.h
+++ b/src/mongo/db/timeseries/timeseries_index_schema_conversion_functions.h
@@ -56,30 +56,6 @@ StatusWith<BSONObj> createBucketsShardKeySpecFromTimeseriesShardKeySpec(
     const TimeseriesOptions& timeseriesOptions, const BSONObj& timeseriesIndexSpecBSON);
 
 /**
- * Maps the buckets collection index spec 'bucketsIndexSpecBSON' to the index schema of the
- * time-series collection using the information provided in 'timeseriesOptions'.
- *
- * If 'bucketsIndexSpecBSON' does not match a valid time-series index format, then boost::none is
- * returned.
- *
- * Conversion Example:
- * On a time-series collection with 'tm' time field and 'mm' metadata field,
- * we may see a compound index on the underlying bucket collection mapped from:
- * {
- *     'meta.tag1': 1,
- *     'control.min.tm': 1,
- *     'control.max.tm': 1
- * }
- * to an index on the time-series collection:
- * {
- *     'mm.tag1': 1,
- *     'tm': 1
- * }
- */
-boost::optional<BSONObj> createTimeseriesIndexSpecFromBucketsIndexSpec(
-    const TimeseriesOptions& timeseriesOptions, const BSONObj& bucketsIndexSpecBSON);
-
-/**
  * Returns a time-series collection index spec equivalent to the given 'bucketsIndex' using the
  * time-series specifications provided in 'timeseriesOptions'. Returns boost::none if the
  * buckets index is not supported on a time-series collection.
diff --git a/src/mongo/db/timeseries/timeseries_index_schema_conversion_functions_test.cpp b/src/mongo/db/timeseries/timeseries_index_schema_conversion_functions_test.cpp
index 2763b5a3385..2efbf4c7cda 100644
--- a/src/mongo/db/timeseries/timeseries_index_schema_conversion_functions_test.cpp
+++ b/src/mongo/db/timeseries/timeseries_index_schema_conversion_functions_test.cpp
@@ -34,6 +34,7 @@
 #include "mongo/bson/bsonobj.h"
 #include "mongo/db/timeseries/timeseries_constants.h"
 #include "mongo/db/timeseries/timeseries_gen.h"
+#include "mongo/idl/server_parameter_test_util.h"
 #include "mongo/unittest/unittest.h"
 
 namespace mongo {
@@ -83,12 +84,13 @@ void testBothWaysIndexSpecConversion(const TimeseriesOptions& timeseriesOptions,
 
     // Test buckets => time-series schema conversion.
 
-    auto timeseriesIndexSpecResult = timeseries::createTimeseriesIndexSpecFromBucketsIndexSpec(
-        timeseriesOptions, bucketsIndexSpec);
+    auto timeseriesIndexSpecResult = timeseries::createTimeseriesIndexFromBucketsIndex(
+        timeseriesOptions, BSON(timeseries::kKeyFieldName << bucketsIndexSpec));
 
     if (testShouldSucceed) {
         ASSERT(timeseriesIndexSpecResult);
-        ASSERT_BSONOBJ_EQ(timeseriesIndexSpec, timeseriesIndexSpecResult.get());
+        ASSERT_BSONOBJ_EQ(timeseriesIndexSpec,
+                          timeseriesIndexSpecResult->getObjectField(timeseries::kKeyFieldName));
     } else {
         // A buckets collection index spec that does not conform to the supported time-series index
         // spec schema should be converted to an empty time-series index spec result.