SERVER-43669 Serialize "sortKey" as BSONArray

5 files changed, 80 insertions, 24 deletions

diff --git a/src/mongo/db/exec/document_value/document.cpp b/src/mongo/db/exec/document_value/document.cpp
index 17c18ba3c79..7400e88058d 100644
--- a/src/mongo/db/exec/document_value/document.cpp
+++ b/src/mongo/db/exec/document_value/document.cpp
@@ -470,7 +470,7 @@ constexpr StringData Document::metaFieldGeoNearPoint;
 constexpr StringData Document::metaFieldSearchScore;
 constexpr StringData Document::metaFieldSearchHighlights;
 
-BSONObj Document::toBsonWithMetaData(bool use42ChangeStreamSortKeys) const {
+BSONObj Document::toBsonWithMetaData(SortKeyFormat sortKeyFormat) const {
     BSONObjBuilder bb;
     toBson(&bb);
     if (!metadata()) {
@@ -482,17 +482,21 @@ BSONObj Document::toBsonWithMetaData(bool use42ChangeStreamSortKeys) const {
     if (metadata().hasRandVal())
         bb.append(metaFieldRandVal, metadata().getRandVal());
     if (metadata().hasSortKey()) {
-        if (use42ChangeStreamSortKeys) {
-            // TODO (SERVER-43361): In 4.2 and earlier, the "sort key" for a change stream document
-            // gets serialized differently than sort keys for normal pipeline documents. Once we no
-            // longer need to support that format, we can remove the 'use42ChangeStreamSortKeys'
-            // flag and this special case along with it.
-            invariant(metadata().isSingleElementKey());
-            metadata().getSortKey().addToBsonObj(&bb, metaFieldSortKey);
-        } else {
-            bb.append(metaFieldSortKey,
-                      DocumentMetadataFields::serializeSortKey(metadata().isSingleElementKey(),
-                                                               metadata().getSortKey()));
+        switch (sortKeyFormat) {
+            case SortKeyFormat::k42ChangeStreamSortKey:
+                invariant(metadata().isSingleElementKey());
+                metadata().getSortKey().addToBsonObj(&bb, metaFieldSortKey);
+                break;
+            case SortKeyFormat::k42SortKey:
+                bb.append(metaFieldSortKey,
+                          DocumentMetadataFields::serializeSortKeyAsObject(
+                              metadata().isSingleElementKey(), metadata().getSortKey()));
+                break;
+            case SortKeyFormat::k44SortKey:
+                bb.append(metaFieldSortKey,
+                          DocumentMetadataFields::serializeSortKeyAsArray(
+                              metadata().isSingleElementKey(), metadata().getSortKey()));
+                break;
         }
     }
     if (metadata().hasGeoNearDistance())
diff --git a/src/mongo/db/exec/document_value/document.h b/src/mongo/db/exec/document_value/document.h
index 9e03aa09475..5b9d953b287 100644
--- a/src/mongo/db/exec/document_value/document.h
+++ b/src/mongo/db/exec/document_value/document.h
@@ -59,6 +59,26 @@ class MutableDocument;
  */
 class Position;
 
+/**
+ * "Sort keys" are stored in memory as a Value with Array type (with an exception for singleton sort
+ * keys). When serializing a sort key for storage in document metadata or as part of a
+ * {$meta: "sortKey"} projection, there are three possible storage formats:
+ */
+enum class SortKeyFormat {
+    // We expect the in-memory sort key to have one object, which has the format:
+    // {_data: ..., _typeBits:...}. This same object becomes the serialized sort key. This format
+    // exists for compatibility with 4.2 and will be deleted in 4.6 (SERVER-43361).
+    k42ChangeStreamSortKey,
+
+    // A sort key with values "a" and "b" would get translated to an object that looks like:
+    // {"": "a", "": "b"}. Also scheduled for deletion in 4.6.
+    k42SortKey,
+
+    // A sort key with values "a" and "b" would get translated to an array that looks like:
+    // ["a", "b"].
+    k44SortKey,
+};
+
 /** A Document is similar to a BSONObj but with a different in-memory representation.
  *
  *  A Document can be treated as a const std::map<std::string, const Value> that is
@@ -239,14 +259,16 @@ public:
     boost::optional<BSONObj> toBsonIfTriviallyConvertible() const;
 
     /**
-     * Like the 'toBson()' method, but includes metadata at the top-level. When
-     * 'use42ChangeStreamSortKeys' is true, we assume that any Value in the "sortKey" metadata
-     * represents the resume token, which gets assigned directly to the "$sortKey" field. Otherwise,
-     * the "$sortKey" field gets assigned using DocumentMetadataFields::serializeSortKey(). Output
-     * is parseable by the 'fromBsonWithMetaData()' method. Note that parsing is able to infer the
-     * value of 'use42ChangeStreamSortKeys' from the format of the '$sortKey' field.
-     */
-    BSONObj toBsonWithMetaData(bool use42ChangeStreamSortKeys = false) const;
+     * Like the 'toBson()' method, but includes metadata at the top-level. When the metadata
+     * includes a sort key, the 'sortKeyFormat' parameter controls how it gets converted from its
+     * in-memory representation as a Value to its serialized representation as either a BSONObj or
+     * BSONArray. The possible formats are described in the comment above the 'SortKeyFormat' enum.
+     *
+     * Note that the 'fromBsonWithMetaData()' function does not need a corresponding 'sortKeyFormat'
+     * parameter, because sort key deserialization is able to infer the sort key format based on the
+     * layout of the object.
+     */
+    BSONObj toBsonWithMetaData(SortKeyFormat sortKeyFormat) const;
 
     /**
      * Like Document(BSONObj) but treats top-level fields with special names as metadata.
diff --git a/src/mongo/db/exec/document_value/document_metadata_fields.cpp b/src/mongo/db/exec/document_value/document_metadata_fields.cpp
index c7c2e4d202d..61438cd2d2d 100644
--- a/src/mongo/db/exec/document_value/document_metadata_fields.cpp
+++ b/src/mongo/db/exec/document_value/document_metadata_fields.cpp
@@ -207,7 +207,8 @@ void DocumentMetadataFields::deserializeForSorter(BufReader& buf, DocumentMetada
     }
 }
 
-BSONObj DocumentMetadataFields::serializeSortKey(bool isSingleElementKey, const Value& value) {
+BSONObj DocumentMetadataFields::serializeSortKeyAsObject(bool isSingleElementKey,
+                                                         const Value& value) {
     // Missing values don't serialize correctly in this format, so use nulls instead, since they are
     // considered equivalent with woCompare().
     if (isSingleElementKey) {
@@ -221,6 +222,21 @@ BSONObj DocumentMetadataFields::serializeSortKey(bool isSingleElementKey, const
     return bb.obj();
 }
 
+BSONArray DocumentMetadataFields::serializeSortKeyAsArray(bool isSingleElementKey,
+                                                          const Value& value) {
+    // Missing values don't serialize correctly in this format, so use nulls instead, since they are
+    // considered equivalent with woCompare().
+    if (isSingleElementKey) {
+        return BSON_ARRAY(missingToNull(value));
+    }
+    invariant(value.isArray());
+    BSONArrayBuilder bb;
+    for (auto&& val : value.getArray()) {
+        bb << missingToNull(val);
+    }
+    return bb.arr();
+}
+
 Value DocumentMetadataFields::deserializeSortKey(bool isSingleElementKey,
                                                  const BSONObj& bsonSortKey) {
     std::vector<Value> keys;
diff --git a/src/mongo/db/exec/document_value/document_metadata_fields.h b/src/mongo/db/exec/document_value/document_metadata_fields.h
index ef4627b14c6..83482a32abc 100644
--- a/src/mongo/db/exec/document_value/document_metadata_fields.h
+++ b/src/mongo/db/exec/document_value/document_metadata_fields.h
@@ -81,8 +81,22 @@ public:
      * sort key. If 'isSingleElementKey' is true, returns a BSON object with 'value' as its only
      * value - and an empty field name. Otherwise returns a BSONObj with one field for each value in
      * the array, each field using the empty string as the key name.
+     *
+     * Note that this format for serializing a sort key is deprecated and will be removed as part of
+     * SERVER-43361. The serialized sort key is a BSONObj with a field for each key component. Each
+     * field name is the empty string, meaning that the fields have the same name
+     * (e.g., {"": 1, "": 2}). The new preferred way to serialize a sort key is as a BSONArray
+     * (e.g.: [1, 2]), which can be done with the 'serializeSortKeyAsArray()' function.
+     */
+    static BSONObj serializeSortKeyAsObject(bool isSingleElementKey, const Value& value);
+
+    /**
+     * Converts a Value representing an in-memory sort key to a BSONArray representing a serialized
+     * sort key. If 'isSingleElementKey' is true, returns a BSONArray with 'value' as its only
+     * element. Otherwise, converts 'value' (which is expected to be an Array) to a BSONArray. Any
+     * Value elements whose value is "missing" get converted to BSONNull.
      */
-    static BSONObj serializeSortKey(bool isSingleElementKey, const Value& value);
+    static BSONArray serializeSortKeyAsArray(bool isSingleElementKey, const Value& value);
 
     /**
      * Converts a BSONObj representing a serialized sort key into a Value, which we use for
diff --git a/src/mongo/db/exec/document_value/document_value_test.cpp b/src/mongo/db/exec/document_value/document_value_test.cpp
index a3092b636b1..d641b699ae5 100644
--- a/src/mongo/db/exec/document_value/document_value_test.cpp
+++ b/src/mongo/db/exec/document_value/document_value_test.cpp
@@ -595,7 +595,7 @@ TEST(MetaFields, IndexKeyMetadataSerializesCorrectly) {
     ASSERT_TRUE(doc.metadata().hasIndexKey());
     ASSERT_BSONOBJ_EQ(doc.metadata().getIndexKey(), BSON("b" << 1));
 
-    auto serialized = doc.toBsonWithMetaData();
+    auto serialized = doc.toBsonWithMetaData(SortKeyFormat::k42SortKey);
     ASSERT_BSONOBJ_EQ(serialized, BSON("a" << 1 << "$indexKey" << BSON("b" << 1)));
 }
 
@@ -709,7 +709,7 @@ TEST(MetaFields, ToAndFromBson) {
     docBuilder.metadata().setSearchHighlights(DOC_ARRAY("abc"_sd
                                                         << "def"_sd));
     Document doc = docBuilder.freeze();
-    BSONObj obj = doc.toBsonWithMetaData();
+    BSONObj obj = doc.toBsonWithMetaData(SortKeyFormat::k42SortKey);
     ASSERT_EQ(10.0, obj[Document::metaFieldTextScore].Double());
     ASSERT_EQ(20, obj[Document::metaFieldRandVal].numberLong());
     ASSERT_EQ(30.0, obj[Document::metaFieldSearchScore].Double());