SERVER-52545 Define listIndexes with IDL

7 files changed, 274 insertions, 43 deletions

diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript
index 09e32294938..af1e5d8a47c 100644
--- a/src/mongo/db/SConscript
+++ b/src/mongo/db/SConscript
@@ -575,10 +575,12 @@ env.Library(
         'drop.idl',
         'drop_database.idl',
         'list_collections.idl',
+        'list_indexes.idl',
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/base',
         'api_parameters',
+        'create_indexes_idl',
     ],
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/bson/mutable/mutable_bson',
@@ -1371,15 +1373,25 @@ env.Library(
 )
 
 env.Library(
+    target='create_indexes_idl',
+    source=[
+        'create_indexes.idl',
+    ],
+    LIBDEPS=[
+        '$BUILD_DIR/mongo/idl/idl_parser',
+    ],
+)
+
+env.Library(
     target='sessions_collection',
     source=[
         'sessions_collection.cpp',
-        'create_indexes.idl',
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/base',
         '$BUILD_DIR/mongo/client/clientdriver_minimal',
         '$BUILD_DIR/mongo/s/write_ops/batch_write_types',
+        'create_indexes_idl',
         'logical_session_id',
     ],
 )
diff --git a/src/mongo/db/catalog/rename_collection.cpp b/src/mongo/db/catalog/rename_collection.cpp
index 0019f1b18ad..8ee6c5c9ca6 100644
--- a/src/mongo/db/catalog/rename_collection.cpp
+++ b/src/mongo/db/catalog/rename_collection.cpp
@@ -33,6 +33,7 @@
 
 #include "mongo/db/catalog/rename_collection.h"
 
+#include "mongo/bson/unordered_fields_bsonobj_comparator.h"
 #include "mongo/db/catalog/collection_catalog.h"
 #include "mongo/db/catalog/database_holder.h"
 #include "mongo/db/catalog/document_validation.h"
@@ -750,14 +751,17 @@ void doLocalRenameIfOptionsAndIndexesHaveNotChanged(OperationContext* opCtx,
 
     auto currentIndexes =
         listIndexesEmptyListIfMissing(opCtx, targetNs, false /* includeBuildUUIDs */);
-    uassert(ErrorCodes::CommandFailed,
-            str::stream() << "indexes of target collection " << targetNs.ns()
-                          << " changed during processing.",
-            originalIndexes.size() == currentIndexes.size() &&
-                std::equal(originalIndexes.begin(),
-                           originalIndexes.end(),
-                           currentIndexes.begin(),
-                           SimpleBSONObjComparator::kInstance.makeEqualTo()));
+
+    UnorderedFieldsBSONObjComparator comparator;
+    uassert(
+        ErrorCodes::CommandFailed,
+        str::stream() << "indexes of target collection " << targetNs.ns()
+                      << " changed during processing.",
+        originalIndexes.size() == currentIndexes.size() &&
+            std::equal(originalIndexes.begin(),
+                       originalIndexes.end(),
+                       currentIndexes.begin(),
+                       [&](auto& lhs, auto& rhs) { return comparator.compare(lhs, rhs) == 0; }));
 
     validateAndRunRenameCollection(opCtx, sourceNs, targetNs, options);
 }
diff --git a/src/mongo/db/commands/list_indexes.cpp b/src/mongo/db/commands/list_indexes.cpp
index 1573dfccd36..f366615b0e6 100644
--- a/src/mongo/db/commands/list_indexes.cpp
+++ b/src/mongo/db/commands/list_indexes.cpp
@@ -27,6 +27,8 @@
  *    it in the license file.
  */
 
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kCommand
+
 #include "mongo/platform/basic.h"
 
 #include <memory>
@@ -44,6 +46,7 @@
 #include "mongo/db/exec/queued_data_stage.h"
 #include "mongo/db/exec/working_set.h"
 #include "mongo/db/index/index_descriptor.h"
+#include "mongo/db/list_indexes_gen.h"
 #include "mongo/db/query/cursor_request.h"
 #include "mongo/db/query/cursor_response.h"
 #include "mongo/db/query/find_common.h"
@@ -51,6 +54,7 @@
 #include "mongo/db/service_context.h"
 #include "mongo/db/storage/durable_catalog.h"
 #include "mongo/db/storage/storage_engine.h"
+#include "mongo/logv2/log.h"
 #include "mongo/util/uuid.h"
 
 namespace mongo {
@@ -62,6 +66,15 @@ using std::vector;
 
 namespace {
 
+void appendListIndexesCursorReply(CursorId cursorId,
+                                  const NamespaceString& cursorNss,
+                                  std::vector<mongo::ListIndexesReplyItem>&& firstBatch,
+                                  BSONObjBuilder& result) {
+    auto reply =
+        ListIndexesReply(ListIndexesReplyCursor(cursorId, cursorNss, std::move(firstBatch)));
+    reply.serialize(&result);
+}
+
 /**
  * Lists the indexes for a given collection.
  * If 'includeBuildUUIDs' is true, then the index build uuid is also returned alongside the index
@@ -146,19 +159,17 @@ public:
              const BSONObj& cmdObj,
              BSONObjBuilder& result) {
         CommandHelpers::handleMarkKillOnClientDisconnect(opCtx);
-        const long long defaultBatchSize = std::numeric_limits<long long>::max();
-        long long batchSize;
-        uassertStatusOK(
-            CursorRequest::parseCommandCursorOptions(cmdObj, defaultBatchSize, &batchSize));
-
-        auto includeBuildUUIDs = cmdObj["includeBuildUUIDs"].trueValue();
+        const auto parsed = ListIndexes::parse({"listIndexes"}, cmdObj);
+        long long batchSize = std::numeric_limits<long long>::max();
+        if (parsed.getCursor() && parsed.getCursor()->getBatchSize()) {
+            batchSize = *parsed.getCursor()->getBatchSize();
+        }
 
         NamespaceString nss;
         std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec;
-        BSONArrayBuilder firstBatch;
+        std::vector<mongo::ListIndexesReplyItem> firstBatch;
         {
-            AutoGetCollectionForReadCommand collection(
-                opCtx, CommandHelpers::parseNsOrUUID(dbname, cmdObj));
+            AutoGetCollectionForReadCommand collection(opCtx, parsed.getNamespaceOrUUID());
             uassert(ErrorCodes::NamespaceNotFound,
                     str::stream() << "ns does not exist: " << collection.getNss().ns(),
                     collection);
@@ -167,8 +178,8 @@ public:
             auto expCtx = make_intrusive<ExpressionContext>(
                 opCtx, std::unique_ptr<CollatorInterface>(nullptr), nss);
 
-            auto indexList =
-                listIndexesInLock(opCtx, collection.getCollection(), nss, includeBuildUUIDs);
+            auto indexList = listIndexesInLock(
+                opCtx, collection.getCollection(), nss, parsed.getIncludeBuildUUIDs());
             auto ws = std::make_unique<WorkingSet>();
             auto root = std::make_unique<QueuedDataStage>(expCtx.get(), ws.get());
 
@@ -190,6 +201,7 @@ public:
                                                             PlanYieldPolicy::YieldPolicy::NO_YIELD,
                                                             nss));
 
+            int bytesBuffered = 0;
             for (long long objCount = 0; objCount < batchSize; objCount++) {
                 BSONObj nextDoc;
                 PlanExecutor::ExecState state = exec->getNext(&nextDoc, nullptr);
@@ -199,16 +211,26 @@ public:
                 invariant(state == PlanExecutor::ADVANCED);
 
                 // If we can't fit this result inside the current batch, then we stash it for later.
-                if (!FindCommon::haveSpaceForNext(nextDoc, objCount, firstBatch.len())) {
+                if (!FindCommon::haveSpaceForNext(nextDoc, objCount, bytesBuffered)) {
                     exec->enqueue(nextDoc);
                     break;
                 }
 
-                firstBatch.append(nextDoc);
+                try {
+                    firstBatch.push_back(ListIndexesReplyItem::parse(
+                        IDLParserErrorContext("ListIndexesReplyItem"), nextDoc));
+                } catch (const DBException& exc) {
+                    LOGV2_ERROR(5254500,
+                                "Could not parse catalog entry while replying to listIndexes",
+                                "entry"_attr = nextDoc,
+                                "error"_attr = exc);
+                    fassertFailed(5254501);
+                }
+                bytesBuffered += nextDoc.objsize();
             }
 
             if (exec->isEOF()) {
-                appendCursorResponseObject(0LL, nss.ns(), firstBatch.arr(), &result);
+                appendListIndexesCursorReply(0 /* cursorId */, nss, std::move(firstBatch), result);
                 return true;
             }
 
@@ -228,8 +250,8 @@ public:
              cmdObj,
              {Privilege(ResourcePattern::forExactNamespace(nss), ActionType::listIndexes)}});
 
-        appendCursorResponseObject(
-            pinnedCursor.getCursor()->cursorid(), nss.ns(), firstBatch.arr(), &result);
+        appendListIndexesCursorReply(
+            pinnedCursor.getCursor()->cursorid(), nss, std::move(firstBatch), result);
 
         return true;
     }
diff --git a/src/mongo/db/create_indexes.idl b/src/mongo/db/create_indexes.idl
index 20b589660ca..d9a58ba026e 100644
--- a/src/mongo/db/create_indexes.idl
+++ b/src/mongo/db/create_indexes.idl
@@ -38,35 +38,45 @@ imports:
 
 structs:
 
+  # A built index appears with these fields, which must be the same as ListIndexReplyItem's fields
+  # in list_indexes.idl.
+  # TODO (SERVER-52539): Write a test that asserts they're the same.
   NewIndexSpec:
     description: "A type representing a spec for a new index"
     strict: true
     fields:
-      key: object
+      v:
+        type: safeInt
+        optional: true
+      key: object_owned
       name: string
+      ns:
+        # MongoDB 4.2 and older generate this field, see SERVER-41696.
+        type: string
+        ignore: true
       background:
-        type: bool
+        type: safeBool
         optional: true
       unique:
-        type: bool
+        type: safeBool
         optional: true
       hidden:
-        type: bool
+        type: safeBool
         optional: true
       partialFilterExpression:
-        type: object
+        type: object_owned
         optional: true
       sparse:
-        type: bool
+        type: safeBool
         optional: true
       expireAfterSeconds:
-        type: int
+        type: safeInt
         optional: true
       storageEngine:
-        type: object
+        type: object_owned
         optional: true
       weights:
-        type: object
+        type: object_owned
         optional: true
       default_language:
         type: string
@@ -75,26 +85,39 @@ structs:
         type: string
         optional: true
       textIndexVersion:
-        type: int
+        type: safeInt
         optional: true
       2dsphereIndexVersion:
-        type: int
+        type: safeInt
         optional: true
       bits:
-        type: int
+        type: safeInt
         optional: true
       min:
-        type: double
+        type: safeDouble
         optional: true
       max:
-        type: double
+        type: safeDouble
         optional: true
       bucketSize:
-        type: double
+        type: safeDouble
         optional: true
       collation:
-        type: object
+        type: object_owned
+        optional: true
+      wildcardProjection:
+        type: object_owned
+        optional: true
+      coarsestIndexedLevel:
+        type: safeInt
+        optional: true
+      finestIndexedLevel:
+        type: safeInt
+        optional: true
+      dropDups:
+        type: safeBool
         optional: true
+        unstable: true
 
   CreateIndexesCmd:
     description: "A struct representing a createIndexes command"
diff --git a/src/mongo/db/list_indexes.idl b/src/mongo/db/list_indexes.idl
new file mode 100644
index 00000000000..81b592c9a10
--- /dev/null
+++ b/src/mongo/db/list_indexes.idl
@@ -0,0 +1,167 @@
+# Copyright (C) 2020-present MongoDB, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the Server Side Public License, version 1,
+# as published by MongoDB, Inc.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# Server Side Public License for more details.
+#
+# You should have received a copy of the Server Side Public License
+# along with this program. If not, see
+# <http://www.mongodb.com/licensing/server-side-public-license>.
+#
+# As a special exception, the copyright holders give permission to link the
+# code of portions of this program with the OpenSSL library under certain
+# conditions as described in each individual source file and distribute
+# linked combinations including the program with the OpenSSL library. You
+# must comply with the Server Side Public License in all respects for
+# all of the code used other than as permitted herein. If you modify file(s)
+# with this exception, you may extend this exception to your version of the
+# file(s), but you are not obligated to do so. If you do not wish to do so,
+# delete this exception statement from your version. If you delete this
+# exception statement from all source files in the program, then also delete
+# it in the license file.
+#
+
+global:
+    cpp_namespace: "mongo"
+    cpp_includes:
+        - "mongo/util/uuid.h"
+
+imports:
+    - "mongo/idl/basic_types.idl"
+    - "mongo/db/create_indexes.idl"
+
+structs:
+    ListIndexesReplyItem:
+        description: "An item in the listIndexes command's reply"
+        fields:
+            #
+            # A built index appears with these fields, which must be the same as NewIndexSpec's
+            # fields. The "key" and "name" fields are required for built indexes, but marked
+            # optional to allow the in-progress index build format using the fields below.
+            #
+            v:
+                type: safeInt
+                optional: true
+            key:
+                type: object_owned
+                optional: true
+            name:
+                type: string
+                optional: true
+            ns:
+                # MongoDB 4.2 and older generate this field, see SERVER-41696.
+                type: string
+                ignore: true
+            background:
+                type: safeBool
+                optional: true
+            unique:
+                type: safeBool
+                optional: true
+            hidden:
+                type: safeBool
+                optional: true
+            partialFilterExpression:
+                type: object_owned
+                optional: true
+            sparse:
+                type: safeBool
+                optional: true
+            expireAfterSeconds:
+                type: safeInt
+                optional: true
+            storageEngine:
+                type: object_owned
+                optional: true
+            weights:
+                type: object_owned
+                optional: true
+            default_language:
+                type: string
+                optional: true
+            language_override:
+                type: string
+                optional: true
+            textIndexVersion:
+                type: safeInt
+                optional: true
+            2dsphereIndexVersion:
+                type: safeInt
+                optional: true
+            bits:
+                type: safeInt
+                optional: true
+            min:
+                type: safeDouble
+                optional: true
+            max:
+                type: safeDouble
+                optional: true
+            bucketSize:
+                type: safeDouble
+                optional: true
+            collation:
+                type: object_owned
+                optional: true
+            wildcardProjection:
+                type: object_owned
+                optional: true
+            coarsestIndexedLevel:
+                type: safeInt
+                optional: true
+            finestIndexedLevel:
+                type: safeInt
+                optional: true
+            dropDups:
+                type: safeBool
+                optional: true
+            #
+            # An index build in progress appears with these two fields. They're required, but marked
+            # optional to permit the built index format using the fields above instead.
+            #
+            buildUUID:
+                type: uuid
+                optional: true
+            spec:
+                type: NewIndexSpec
+                optional: true
+
+    ListIndexesReplyCursor:
+        description: "Cursor object"
+        fields:
+            id: long
+            ns: namespacestring
+            firstBatch: array<ListIndexesReplyItem>
+
+    ListIndexesReply:
+        description: "The listIndexes command's reply."
+        fields:
+            cursor: ListIndexesReplyCursor
+            # Included so mongos can parse shards' listIndexes replies.
+            ok:
+                type: safeDouble
+                validator: { gte: 0.0, lte: 1.0 }
+                optional: true
+
+commands:
+    listIndexes:
+        description: "Parser for the listIndexes command"
+        command_name: listIndexes
+        namespace: concatenate_with_db_or_uuid
+        cpp_name: listIndexes
+        strict: true
+        api_version: "1"
+        fields:
+            cursor:
+                type: SimpleCursorOptions
+                optional: true
+            includeBuildUUIDs:
+                type: safeBool
+                unstable: true
+                default: false
+        reply_type: ListIndexesReply
diff --git a/src/mongo/db/s/config/sharding_catalog_manager_config_initialization_test.cpp b/src/mongo/db/s/config/sharding_catalog_manager_config_initialization_test.cpp
index 19b0fc0b720..d6ff9610264 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager_config_initialization_test.cpp
+++ b/src/mongo/db/s/config/sharding_catalog_manager_config_initialization_test.cpp
@@ -64,10 +64,13 @@ void assertBSONObjsSame(const std::vector<BSONObj>& expectedBSON,
                         const std::vector<BSONObj>& foundBSON) {
     ASSERT_EQUALS(expectedBSON.size(), foundBSON.size());
 
+    auto flags =
+        BSONObj::ComparisonRules::kIgnoreFieldOrder | BSONObj::ComparisonRules::kConsiderFieldName;
+
     for (const auto& expectedObj : expectedBSON) {
         bool wasFound = false;
         for (const auto& foundObj : foundBSON) {
-            if (expectedObj.woCompare(foundObj) == 0) {
+            if (expectedObj.woCompare(foundObj, {}, flags) == 0) {
                 wasFound = true;
                 break;
             }
diff --git a/src/mongo/db/timeseries/timeseries.idl b/src/mongo/db/timeseries/timeseries.idl
index c2ecaf9227b..814bb634b65 100644
--- a/src/mongo/db/timeseries/timeseries.idl
+++ b/src/mongo/db/timeseries/timeseries.idl
@@ -50,5 +50,5 @@ structs:
             expireAfterSeconds:
                 description: "The number of seconds after which old time-series data should be
                               deleted."
-                type: long
+                type: safeInt64
                 optional: true