SERVER-44915 Extend output to include full index options and shard name

10 files changed, 185 insertions, 35 deletions

diff --git a/jstests/aggregation/sources/indexStats/verify_index_stats_output.js b/jstests/aggregation/sources/indexStats/verify_index_stats_output.js
new file mode 100644
index 00000000000..0209cc3e300
--- /dev/null
+++ b/jstests/aggregation/sources/indexStats/verify_index_stats_output.js
@@ -0,0 +1,84 @@
+/**
+ * Basic test to verify the output of $indexStats.
+ *
+ * @tags: [assumes_read_concern_unchanged, requires_wiredtiger, do_not_wrap_aggregations_in_facets]
+ */
+(function() {
+"use strict";
+load('jstests/noPassthrough/libs/index_build.js');  // for waitForIndexBuildToStart().
+load('jstests/libs/fixture_helpers.js');            // for runCommandOnEachPrimary.
+load("jstests/aggregation/extras/utils.js");        // for resultsEq.
+
+const coll = db.index_stats_output;
+coll.drop();
+
+let bulk = coll.initializeUnorderedBulkOp();
+const nDocs = 100;
+for (let i = 0; i < nDocs; i++) {
+    bulk.insert({_id: i, a: i});
+}
+assert.commandWorked(bulk.execute());
+
+const indexKey = {
+    _id: 1,
+    a: 1
+};
+const indexName = "testIndex";
+
+// Verify that in progress index builds report matching 'spec' and 'building: true' in the output of
+// $indexStats.
+FixtureHelpers.runCommandOnEachPrimary({
+    db: db.getSiblingDB("admin"),
+    cmdObj: {configureFailPoint: "hangAfterStartingIndexBuild", mode: "alwaysOn"}
+});
+
+const join = startParallelShell(() => {
+    const indexName = "testIndex";
+    const indexKey = {_id: 1, a: 1};
+    assert.commandWorked(db.index_stats_output.createIndex(indexKey, {unique: 1, name: indexName}));
+});
+
+IndexBuildTest.waitForIndexBuildToStart(db, coll.getName(), indexName);
+
+let pausedOutput = coll.aggregate([{$indexStats: {}}, {$match: {name: indexName}}]).toArray();
+
+let allShards = [];
+let shardsFound = [];
+db.getSiblingDB("config").shards.find().forEach(function(shard) {
+    allShards.push(shard._id);
+});
+
+for (const indexStats of pausedOutput) {
+    assert.hasFields(indexStats, ["building", "spec"]);
+    // Each index should report building: true since the index build was paused.
+    assert.eq(indexStats["building"], true);
+    // Each index should report a spec that matches the parameters passed to createIndex().
+    let spec = indexStats["spec"];
+    assert.hasFields(spec, ["unique", "name", "key"]);
+    assert.eq(spec["unique"], true);
+    assert.eq(spec["name"], indexName);
+    assert.eq(spec["key"], indexKey);
+    // In the sharded case, record the reported shard names and compare them against the
+    // names of known shards.
+    if (indexStats.hasOwnProperty("shard")) {
+        shardsFound.push(indexStats["shard"]);
+    }
+}
+
+for (const shard of shardsFound) {
+    assert.contains(shard, allShards);
+}
+
+FixtureHelpers.runCommandOnEachPrimary({
+    db: db.getSiblingDB("admin"),
+    cmdObj: {configureFailPoint: "hangAfterStartingIndexBuild", mode: "off"}
+});
+join();
+
+// Verify that there is no 'building' field in the $indexStats output for our created index once the
+// index build is complete.
+let finishedOutput = coll.aggregate([{$indexStats: {}}, {$match: {name: indexName}}]).toArray();
+for (const indexStats of finishedOutput) {
+    assert(!indexStats.hasOwnProperty("building"), tojson(indexStats));
+}
+})();
\ No newline at end of file
diff --git a/jstests/libs/override_methods/detect_spawning_own_mongod.js b/jstests/libs/override_methods/detect_spawning_own_mongod.js
index 42b95a58d51..3e542f048ec 100644
--- a/jstests/libs/override_methods/detect_spawning_own_mongod.js
+++ b/jstests/libs/override_methods/detect_spawning_own_mongod.js
@@ -27,12 +27,23 @@ const STOverrideConstructor = function() {
 // still keep any static properties it has.
 ShardingTest = Object.assign(STOverrideConstructor, ShardingTest);
 
-const RSTOverrideConstructor = function() {
-    throw new Error("Detected ReplSetTest() call in js test from passthrough suite. " +
-                    "Consider moving the test to one of the jstests/noPassthrough/, " +
-                    "jstests/replsets/, or jstests/sharding/ directories.");
+const RSTOverrideConstructor = function(opts) {
+    if (typeof opts !== 'string' && !(opts instanceof String)) {
+        throw new Error("Detected ReplSetTest() call in js test from passthrough suite. " +
+                        "Consider moving the test to one of the jstests/noPassthrough/, " +
+                        "jstests/replsets/, or jstests/sharding/ directories.");
+    } else {
+        // If we are creating a ReplSetTest using a pre-existing replica set, simply reassign
+        // the old constructor and invoke it.
+        Object.assign(this, ReplSetTest.overridenConstructor);
+        return this.overridenConstructor(opts);
+    }
 };
 
+// Capture the old constructor for ReplSetTest in the event that the call to ReplSetTest() is
+// attempting to reconstruct a replica set and not creating a new one.
+ReplSetTest.overridenConstructor = ReplSetTest;
+
 // Same as the above Object.assign() call. In particular, we want to preserve the
 // ReplSetTest.kDefaultTimeoutMS property, which should be accessible to tests in the
 // passthrough suite.
diff --git a/jstests/noPassthrough/libs/index_build.js b/jstests/noPassthrough/libs/index_build.js
index e9d0c17191d..e60dabeb72c 100644
--- a/jstests/noPassthrough/libs/index_build.js
+++ b/jstests/noPassthrough/libs/index_build.js
@@ -19,22 +19,29 @@ class IndexBuildTest {
      * Accepts optional filter that can be used to customize the db.currentOp() query.
      */
     static getIndexBuildOpId(database, collectionName, indexName, filter) {
-        const result = database.currentOp(filter || true);
-        assert.commandWorked(result);
+        let pipeline = [{$currentOp: {allUsers: true}}];
+        if (filter) {
+            pipeline.push({$match: filter});
+        }
+        const result = database.getSiblingDB("admin")
+                           .aggregate(pipeline, {readConcern: {level: "local"}})
+                           .toArray();
         let indexBuildOpId = -1;
         let indexBuildObj = {};
         let indexBuildNamespace = "";
 
-        result.inprog.forEach(function(op) {
+        result.forEach(function(op) {
             if (op.op != 'command') {
                 return;
             }
-            if (op.command.createIndexes === undefined) {
+            const cmdBody = op.command;
+
+            if (cmdBody.createIndexes === undefined) {
                 return;
             }
             // If no collection is provided, return any index build.
-            if (!collectionName || op.command.createIndexes === collectionName) {
-                op.command.indexes.forEach((index) => {
+            if (!collectionName || cmdBody.createIndexes === collectionName) {
+                cmdBody.indexes.forEach((index) => {
                     if (!indexName || index.name === indexName) {
                         indexBuildOpId = op.opid;
                         indexBuildObj = index;
diff --git a/src/mongo/db/pipeline/document_source_index_stats.cpp b/src/mongo/db/pipeline/document_source_index_stats.cpp
index 7d02339794d..5612135cd78 100644
--- a/src/mongo/db/pipeline/document_source_index_stats.cpp
+++ b/src/mongo/db/pipeline/document_source_index_stats.cpp
@@ -48,21 +48,16 @@ const char* DocumentSourceIndexStats::getSourceName() const {
 }
 
 DocumentSource::GetNextResult DocumentSourceIndexStats::doGetNext() {
-    if (_indexStatsMap.empty()) {
-        _indexStatsMap = pExpCtx->mongoProcessInterface->getIndexStats(pExpCtx->opCtx, pExpCtx->ns);
-        _indexStatsIter = _indexStatsMap.begin();
+    if (_indexStats.empty()) {
+        _indexStats = pExpCtx->mongoProcessInterface->getIndexStats(
+            pExpCtx->opCtx, pExpCtx->ns, _processName, pExpCtx->fromMongos);
+        _indexStatsIter = _indexStats.cbegin();
     }
 
-    if (_indexStatsIter != _indexStatsMap.end()) {
-        const auto& stats = _indexStatsIter->second;
-        MutableDocument doc;
-        doc["name"] = Value(_indexStatsIter->first);
-        doc["key"] = Value(stats.indexKey);
-        doc["host"] = Value(_processName);
-        doc["accesses"]["ops"] = Value(stats.accesses.loadRelaxed());
-        doc["accesses"]["since"] = Value(stats.trackerStartTime);
+    if (_indexStatsIter != _indexStats.cend()) {
+        Document doc{std::move(*_indexStatsIter)};
         ++_indexStatsIter;
-        return doc.freeze();
+        return doc;
     }
 
     return GetNextResult::makeEOF();
diff --git a/src/mongo/db/pipeline/document_source_index_stats.h b/src/mongo/db/pipeline/document_source_index_stats.h
index 9198f817b33..d750f691e55 100644
--- a/src/mongo/db/pipeline/document_source_index_stats.h
+++ b/src/mongo/db/pipeline/document_source_index_stats.h
@@ -95,8 +95,8 @@ private:
     DocumentSourceIndexStats(const boost::intrusive_ptr<ExpressionContext>& pExpCtx);
     GetNextResult doGetNext() final;
 
-    CollectionIndexUsageMap _indexStatsMap;
-    CollectionIndexUsageMap::const_iterator _indexStatsIter;
+    std::vector<Document> _indexStats;
+    std::vector<Document>::const_iterator _indexStatsIter;
     std::string _processName;
 };
 
diff --git a/src/mongo/db/pipeline/mongo_process_interface.h b/src/mongo/db/pipeline/mongo_process_interface.h
index 82af588ab78..162abe4b428 100644
--- a/src/mongo/db/pipeline/mongo_process_interface.h
+++ b/src/mongo/db/pipeline/mongo_process_interface.h
@@ -170,8 +170,18 @@ public:
                                             bool multi,
                                             boost::optional<OID> targetEpoch) = 0;
 
-    virtual CollectionIndexUsageMap getIndexStats(OperationContext* opCtx,
-                                                  const NamespaceString& ns) = 0;
+    /**
+     * Returns index usage statistics for each index on collection 'ns' along with additional
+     * information including the index specification and whether the index is currently being built.
+     *
+     * By passing true for 'addShardName', the caller can request that each document in the
+     * resulting vector includes a 'shard' field which denotes this node's shard name. It is illegal
+     * to set this option unless this node is a shardsvr.
+     */
+    virtual std::vector<Document> getIndexStats(OperationContext* opCtx,
+                                                const NamespaceString& ns,
+                                                StringData host,
+                                                bool addShardName) = 0;
 
     virtual std::list<BSONObj> getIndexSpecs(OperationContext* opCtx,
                                              const NamespaceString& ns,
diff --git a/src/mongo/db/pipeline/mongos_process_interface.h b/src/mongo/db/pipeline/mongos_process_interface.h
index fcb3a1d1a5d..844abfb0ecf 100644
--- a/src/mongo/db/pipeline/mongos_process_interface.h
+++ b/src/mongo/db/pipeline/mongos_process_interface.h
@@ -87,10 +87,13 @@ public:
         MONGO_UNREACHABLE;
     }
 
-    CollectionIndexUsageMap getIndexStats(OperationContext* opCtx,
-                                          const NamespaceString& ns) final {
+    std::vector<Document> getIndexStats(OperationContext* opCtx,
+                                        const NamespaceString& ns,
+                                        StringData host,
+                                        bool addShardName) final {
         MONGO_UNREACHABLE;
     }
+
     std::list<BSONObj> getIndexSpecs(OperationContext* opCtx,
                                      const NamespaceString& ns,
                                      bool includeBuildUUIDs) final {
diff --git a/src/mongo/db/pipeline/process_interface_standalone.cpp b/src/mongo/db/pipeline/process_interface_standalone.cpp
index 9c6fedade8c..11698b4b85b 100644
--- a/src/mongo/db/pipeline/process_interface_standalone.cpp
+++ b/src/mongo/db/pipeline/process_interface_standalone.cpp
@@ -262,17 +262,51 @@ StatusWith<MongoProcessInterface::UpdateResult> MongoInterfaceStandalone::update
     return updateResult;
 }
 
-CollectionIndexUsageMap MongoInterfaceStandalone::getIndexStats(OperationContext* opCtx,
-                                                                const NamespaceString& ns) {
+std::vector<Document> MongoInterfaceStandalone::getIndexStats(OperationContext* opCtx,
+                                                              const NamespaceString& ns,
+                                                              StringData host,
+                                                              bool addShardName) {
     AutoGetCollectionForReadCommand autoColl(opCtx, ns);
 
     Collection* collection = autoColl.getCollection();
+    std::vector<Document> indexStats;
     if (!collection) {
         LOG(2) << "Collection not found on index stats retrieval: " << ns.ns();
-        return CollectionIndexUsageMap();
+        return indexStats;
     }
 
-    return CollectionQueryInfo::get(collection).getIndexUsageStats();
+    auto indexStatsMap = CollectionQueryInfo::get(collection).getIndexUsageStats();
+    for (auto&& indexStatsMapIter : indexStatsMap) {
+        auto indexName = indexStatsMapIter.first;
+        auto stats = indexStatsMapIter.second;
+        MutableDocument doc;
+        doc["name"] = Value(indexName);
+        doc["key"] = Value(stats.indexKey);
+        doc["host"] = Value(host);
+        doc["accesses"]["ops"] = Value(stats.accesses.loadRelaxed());
+        doc["accesses"]["since"] = Value(stats.trackerStartTime);
+
+        if (addShardName)
+            doc["shard"] = Value(getShardName(opCtx));
+
+        // Retrieve the relevant index entry.
+        auto idxCatalog = collection->getIndexCatalog();
+        auto idx = idxCatalog->findIndexByName(opCtx,
+                                               indexName,
+                                               /* includeUnfinishedIndexes */ true);
+        uassert(ErrorCodes::IndexNotFound,
+                "Could not find entry in IndexCatalog for index " + indexName,
+                idx);
+        auto entry = idxCatalog->getEntry(idx);
+        doc["spec"] = Value(idx->infoObj());
+
+        if (!entry->isReady(opCtx)) {
+            doc["building"] = Value(true);
+        }
+
+        indexStats.push_back(doc.freeze());
+    }
+    return indexStats;
 }
 
 std::list<BSONObj> MongoInterfaceStandalone::getIndexSpecs(OperationContext* opCtx,
diff --git a/src/mongo/db/pipeline/process_interface_standalone.h b/src/mongo/db/pipeline/process_interface_standalone.h
index cb97e3267fb..4e0be10e37b 100644
--- a/src/mongo/db/pipeline/process_interface_standalone.h
+++ b/src/mongo/db/pipeline/process_interface_standalone.h
@@ -76,7 +76,11 @@ public:
                                     bool multi,
                                     boost::optional<OID> targetEpoch) override;
 
-    CollectionIndexUsageMap getIndexStats(OperationContext* opCtx, const NamespaceString& ns) final;
+    std::vector<Document> getIndexStats(OperationContext* opCtx,
+                                        const NamespaceString& ns,
+                                        StringData host,
+                                        bool addShardName) final;
+
     std::list<BSONObj> getIndexSpecs(OperationContext* opCtx,
                                      const NamespaceString& ns,
                                      bool includeBuildUUIDs);
diff --git a/src/mongo/db/pipeline/stub_mongo_process_interface.h b/src/mongo/db/pipeline/stub_mongo_process_interface.h
index a4869edf1c1..efccf8ce08d 100644
--- a/src/mongo/db/pipeline/stub_mongo_process_interface.h
+++ b/src/mongo/db/pipeline/stub_mongo_process_interface.h
@@ -77,8 +77,10 @@ public:
         MONGO_UNREACHABLE;
     }
 
-    CollectionIndexUsageMap getIndexStats(OperationContext* opCtx,
-                                          const NamespaceString& ns) override {
+    std::vector<Document> getIndexStats(OperationContext* opCtx,
+                                        const NamespaceString& ns,
+                                        StringData host,
+                                        bool addShardName) override {
         MONGO_UNREACHABLE;
     }