SERVER-12335 indexKey $meta projection not populating fields

22 files changed, 224 insertions, 90 deletions

diff --git a/src/mongo/db/exec/exclusion_projection_executor.h b/src/mongo/db/exec/exclusion_projection_executor.h
index a025d693e26..9d1a898ed00 100644
--- a/src/mongo/db/exec/exclusion_projection_executor.h
+++ b/src/mongo/db/exec/exclusion_projection_executor.h
@@ -52,6 +52,15 @@ public:
 
     void reportDependencies(DepsTracker* deps) const final {
         // We have no dependencies on specific fields, since we only know the fields to be removed.
+        // We may have expression dependencies though, as $meta expression can be used with
+        // exclusion.
+        for (auto&& expressionPair : _expressions) {
+            expressionPair.second->addDependencies(deps);
+        }
+
+        for (auto&& childPair : _children) {
+            childPair.second->reportDependencies(deps);
+        }
     }
 
 protected:
@@ -107,6 +116,7 @@ public:
     }
 
     DepsTracker::State addDependencies(DepsTracker* deps) const final {
+        _root->reportDependencies(deps);
         if (_rootReplacementExpression) {
             _rootReplacementExpression->addDependencies(deps);
         }
diff --git a/src/mongo/db/exec/exclusion_projection_executor_test.cpp b/src/mongo/db/exec/exclusion_projection_executor_test.cpp
index 59d1748d5fe..a65fa95300e 100644
--- a/src/mongo/db/exec/exclusion_projection_executor_test.cpp
+++ b/src/mongo/db/exec/exclusion_projection_executor_test.cpp
@@ -315,6 +315,68 @@ TEST(ExclusionProjectionExecutionTest, ShouldAlwaysKeepMetadataFromOriginalDoc)
     ASSERT_DOCUMENT_EQ(result, expectedDoc.freeze());
 }
 
+TEST(ExclusionProjectionExecutionTest, ShouldEvalauateMetaExpressions) {
+    auto exclusion =
+        makeExclusionProjectionWithDefaultPolicies(fromjson("{a: 0, c: {$meta: 'textScore'}, "
+                                                            "d: {$meta: 'randVal'}, "
+                                                            "e: {$meta: 'searchScore'}, "
+                                                            "f: {$meta: 'searchHighlights'}, "
+                                                            "g: {$meta: 'geoNearDistance'}, "
+                                                            "h: {$meta: 'geoNearPoint'}, "
+                                                            "i: {$meta: 'recordId'}, "
+                                                            "j: {$meta: 'indexKey'}, "
+                                                            "k: {$meta: 'sortKey'}}"));
+
+    MutableDocument inputDocBuilder(Document{{"a", 1}, {"b", 2}});
+    inputDocBuilder.metadata().setTextScore(0.0);
+    inputDocBuilder.metadata().setRandVal(1.0);
+    inputDocBuilder.metadata().setSearchScore(2.0);
+    inputDocBuilder.metadata().setSearchHighlights(Value{"foo"_sd});
+    inputDocBuilder.metadata().setGeoNearDistance(3.0);
+    inputDocBuilder.metadata().setGeoNearPoint(Value{BSON_ARRAY(4 << 5)});
+    inputDocBuilder.metadata().setRecordId(RecordId{6});
+    inputDocBuilder.metadata().setIndexKey(BSON("foo" << 7));
+    inputDocBuilder.metadata().setSortKey(Value{Document{{"bar", 8}}}, true);
+    Document inputDoc = inputDocBuilder.freeze();
+
+    auto result = exclusion->applyTransformation(inputDoc);
+
+    ASSERT_DOCUMENT_EQ(result,
+                       Document{fromjson("{b: 2, c: 0.0, d: 1.0, e: 2.0, f: 'foo', g: 3.0, "
+                                         "h: [4, 5], i: 6, j: {foo: 7}, k: {'': {bar: 8}}}")});
+}
+
+TEST(ExclusionProjectionExecutionTest, ShouldAddMetaExpressionsToDependencies) {
+    auto exclusion =
+        makeExclusionProjectionWithDefaultPolicies(fromjson("{a: 0, c: {$meta: 'textScore'}, "
+                                                            "d: {$meta: 'randVal'}, "
+                                                            "e: {$meta: 'searchScore'}, "
+                                                            "f: {$meta: 'searchHighlights'}, "
+                                                            "g: {$meta: 'geoNearDistance'}, "
+                                                            "h: {$meta: 'geoNearPoint'}, "
+                                                            "i: {$meta: 'recordId'}, "
+                                                            "j: {$meta: 'indexKey'}, "
+                                                            "k: {$meta: 'sortKey'}}"));
+
+    DepsTracker deps;
+    exclusion->addDependencies(&deps);
+
+    ASSERT_EQ(deps.fields.size(), 0UL);
+
+    // We do not add the dependencies for SEARCH_SCORE or SEARCH_HIGHLIGHTS because those values
+    // are not stored in the collection (or in mongod at all).
+    ASSERT_FALSE(deps.metadataDeps()[DocumentMetadataFields::kSearchScore]);
+    ASSERT_FALSE(deps.metadataDeps()[DocumentMetadataFields::kSearchHighlights]);
+
+    ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kTextScore]);
+    ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kRandVal]);
+    ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kGeoNearDist]);
+    ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kGeoNearPoint]);
+    ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kRecordId]);
+    ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kIndexKey]);
+    ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kSortKey]);
+}
+
 //
 // _id exclusion policy.
 //
diff --git a/src/mongo/db/exec/inclusion_projection_executor_test.cpp b/src/mongo/db/exec/inclusion_projection_executor_test.cpp
index b9df51e21ba..fc221f0fcc7 100644
--- a/src/mongo/db/exec/inclusion_projection_executor_test.cpp
+++ b/src/mongo/db/exec/inclusion_projection_executor_test.cpp
@@ -623,6 +623,68 @@ TEST(InclusionProjectionExecutionTest, ShouldAlwaysKeepMetadataFromOriginalDoc)
     ASSERT_DOCUMENT_EQ(result, expectedDoc.freeze());
 }
 
+TEST(InclusionProjectionExecutionTest, ShouldAddMetaExpressionsToDependencies) {
+    auto inclusion =
+        makeInclusionProjectionWithDefaultPolicies(fromjson("{a: 1, c: {$meta: 'textScore'}, "
+                                                            "d: {$meta: 'randVal'}, "
+                                                            "e: {$meta: 'searchScore'}, "
+                                                            "f: {$meta: 'searchHighlights'}, "
+                                                            "g: {$meta: 'geoNearDistance'}, "
+                                                            "h: {$meta: 'geoNearPoint'}, "
+                                                            "i: {$meta: 'recordId'}, "
+                                                            "j: {$meta: 'indexKey'}, "
+                                                            "k: {$meta: 'sortKey'}}"));
+
+    DepsTracker deps;
+    inclusion->addDependencies(&deps);
+
+    ASSERT_EQ(deps.fields.size(), 2UL);
+
+    // We do not add the dependencies for SEARCH_SCORE or SEARCH_HIGHLIGHTS because those values
+    // are not stored in the collection (or in mongod at all).
+    ASSERT_FALSE(deps.metadataDeps()[DocumentMetadataFields::kSearchScore]);
+    ASSERT_FALSE(deps.metadataDeps()[DocumentMetadataFields::kSearchHighlights]);
+
+    ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kTextScore]);
+    ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kRandVal]);
+    ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kGeoNearDist]);
+    ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kGeoNearPoint]);
+    ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kRecordId]);
+    ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kIndexKey]);
+    ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kSortKey]);
+}
+
+TEST(InclusionProjectionExecutionTest, ShouldEvalauateMetaExpressions) {
+    auto inclusion =
+        makeInclusionProjectionWithDefaultPolicies(fromjson("{a: 1, c: {$meta: 'textScore'}, "
+                                                            "d: {$meta: 'randVal'}, "
+                                                            "e: {$meta: 'searchScore'}, "
+                                                            "f: {$meta: 'searchHighlights'}, "
+                                                            "g: {$meta: 'geoNearDistance'}, "
+                                                            "h: {$meta: 'geoNearPoint'}, "
+                                                            "i: {$meta: 'recordId'}, "
+                                                            "j: {$meta: 'indexKey'}, "
+                                                            "k: {$meta: 'sortKey'}}"));
+
+    MutableDocument inputDocBuilder(Document{{"a", 1}});
+    inputDocBuilder.metadata().setTextScore(0.0);
+    inputDocBuilder.metadata().setRandVal(1.0);
+    inputDocBuilder.metadata().setSearchScore(2.0);
+    inputDocBuilder.metadata().setSearchHighlights(Value{"foo"_sd});
+    inputDocBuilder.metadata().setGeoNearDistance(3.0);
+    inputDocBuilder.metadata().setGeoNearPoint(Value{BSON_ARRAY(4 << 5)});
+    inputDocBuilder.metadata().setRecordId(RecordId{6});
+    inputDocBuilder.metadata().setIndexKey(BSON("foo" << 7));
+    inputDocBuilder.metadata().setSortKey(Value{Document{{"bar", 8}}}, true);
+    Document inputDoc = inputDocBuilder.freeze();
+
+    auto result = inclusion->applyTransformation(inputDoc);
+
+    ASSERT_DOCUMENT_EQ(result,
+                       Document{fromjson("{a: 1, c: 0.0, d: 1.0, e: 2.0, f: 'foo', g: 3.0, "
+                                         "h: [4, 5], i: 6, j: {foo: 7}, k: {'': {bar: 8}}}")});
+}
+
 //
 // _id inclusion policy.
 //
diff --git a/src/mongo/db/pipeline/dependencies.cpp b/src/mongo/db/pipeline/dependencies.cpp
index cfe063b976e..b2cde6209b3 100644
--- a/src/mongo/db/pipeline/dependencies.cpp
+++ b/src/mongo/db/pipeline/dependencies.cpp
@@ -83,16 +83,9 @@ BSONObj DepsTracker::toProjectionWithoutMetadata() const {
 }
 
 void DepsTracker::setNeedsMetadata(DocumentMetadataFields::MetaType type, bool required) {
-    // For everything but sortKey/randval metadata, check that it's available. A pipeline can
-    // generate those types of metadata.
-
-    if (type != DocumentMetadataFields::MetaType::kSortKey &&
-        type != DocumentMetadataFields::MetaType::kRandVal) {
-        uassert(40218,
-                str::stream() << "pipeline requires " << type
-                              << " metadata, but it is not available",
-                !required || isMetadataAvailable(type));
-    }
+    uassert(40218,
+            str::stream() << "pipeline requires " << type << " metadata, but it is not available",
+            !required || !_unavailableMetadata[type]);
 
     // If the metadata type is not required, then it should not be recorded as a metadata
     // dependency.
diff --git a/src/mongo/db/pipeline/dependencies.h b/src/mongo/db/pipeline/dependencies.h
index eff5610b089..aab2e21bfb4 100644
--- a/src/mongo/db/pipeline/dependencies.h
+++ b/src/mongo/db/pipeline/dependencies.h
@@ -86,13 +86,20 @@ struct DepsTracker {
         QueryMetadataBitSet(1 << DocumentMetadataFields::kTextScore);
 
     /**
+     * By default, certain metadata is unavailable to the pipeline, unless explicitly specified
+     * that it is available. This state represents all metadata which is not available by default.
+     */
+    static constexpr auto kDefaultUnavailableMetadata = QueryMetadataBitSet(
+        (1 << DocumentMetadataFields::kTextScore) | (1 << DocumentMetadataFields::kGeoNearDist) |
+        (1 << DocumentMetadataFields::kGeoNearPoint));
+
+    /**
      * Represents a state where no metadata is available.
      */
     static constexpr auto kNoMetadata = QueryMetadataBitSet();
 
-
-    DepsTracker(QueryMetadataBitSet metadataAvailable = kNoMetadata)
-        : _metadataAvailable(metadataAvailable) {}
+    DepsTracker(const QueryMetadataBitSet& unavailableMetadata = kNoMetadata)
+        : _unavailableMetadata{unavailableMetadata} {}
 
     /**
      * Returns a projection object covering the non-metadata dependencies tracked by this class, or
@@ -115,19 +122,11 @@ struct DepsTracker {
     }
 
     /**
-     * Returns a value with bits set indicating the types of metadata available.
-     */
-    QueryMetadataBitSet getMetadataAvailable() const {
-        return _metadataAvailable;
-    }
-
-    /**
-     * Returns true if the DepsTracker the metadata 'type' is available to the pipeline. It is
-     * illegal to call this with MetadataType::SORT_KEY, since the sort key will always be available
-     * if needed.
+     * Returns a value with bits set indicating the types of metadata not available to the
+     * pipeline.
      */
-    bool isMetadataAvailable(DocumentMetadataFields::MetaType type) const {
-        return _metadataAvailable[type];
+    QueryMetadataBitSet getUnavailableMetadata() const {
+        return _unavailableMetadata;
     }
 
     /**
@@ -174,8 +173,8 @@ struct DepsTracker {
     bool needWholeDocument = false;  // If true, ignore 'fields'; the whole document is needed.
 
 private:
-    // Represents all metadata available to the pipeline.
-    QueryMetadataBitSet _metadataAvailable;
+    // Represents all metadata not available to the pipeline.
+    QueryMetadataBitSet _unavailableMetadata;
 
     // Represents which metadata is used by the pipeline. This is populated while performing
     // dependency analysis.
diff --git a/src/mongo/db/pipeline/dependencies_test.cpp b/src/mongo/db/pipeline/dependencies_test.cpp
index b6c688da8b0..58e1066d158 100644
--- a/src/mongo/db/pipeline/dependencies_test.cpp
+++ b/src/mongo/db/pipeline/dependencies_test.cpp
@@ -130,7 +130,7 @@ TEST(DependenciesToProjectionTest, ShouldOutputEmptyObjectIfEntireDocumentNeeded
 
 TEST(DependenciesToProjectionTest, ShouldOnlyRequestTextScoreIfEntireDocumentAndTextScoreNeeded) {
     const char* array[] = {"a"};  // needTextScore with needWholeDocument
-    DepsTracker deps(DepsTracker::kOnlyTextScore);
+    DepsTracker deps(DepsTracker::kAllMetadata & ~DepsTracker::kOnlyTextScore);
     deps.fields = arrayToSet(array);
     deps.needWholeDocument = true;
     deps.setNeedsMetadata(DocumentMetadataFields::kTextScore, true);
@@ -142,7 +142,7 @@ TEST(DependenciesToProjectionTest, ShouldOnlyRequestTextScoreIfEntireDocumentAnd
 TEST(DependenciesToProjectionTest,
      ShouldRequireFieldsAndTextScoreIfTextScoreNeededWithoutWholeDocument) {
     const char* array[] = {"a"};  // needTextScore without needWholeDocument
-    DepsTracker deps(DepsTracker::kOnlyTextScore);
+    DepsTracker deps(DepsTracker::kAllMetadata & ~DepsTracker::kOnlyTextScore);
     deps.fields = arrayToSet(array);
     deps.setNeedsMetadata(DocumentMetadataFields::kTextScore, true);
     ASSERT_BSONOBJ_EQ(deps.toProjectionWithoutMetadata(), BSON("a" << 1 << "_id" << 0));
@@ -151,7 +151,7 @@ TEST(DependenciesToProjectionTest,
 }
 
 TEST(DependenciesToProjectionTest, ShouldProduceEmptyObjectIfThereAreNoDependencies) {
-    DepsTracker deps(DepsTracker::kOnlyTextScore);
+    DepsTracker deps(DepsTracker::kAllMetadata & ~DepsTracker::kOnlyTextScore);
     deps.fields = {};
     deps.needWholeDocument = false;
     deps.setNeedsMetadata(DocumentMetadataFields::kTextScore, false);
@@ -159,7 +159,7 @@ TEST(DependenciesToProjectionTest, ShouldProduceEmptyObjectIfThereAreNoDependenc
 }
 
 TEST(DependenciesToProjectionTest, ShouldReturnEmptyObjectIfOnlyTextScoreIsNeeded) {
-    DepsTracker deps(DepsTracker::kOnlyTextScore);
+    DepsTracker deps(DepsTracker::kAllMetadata & ~DepsTracker::kOnlyTextScore);
     deps.fields = {};
     deps.needWholeDocument = false;
     deps.setNeedsMetadata(DocumentMetadataFields::kTextScore, true);
@@ -171,7 +171,7 @@ TEST(DependenciesToProjectionTest, ShouldReturnEmptyObjectIfOnlyTextScoreIsNeede
 
 TEST(DependenciesToProjectionTest,
      ShouldRequireTextScoreIfNoFieldsPresentButWholeDocumentIsNeeded) {
-    DepsTracker deps(DepsTracker::kOnlyTextScore);
+    DepsTracker deps(DepsTracker::kAllMetadata & ~DepsTracker::kOnlyTextScore);
     deps.fields = {};
     deps.needWholeDocument = true;
     deps.setNeedsMetadata(DocumentMetadataFields::kTextScore, true);
diff --git a/src/mongo/db/pipeline/document_source_add_fields_test.cpp b/src/mongo/db/pipeline/document_source_add_fields_test.cpp
index b5a1dd85baa..75fb1e9eb02 100644
--- a/src/mongo/db/pipeline/document_source_add_fields_test.cpp
+++ b/src/mongo/db/pipeline/document_source_add_fields_test.cpp
@@ -146,7 +146,7 @@ TEST_F(AddFieldsTest, ShouldAddReferencedFieldsToDependencies) {
     auto addFields = DocumentSourceAddFields::create(
         fromjson("{a: true, x: '$b', y: {$and: ['$c','$d']}, z: {$meta: 'textScore'}}"),
         getExpCtx());
-    DepsTracker dependencies(DepsTracker::kOnlyTextScore);
+    DepsTracker dependencies(DepsTracker::kAllMetadata & ~DepsTracker::kOnlyTextScore);
     ASSERT_EQUALS(DepsTracker::State::SEE_NEXT, addFields->getDependencies(&dependencies));
     ASSERT_EQUALS(3U, dependencies.fields.size());
 
diff --git a/src/mongo/db/pipeline/document_source_bucket_auto_test.cpp b/src/mongo/db/pipeline/document_source_bucket_auto_test.cpp
index 9c68c05a50c..abe9633b5aa 100644
--- a/src/mongo/db/pipeline/document_source_bucket_auto_test.cpp
+++ b/src/mongo/db/pipeline/document_source_bucket_auto_test.cpp
@@ -457,7 +457,7 @@ TEST_F(BucketAutoTests, ShouldNeedTextScoreInDependenciesFromGroupByField) {
     auto bucketAuto =
         createBucketAuto(fromjson("{$bucketAuto : {groupBy : {$meta: 'textScore'}, buckets : 2}}"));
 
-    DepsTracker dependencies(DepsTracker::kOnlyTextScore);
+    DepsTracker dependencies(DepsTracker::kAllMetadata & ~DepsTracker::kOnlyTextScore);
     ASSERT_EQUALS(DepsTracker::State::EXHAUSTIVE_ALL, bucketAuto->getDependencies(&dependencies));
     ASSERT_EQUALS(0U, dependencies.fields.size());
 
@@ -470,7 +470,7 @@ TEST_F(BucketAutoTests, ShouldNeedTextScoreInDependenciesFromOutputField) {
         createBucketAuto(fromjson("{$bucketAuto : {groupBy : '$x', buckets : 2, output: {avg : "
                                   "{$avg : {$meta : 'textScore'}}}}}"));
 
-    DepsTracker dependencies(DepsTracker::kOnlyTextScore);
+    DepsTracker dependencies(DepsTracker::kAllMetadata & ~DepsTracker::kOnlyTextScore);
     ASSERT_EQUALS(DepsTracker::State::EXHAUSTIVE_ALL, bucketAuto->getDependencies(&dependencies));
     ASSERT_EQUALS(1U, dependencies.fields.size());
 
diff --git a/src/mongo/db/pipeline/document_source_facet.cpp b/src/mongo/db/pipeline/document_source_facet.cpp
index 71e9571f869..fe0f40913a8 100644
--- a/src/mongo/db/pipeline/document_source_facet.cpp
+++ b/src/mongo/db/pipeline/document_source_facet.cpp
@@ -296,7 +296,7 @@ bool DocumentSourceFacet::usedDisk() {
 DepsTracker::State DocumentSourceFacet::getDependencies(DepsTracker* deps) const {
     const bool scopeHasVariables = pExpCtx->variablesParseState.hasDefinedVariables();
     for (auto&& facet : _facets) {
-        auto subDepsTracker = facet.pipeline->getDependencies(deps->getMetadataAvailable());
+        auto subDepsTracker = facet.pipeline->getDependencies(deps->getUnavailableMetadata());
 
         deps->fields.insert(subDepsTracker.fields.begin(), subDepsTracker.fields.end());
         deps->vars.insert(subDepsTracker.vars.begin(), subDepsTracker.vars.end());
diff --git a/src/mongo/db/pipeline/document_source_facet_test.cpp b/src/mongo/db/pipeline/document_source_facet_test.cpp
index 980786689a8..6fa1ae50e88 100644
--- a/src/mongo/db/pipeline/document_source_facet_test.cpp
+++ b/src/mongo/db/pipeline/document_source_facet_test.cpp
@@ -713,7 +713,7 @@ TEST_F(DocumentSourceFacetTest, ShouldRequireTextScoreIfAnyPipelineRequiresTextS
     facets.emplace_back("needsTextScore", std::move(thirdPipeline));
     auto facetStage = DocumentSourceFacet::create(std::move(facets), ctx);
 
-    DepsTracker deps(DepsTracker::kOnlyTextScore);
+    DepsTracker deps(DepsTracker::kAllMetadata & ~DepsTracker::kOnlyTextScore);
     ASSERT_EQ(facetStage->getDependencies(&deps), DepsTracker::State::EXHAUSTIVE_ALL);
     ASSERT_TRUE(deps.needWholeDocument);
     ASSERT_TRUE(deps.getNeedsMetadata(DocumentMetadataFields::kTextScore));
@@ -733,7 +733,7 @@ TEST_F(DocumentSourceFacetTest, ShouldThrowIfAnyPipelineRequiresTextScoreButItIs
     facets.emplace_back("needsTextScore", std::move(secondPipeline));
     auto facetStage = DocumentSourceFacet::create(std::move(facets), ctx);
 
-    DepsTracker deps(DepsTracker::kNoMetadata);
+    DepsTracker deps(DepsTracker::kAllMetadata);
     ASSERT_THROWS(facetStage->getDependencies(&deps), AssertionException);
 }
 
diff --git a/src/mongo/db/pipeline/document_source_lookup.cpp b/src/mongo/db/pipeline/document_source_lookup.cpp
index 5ffc973c722..9688aa1723b 100644
--- a/src/mongo/db/pipeline/document_source_lookup.cpp
+++ b/src/mongo/db/pipeline/document_source_lookup.cpp
@@ -724,12 +724,12 @@ DepsTracker::State DocumentSourceLookUp::getDependencies(DepsTracker* deps) cons
         // We will use the introspection pipeline which we prebuilt during construction.
         invariant(_resolvedIntrospectionPipeline);
 
-        // We are not attempting to enforce that any referenced metadata are in fact available,
+        // We are not attempting to enforce that any referenced metadata are in fact unavailable,
         // this is done elsewhere. We only need to know what variable dependencies exist in the
         // subpipeline for the top-level pipeline. So without knowledge of what metadata is in fact
-        // available, we "lie" and say that all metadata is available to avoid tripping any
+        // unavailable, we "lie" and say that all metadata is available to avoid tripping any
         // assertions.
-        DepsTracker subDeps(DepsTracker::kAllMetadata);
+        DepsTracker subDeps(DepsTracker::kNoMetadata);
 
         // Get the subpipeline dependencies. Subpipeline stages may reference both 'let' variables
         // declared by this $lookup and variables declared externally.
diff --git a/src/mongo/db/pipeline/document_source_match.cpp b/src/mongo/db/pipeline/document_source_match.cpp
index 494b3535879..7da9d5a3f3c 100644
--- a/src/mongo/db/pipeline/document_source_match.cpp
+++ b/src/mongo/db/pipeline/document_source_match.cpp
@@ -505,8 +505,8 @@ void DocumentSourceMatch::rebuild(BSONObj filter) {
         _predicate, pExpCtx, ExtensionsCallbackNoop(), Pipeline::kAllowedMatcherFeatures));
     _isTextQuery = isTextQuery(_predicate);
     _dependencies =
-        DepsTracker(_isTextQuery ? QueryMetadataBitSet().set(DocumentMetadataFields::kTextScore)
-                                 : DepsTracker::kNoMetadata);
+        DepsTracker(_isTextQuery ? DepsTracker::kAllMetadata & ~DepsTracker::kOnlyTextScore
+                                 : DepsTracker::kAllMetadata);
     getDependencies(&_dependencies);
 }
 
diff --git a/src/mongo/db/pipeline/document_source_match_test.cpp b/src/mongo/db/pipeline/document_source_match_test.cpp
index 3d42a81f91d..cb4821161fd 100644
--- a/src/mongo/db/pipeline/document_source_match_test.cpp
+++ b/src/mongo/db/pipeline/document_source_match_test.cpp
@@ -225,7 +225,7 @@ TEST_F(DocumentSourceMatchTest, ShouldAddDependenciesOfAllBranchesOfOrClause) {
 
 TEST_F(DocumentSourceMatchTest, TextSearchShouldRequireWholeDocumentAndTextScore) {
     auto match = DocumentSourceMatch::create(fromjson("{$text: {$search: 'hello'} }"), getExpCtx());
-    DepsTracker dependencies(DepsTracker::kOnlyTextScore);
+    DepsTracker dependencies(DepsTracker::kAllMetadata & ~DepsTracker::kOnlyTextScore);
     ASSERT_EQUALS(DepsTracker::State::EXHAUSTIVE_FIELDS, match->getDependencies(&dependencies));
     ASSERT_EQUALS(true, dependencies.needWholeDocument);
     ASSERT_EQUALS(true, dependencies.getNeedsMetadata(DocumentMetadataFields::kTextScore));
diff --git a/src/mongo/db/pipeline/document_source_project_test.cpp b/src/mongo/db/pipeline/document_source_project_test.cpp
index ff4fa9d6b5f..9764298d528 100644
--- a/src/mongo/db/pipeline/document_source_project_test.cpp
+++ b/src/mongo/db/pipeline/document_source_project_test.cpp
@@ -171,7 +171,7 @@ TEST_F(ProjectStageTest, InclusionShouldAddDependenciesOfIncludedAndComputedFiel
         fromjson("{a: true, x: '$b', y: {$and: ['$c','$d']}, z: {$meta: 'textScore'}}"),
         getExpCtx(),
         "$project"_sd);
-    DepsTracker dependencies(DepsTracker::kOnlyTextScore);
+    DepsTracker dependencies(DepsTracker::kAllMetadata & ~DepsTracker::kOnlyTextScore);
     ASSERT_EQUALS(DepsTracker::State::EXHAUSTIVE_FIELDS, project->getDependencies(&dependencies));
     ASSERT_EQUALS(5U, dependencies.fields.size());
 
diff --git a/src/mongo/db/pipeline/document_source_sequential_document_cache.cpp b/src/mongo/db/pipeline/document_source_sequential_document_cache.cpp
index 971971ae19b..a23563adca5 100644
--- a/src/mongo/db/pipeline/document_source_sequential_document_cache.cpp
+++ b/src/mongo/db/pipeline/document_source_sequential_document_cache.cpp
@@ -111,11 +111,11 @@ Pipeline::SourceContainer::iterator DocumentSourceSequentialDocumentCache::doOpt
 
     // In the context of this optimization, we are only interested in figuring out
     // which external variables are referenced in the pipeline. We are not attempting
-    // to enforce that any referenced metadata are in fact available, this is done
-    // elsewhere. So without knowledge of what metadata is in fact available, here
+    // to enforce that any referenced metadata are in fact unavailable, this is done
+    // elsewhere. So without knowledge of what metadata is in fact unavailable, here
     // we "lie" and say that all metadata is available to avoid tripping any
     // assertions.
-    DepsTracker deps(DepsTracker::kAllMetadata);
+    DepsTracker deps(DepsTracker::kNoMetadata);
 
     // Iterate through the pipeline stages until we find one which references an external variable.
     for (; prefixSplit != container->end(); ++prefixSplit) {
diff --git a/src/mongo/db/pipeline/pipeline.cpp b/src/mongo/db/pipeline/pipeline.cpp
index 03a242a3e29..6296810a241 100644
--- a/src/mongo/db/pipeline/pipeline.cpp
+++ b/src/mongo/db/pipeline/pipeline.cpp
@@ -486,14 +486,14 @@ void Pipeline::addFinalSource(intrusive_ptr<DocumentSource> source) {
     _sources.push_back(source);
 }
 
-DepsTracker Pipeline::getDependencies(QueryMetadataBitSet metadataAvailable) const {
-    DepsTracker deps(metadataAvailable);
+DepsTracker Pipeline::getDependencies(QueryMetadataBitSet unavailableMetadata) const {
+    DepsTracker deps(unavailableMetadata);
     const bool scopeHasVariables = pCtx->variablesParseState.hasDefinedVariables();
     bool skipFieldsAndMetadataDeps = false;
     bool knowAllFields = false;
     bool knowAllMeta = false;
     for (auto&& source : _sources) {
-        DepsTracker localDeps(deps.getMetadataAvailable());
+        DepsTracker localDeps(deps.getUnavailableMetadata());
         DepsTracker::State status = source->getDependencies(&localDeps);
 
         deps.vars.insert(localDeps.vars.begin(), localDeps.vars.end());
@@ -532,7 +532,7 @@ DepsTracker Pipeline::getDependencies(QueryMetadataBitSet metadataAvailable) con
     if (!knowAllFields)
         deps.needWholeDocument = true;  // don't know all fields we need
 
-    if (metadataAvailable[DocumentMetadataFields::kTextScore]) {
+    if (!unavailableMetadata[DocumentMetadataFields::kTextScore]) {
         // If there is a text score, assume we need to keep it if we can't prove we don't. If we are
         // the first half of a pipeline which has been split, future stages might need it.
         if (!knowAllMeta) {
diff --git a/src/mongo/db/pipeline/pipeline.h b/src/mongo/db/pipeline/pipeline.h
index 39ba6372687..7c32a51c197 100644
--- a/src/mongo/db/pipeline/pipeline.h
+++ b/src/mongo/db/pipeline/pipeline.h
@@ -248,10 +248,10 @@ public:
     std::vector<Value> writeExplainOps(ExplainOptions::Verbosity verbosity) const;
 
     /**
-     * Returns the dependencies needed by this pipeline. 'metadataAvailable' should reflect what
-     * metadata is present on documents that are input to the front of the pipeline.
+     * Returns the dependencies needed by this pipeline. 'unavailableMetadata' should reflect what
+     * metadata is not present on documents that are input to the front of the pipeline.
      */
-    DepsTracker getDependencies(QueryMetadataBitSet metadataAvailable) const;
+    DepsTracker getDependencies(QueryMetadataBitSet unavailableMetadata) const;
 
     const SourceContainer& getSources() const {
         return _sources;
diff --git a/src/mongo/db/pipeline/pipeline_d.cpp b/src/mongo/db/pipeline/pipeline_d.cpp
index 4f645f91dfa..3b44b87a88b 100644
--- a/src/mongo/db/pipeline/pipeline_d.cpp
+++ b/src/mongo/db/pipeline/pipeline_d.cpp
@@ -362,7 +362,7 @@ PipelineD::buildInnerQueryExecutor(Collection* collection,
                 // Pipeline) and an exception is thrown, an invariant will trigger in the
                 // DocumentSourceCursor. This is a design flaw in DocumentSourceCursor.
 
-                auto deps = pipeline->getDependencies(DepsTracker::kNoMetadata);
+                auto deps = pipeline->getDependencies(DepsTracker::kAllMetadata);
                 const bool shouldProduceEmptyDocs = deps.hasNoRequirements();
                 auto attachExecutorCallback =
                     [shouldProduceEmptyDocs](
@@ -542,9 +542,9 @@ PipelineD::buildInnerQueryExecutorGeneric(Collection* collection,
     // layer, but that is handled elsewhere.
     const auto limit = extractLimitForPushdown(pipeline);
 
-    auto metadataAvailable = DocumentSourceMatch::isTextQuery(queryObj)
-        ? DepsTracker::kOnlyTextScore
-        : DepsTracker::kNoMetadata;
+    auto unavailableMetadata = DocumentSourceMatch::isTextQuery(queryObj)
+        ? DepsTracker::kDefaultUnavailableMetadata & ~DepsTracker::kOnlyTextScore
+        : DepsTracker::kDefaultUnavailableMetadata;
 
     // Create the PlanExecutor.
     bool shouldProduceEmptyDocs = false;
@@ -554,7 +554,7 @@ PipelineD::buildInnerQueryExecutorGeneric(Collection* collection,
                                                 pipeline,
                                                 sortStage,
                                                 std::move(rewrittenGroupStage),
-                                                metadataAvailable,
+                                                unavailableMetadata,
                                                 queryObj,
                                                 limit,
                                                 aggRequest,
@@ -604,18 +604,19 @@ PipelineD::buildInnerQueryExecutorGeoNear(Collection* collection,
     BSONObj fullQuery = geoNearStage->asNearQuery(nearFieldName);
 
     bool shouldProduceEmptyDocs = false;
-    auto exec = uassertStatusOK(prepareExecutor(expCtx,
-                                                collection,
-                                                nss,
-                                                pipeline,
-                                                nullptr, /* sortStage */
-                                                nullptr, /* rewrittenGroupStage */
-                                                DepsTracker::kAllGeoNearData,
-                                                std::move(fullQuery),
-                                                boost::none, /* limit */
-                                                aggRequest,
-                                                Pipeline::kGeoNearMatcherFeatures,
-                                                &shouldProduceEmptyDocs));
+    auto exec = uassertStatusOK(
+        prepareExecutor(expCtx,
+                        collection,
+                        nss,
+                        pipeline,
+                        nullptr, /* sortStage */
+                        nullptr, /* rewrittenGroupStage */
+                        DepsTracker::kDefaultUnavailableMetadata & ~DepsTracker::kAllGeoNearData,
+                        std::move(fullQuery),
+                        boost::none, /* limit */
+                        aggRequest,
+                        Pipeline::kGeoNearMatcherFeatures,
+                        &shouldProduceEmptyDocs));
 
     auto attachExecutorCallback = [shouldProduceEmptyDocs,
                                    distanceField = geoNearStage->getDistanceField(),
@@ -645,7 +646,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> PipelineD::prep
     Pipeline* pipeline,
     const boost::intrusive_ptr<DocumentSourceSort>& sortStage,
     std::unique_ptr<GroupFromFirstDocumentTransformation> rewrittenGroupStage,
-    QueryMetadataBitSet metadataAvailable,
+    QueryMetadataBitSet unavailableMetadata,
     const BSONObj& queryObj,
     boost::optional<long long> limit,
     const AggregationRequest* aggRequest,
@@ -687,7 +688,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> PipelineD::prep
     // Perform dependency analysis. In order to minimize the dependency set, we only analyze the
     // stages that remain in the pipeline after pushdown. In particular, any dependencies for a
     // $match or $sort pushed down into the query layer will not be reflected here.
-    auto deps = pipeline->getDependencies(metadataAvailable);
+    auto deps = pipeline->getDependencies(unavailableMetadata);
     *hasNoRequirements = deps.hasNoRequirements();
 
     // If we're pushing down a sort, and a merge will be required later, then we need the query
diff --git a/src/mongo/db/pipeline/pipeline_test.cpp b/src/mongo/db/pipeline/pipeline_test.cpp
index 0b4a7d9725d..570e945fb04 100644
--- a/src/mongo/db/pipeline/pipeline_test.cpp
+++ b/src/mongo/db/pipeline/pipeline_test.cpp
@@ -2793,11 +2793,12 @@ using PipelineDependenciesTest = AggregationContextFixture;
 TEST_F(PipelineDependenciesTest, EmptyPipelineShouldRequireWholeDocument) {
     auto pipeline = unittest::assertGet(Pipeline::create({}, getExpCtx()));
 
-    auto depsTracker = pipeline->getDependencies(DepsTracker::kNoMetadata);
+    auto depsTracker = pipeline->getDependencies(DepsTracker::kAllMetadata);
     ASSERT_TRUE(depsTracker.needWholeDocument);
     ASSERT_FALSE(depsTracker.getNeedsMetadata(DocumentMetadataFields::kTextScore));
 
-    depsTracker = pipeline->getDependencies(DepsTracker::kOnlyTextScore);
+    depsTracker =
+        pipeline->getDependencies(DepsTracker::kAllMetadata & ~DepsTracker::kOnlyTextScore);
     ASSERT_TRUE(depsTracker.needWholeDocument);
     ASSERT_TRUE(depsTracker.getNeedsMetadata(DocumentMetadataFields::kTextScore));
 }
@@ -2887,7 +2888,7 @@ TEST_F(PipelineDependenciesTest, ShouldRequireWholeDocumentIfAnyStageDoesNotSupp
     auto notSupported = DocumentSourceDependenciesNotSupported::create();
     auto pipeline = unittest::assertGet(Pipeline::create({needsASeeNext, notSupported}, ctx));
 
-    auto depsTracker = pipeline->getDependencies(DepsTracker::kNoMetadata);
+    auto depsTracker = pipeline->getDependencies(DepsTracker::kAllMetadata);
     ASSERT_TRUE(depsTracker.needWholeDocument);
     // The inputs did not have a text score available, so we should not require a text score.
     ASSERT_FALSE(depsTracker.getNeedsMetadata(DocumentMetadataFields::kTextScore));
@@ -2895,7 +2896,7 @@ TEST_F(PipelineDependenciesTest, ShouldRequireWholeDocumentIfAnyStageDoesNotSupp
     // Now in the other order.
     pipeline = unittest::assertGet(Pipeline::create({notSupported, needsASeeNext}, ctx));
 
-    depsTracker = pipeline->getDependencies(DepsTracker::kNoMetadata);
+    depsTracker = pipeline->getDependencies(DepsTracker::kAllMetadata);
     ASSERT_TRUE(depsTracker.needWholeDocument);
 }
 
@@ -2927,7 +2928,7 @@ TEST_F(PipelineDependenciesTest, ShouldNotAddAnyRequiredFieldsAfterFirstStageWit
     auto needsASeeNext = DocumentSourceNeedsASeeNext::create();
     auto pipeline = unittest::assertGet(Pipeline::create({needsOnlyB, needsASeeNext}, ctx));
 
-    auto depsTracker = pipeline->getDependencies(DepsTracker::kNoMetadata);
+    auto depsTracker = pipeline->getDependencies(DepsTracker::kAllMetadata);
     ASSERT_FALSE(depsTracker.needWholeDocument);
     ASSERT_FALSE(depsTracker.getNeedsMetadata(DocumentMetadataFields::kTextScore));
 
@@ -2941,7 +2942,7 @@ TEST_F(PipelineDependenciesTest, ShouldNotRequireTextScoreIfThereIsNoScoreAvaila
     auto ctx = getExpCtx();
     auto pipeline = unittest::assertGet(Pipeline::create({}, ctx));
 
-    auto depsTracker = pipeline->getDependencies(DepsTracker::kNoMetadata);
+    auto depsTracker = pipeline->getDependencies(DepsTracker::kAllMetadata);
     ASSERT_FALSE(depsTracker.getNeedsMetadata(DocumentMetadataFields::kTextScore));
 }
 
@@ -2950,19 +2951,21 @@ TEST_F(PipelineDependenciesTest, ShouldThrowIfTextScoreIsNeededButNotPresent) {
     auto needsText = DocumentSourceNeedsOnlyTextScore::create();
     auto pipeline = unittest::assertGet(Pipeline::create({needsText}, ctx));
 
-    ASSERT_THROWS(pipeline->getDependencies(DepsTracker::kNoMetadata), AssertionException);
+    ASSERT_THROWS(pipeline->getDependencies(DepsTracker::kAllMetadata), AssertionException);
 }
 
 TEST_F(PipelineDependenciesTest, ShouldRequireTextScoreIfAvailableAndNoStageReturnsExhaustiveMeta) {
     auto ctx = getExpCtx();
     auto pipeline = unittest::assertGet(Pipeline::create({}, ctx));
 
-    auto depsTracker = pipeline->getDependencies(DepsTracker::kOnlyTextScore);
+    auto depsTracker =
+        pipeline->getDependencies(DepsTracker::kAllMetadata & ~DepsTracker::kOnlyTextScore);
     ASSERT_TRUE(depsTracker.getNeedsMetadata(DocumentMetadataFields::kTextScore));
 
     auto needsASeeNext = DocumentSourceNeedsASeeNext::create();
     pipeline = unittest::assertGet(Pipeline::create({needsASeeNext}, ctx));
-    depsTracker = pipeline->getDependencies(DepsTracker::kOnlyTextScore);
+    depsTracker =
+        pipeline->getDependencies(DepsTracker::kAllMetadata & ~DepsTracker::kOnlyTextScore);
     ASSERT_TRUE(depsTracker.getNeedsMetadata(DocumentMetadataFields::kTextScore));
 }
 
@@ -2972,7 +2975,8 @@ TEST_F(PipelineDependenciesTest, ShouldNotRequireTextScoreIfAvailableButDefinite
     auto needsText = DocumentSourceNeedsOnlyTextScore::create();
     auto pipeline = unittest::assertGet(Pipeline::create({stripsTextScore, needsText}, ctx));
 
-    auto depsTracker = pipeline->getDependencies(DepsTracker::kOnlyTextScore);
+    auto depsTracker =
+        pipeline->getDependencies(DepsTracker::kAllMetadata & ~DepsTracker::kOnlyTextScore);
 
     // 'stripsTextScore' claims that no further stage will need metadata information, so we
     // shouldn't have the text score as a dependency.
diff --git a/src/mongo/db/query/planner_access.cpp b/src/mongo/db/query/planner_access.cpp
index ace8c86c703..736bf7021e9 100644
--- a/src/mongo/db/query/planner_access.cpp
+++ b/src/mongo/db/query/planner_access.cpp
@@ -321,7 +321,8 @@ std::unique_ptr<QuerySolutionNode> QueryPlannerAccess::makeLeafNode(
         // because expr might be inside an array operator that provides a path prefix.
         auto isn = std::make_unique<IndexScanNode>(index);
         isn->bounds.fields.resize(index.keyPattern.nFields());
-        isn->addKeyMetadata = query.getQueryRequest().returnKey();
+        isn->addKeyMetadata = query.getQueryRequest().returnKey() ||
+            query.metadataDeps()[DocumentMetadataFields::kIndexKey];
         isn->queryCollator = query.getCollator();
 
         // Get the ixtag->pos-th element of the index key pattern.
@@ -1340,7 +1341,8 @@ std::unique_ptr<QuerySolutionNode> QueryPlannerAccess::scanWholeIndex(
 
     // Build an ixscan over the id index, use it, and return it.
     unique_ptr<IndexScanNode> isn = std::make_unique<IndexScanNode>(index);
-    isn->addKeyMetadata = query.getQueryRequest().returnKey();
+    isn->addKeyMetadata = query.getQueryRequest().returnKey() ||
+        query.metadataDeps()[DocumentMetadataFields::kIndexKey];
     isn->queryCollator = query.getCollator();
 
     IndexBoundsBuilder::allValuesBounds(index.keyPattern, &isn->bounds);
@@ -1472,7 +1474,8 @@ std::unique_ptr<QuerySolutionNode> QueryPlannerAccess::makeIndexScan(
     // Build an ixscan over the id index, use it, and return it.
     auto isn = std::make_unique<IndexScanNode>(index);
     isn->direction = 1;
-    isn->addKeyMetadata = query.getQueryRequest().returnKey();
+    isn->addKeyMetadata = query.getQueryRequest().returnKey() ||
+        query.metadataDeps()[DocumentMetadataFields::kIndexKey];
     isn->bounds.isSimpleRange = true;
     isn->bounds.startKey = startKey;
     isn->bounds.endKey = endKey;
diff --git a/src/mongo/db/query/projection.cpp b/src/mongo/db/query/projection.cpp
index 37921ec515e..17c0be336d0 100644
--- a/src/mongo/db/query/projection.cpp
+++ b/src/mongo/db/query/projection.cpp
@@ -43,7 +43,7 @@ namespace {
  * context.
  */
 struct DepsAnalysisData {
-    DepsTracker fieldDependencyTracker{DepsTracker::kAllMetadata};
+    DepsTracker fieldDependencyTracker;
 
     void addRequiredField(const std::string& fieldName) {
         fieldDependencyTracker.fields.insert(fieldName);
diff --git a/src/mongo/s/query/cluster_aggregation_planner.cpp b/src/mongo/s/query/cluster_aggregation_planner.cpp
index 19e24235923..5e3f2e38d55 100644
--- a/src/mongo/s/query/cluster_aggregation_planner.cpp
+++ b/src/mongo/s/query/cluster_aggregation_planner.cpp
@@ -207,7 +207,7 @@ void propagateDocLimitToShards(Pipeline* shardPipe, Pipeline* mergePipe) {
  * Documents.
  */
 void limitFieldsSentFromShardsToMerger(Pipeline* shardPipe, Pipeline* mergePipe) {
-    DepsTracker mergeDeps(mergePipe->getDependencies(DepsTracker::kAllMetadata));
+    DepsTracker mergeDeps(mergePipe->getDependencies(DepsTracker::kNoMetadata));
     if (mergeDeps.needWholeDocument)
         return;  // the merge needs all fields, so nothing we can do.
 
@@ -225,7 +225,7 @@ void limitFieldsSentFromShardsToMerger(Pipeline* shardPipe, Pipeline* mergePipe)
     // 2) Optimization IS NOT applied immediately following a $project or $group since it would
     //    add an unnecessary project (and therefore a deep-copy).
     for (auto&& source : shardPipe->getSources()) {
-        DepsTracker dt(DepsTracker::kAllMetadata);
+        DepsTracker dt(DepsTracker::kNoMetadata);
         if (source->getDependencies(&dt) & DepsTracker::State::EXHAUSTIVE_FIELDS)
             return;
     }