SERVER-26833 Non-blocking text queries when projection ignores score.

With this change, text queries use the non-blocking OR stage in place
of the blocking TEXT_OR stage when it is not necessary to compute the
text score (because the projection does not call for it).

We also removed the unnecessary MatchableTextDocument object with this
change. This object was used in the TEXT_OR stage to apply a filter to
index entries. The query planner adds search predicates as a filter in
the OR/TEXT_OR stage when they can be covered by the index, allowing
them to get filtered out before the full document need be examined

However, the OR stage uses an IndexMatchableDocument which does almost
the same thing. The only difference is that TextMatchableDocument will
fetch documents if the index does not cover the filter. Since that
should never happen (if the query planner is doing its job right), we
shouldn't need TextMatchableDocument.

10 files changed, 91 insertions, 136 deletions

diff --git a/src/mongo/db/exec/or.cpp b/src/mongo/db/exec/or.cpp
index 224a7e47112..329636ce977 100644
--- a/src/mongo/db/exec/or.cpp
+++ b/src/mongo/db/exec/or.cpp
@@ -50,6 +50,12 @@ void OrStage::addChild(PlanStage* child) {
     _children.emplace_back(child);
 }
 
+void OrStage::addChildren(Children childrenToAdd) {
+    _children.insert(_children.end(),
+                     std::make_move_iterator(childrenToAdd.begin()),
+                     std::make_move_iterator(childrenToAdd.end()));
+}
+
 bool OrStage::isEOF() {
     return _currentChild >= _children.size();
 }
diff --git a/src/mongo/db/exec/or.h b/src/mongo/db/exec/or.h
index c97f9d34909..84ded543c82 100644
--- a/src/mongo/db/exec/or.h
+++ b/src/mongo/db/exec/or.h
@@ -49,6 +49,8 @@ public:
 
     void addChild(PlanStage* child);
 
+    void addChildren(Children childrenToAdd);
+
     bool isEOF() final;
 
     StageState doWork(WorkingSetID* out) final;
diff --git a/src/mongo/db/exec/text.cpp b/src/mongo/db/exec/text.cpp
index f698dbd0dc8..726f015bb3e 100644
--- a/src/mongo/db/exec/text.cpp
+++ b/src/mongo/db/exec/text.cpp
@@ -30,8 +30,10 @@
 
 #include <vector>
 
+#include "mongo/db/exec/fetch.h"
 #include "mongo/db/exec/filter.h"
 #include "mongo/db/exec/index_scan.h"
+#include "mongo/db/exec/or.h"
 #include "mongo/db/exec/scoped_timer.h"
 #include "mongo/db/exec/text_match.h"
 #include "mongo/db/exec/text_or.h"
@@ -60,7 +62,7 @@ TextStage::TextStage(OperationContext* opCtx,
                      WorkingSet* ws,
                      const MatchExpression* filter)
     : PlanStage(kStageType, opCtx), _params(params) {
-    _children.emplace_back(buildTextTree(opCtx, ws, filter));
+    _children.emplace_back(buildTextTree(opCtx, ws, filter, params.wantTextScore));
     _specificStats.indexPrefix = _params.indexPrefix;
     _specificStats.indexName = _params.index->indexName();
     _specificStats.parsedTextQuery = _params.query.toBSON();
@@ -94,10 +96,10 @@ const SpecificStats* TextStage::getSpecificStats() const {
 
 unique_ptr<PlanStage> TextStage::buildTextTree(OperationContext* opCtx,
                                                WorkingSet* ws,
-                                               const MatchExpression* filter) const {
-    auto textScorer = make_unique<TextOrStage>(opCtx, _params.spec, ws, filter, _params.index);
-
+                                               const MatchExpression* filter,
+                                               bool wantTextScore) const {
     // Get all the index scans for each term in our query.
+    std::vector<std::unique_ptr<PlanStage>> indexScanList;
     for (const auto& term : _params.query.getTermsForBounds()) {
         IndexScanParams ixparams;
 
@@ -110,14 +112,40 @@ unique_ptr<PlanStage> TextStage::buildTextTree(OperationContext* opCtx,
         ixparams.descriptor = _params.index;
         ixparams.direction = -1;
 
-        textScorer->addChild(make_unique<IndexScan>(opCtx, ixparams, ws, nullptr));
+        indexScanList.push_back(stdx::make_unique<IndexScan>(opCtx, ixparams, ws, nullptr));
     }
 
-    auto matcher =
-        make_unique<TextMatchStage>(opCtx, std::move(textScorer), _params.query, _params.spec, ws);
+    // Build the union of the index scans as a TEXT_OR or an OR stage, depending on whether the
+    // projection requires the "textScore" $meta field.
+    std::unique_ptr<PlanStage> textMatchStage;
+    if (wantTextScore) {
+        // We use a TEXT_OR stage to get the union of the results from the index scans and then
+        // compute their text scores. This is a blocking operation.
+        auto textScorer = make_unique<TextOrStage>(opCtx, _params.spec, ws, filter, _params.index);
+
+        textScorer->addChildren(std::move(indexScanList));
+
+        textMatchStage = make_unique<TextMatchStage>(
+            opCtx, std::move(textScorer), _params.query, _params.spec, ws);
+    } else {
+        // Because we don't need the text score, we can use a non-blocking OR stage to get the union
+        // of the index scans.
+        auto textSearcher = make_unique<OrStage>(opCtx, ws, true, filter);
+
+        textSearcher->addChildren(std::move(indexScanList));
+
+        // Unlike the TEXT_OR stage, the OR stage does not fetch the documents that it outputs. We
+        // add our own FETCH stage to satisfy the requirement of the TEXT_MATCH stage that its
+        // WorkingSetMember inputs have fetched data.
+        const MatchExpression* emptyFilter = nullptr;
+        auto fetchStage = make_unique<FetchStage>(
+            opCtx, ws, textSearcher.release(), emptyFilter, _params.index->getCollection());
+
+        textMatchStage = make_unique<TextMatchStage>(
+            opCtx, std::move(fetchStage), _params.query, _params.spec, ws);
+    }
 
-    unique_ptr<PlanStage> treeRoot = std::move(matcher);
-    return treeRoot;
+    return textMatchStage;
 }
 
 }  // namespace mongo
diff --git a/src/mongo/db/exec/text.h b/src/mongo/db/exec/text.h
index b488181ca0a..6e81d27e73c 100644
--- a/src/mongo/db/exec/text.h
+++ b/src/mongo/db/exec/text.h
@@ -63,6 +63,10 @@ struct TextStageParams {
 
     // The text query.
     FTSQueryImpl query;
+
+    // True if we need the text score in the output, because the projection includes the 'textScore'
+    // metadata field.
+    bool wantTextScore = true;
 };
 
 /**
@@ -77,7 +81,6 @@ public:
               WorkingSet* ws,
               const MatchExpression* filter);
 
-
     StageState doWork(WorkingSetID* out) final;
     bool isEOF() final;
 
@@ -97,7 +100,8 @@ private:
      */
     unique_ptr<PlanStage> buildTextTree(OperationContext* opCtx,
                                         WorkingSet* ws,
-                                        const MatchExpression* filter) const;
+                                        const MatchExpression* filter,
+                                        bool wantTextScore) const;
 
     // Parameters of this text stage.
     TextStageParams _params;
diff --git a/src/mongo/db/exec/text_match.h b/src/mongo/db/exec/text_match.h
index 8fff04e4d54..5a8644d1b6e 100644
--- a/src/mongo/db/exec/text_match.h
+++ b/src/mongo/db/exec/text_match.h
@@ -52,8 +52,8 @@ class RecordID;
  * A stage that returns every document in the child that satisfies the FTS text matcher built with
  * the query parameter.
  *
- * Prerequisites: A single child stage that passes up WorkingSetMembers in the LOC_AND_OBJ state,
- * with associated text scores.
+ * Prerequisites: A single child stage that passes up WorkingSetMembers in the RID_AND_OBJ state.
+ * Members must also have text score metadata if it is necessary for the final projection.
  */
 class TextMatchStage final : public PlanStage {
 public:
diff --git a/src/mongo/db/exec/text_or.cpp b/src/mongo/db/exec/text_or.cpp
index bea23d39e34..d7152e6e48c 100644
--- a/src/mongo/db/exec/text_or.cpp
+++ b/src/mongo/db/exec/text_or.cpp
@@ -32,14 +32,13 @@
 #include <vector>
 
 #include "mongo/db/concurrency/write_conflict_exception.h"
+#include "mongo/db/exec/filter.h"
 #include "mongo/db/exec/index_scan.h"
 #include "mongo/db/exec/scoped_timer.h"
 #include "mongo/db/exec/working_set.h"
 #include "mongo/db/exec/working_set_common.h"
 #include "mongo/db/exec/working_set_computed_data.h"
 #include "mongo/db/jsobj.h"
-#include "mongo/db/matcher/matchable.h"
-#include "mongo/db/query/internal_plans.h"
 #include "mongo/db/record_id.h"
 #include "mongo/stdx/memory.h"
 
@@ -73,6 +72,12 @@ void TextOrStage::addChild(unique_ptr<PlanStage> child) {
     _children.push_back(std::move(child));
 }
 
+void TextOrStage::addChildren(Children childrenToAdd) {
+    _children.insert(_children.end(),
+                     std::make_move_iterator(childrenToAdd.begin()),
+                     std::make_move_iterator(childrenToAdd.end()));
+}
+
 bool TextOrStage::isEOF() {
     return _internalState == State::kDone;
 }
@@ -251,81 +256,6 @@ PlanStage::StageState TextOrStage::returnResults(WorkingSetID* out) {
     return PlanStage::ADVANCED;
 }
 
-/**
- * Provides support for covered matching on non-text fields of a compound text index.
- */
-class TextMatchableDocument : public MatchableDocument {
-public:
-    TextMatchableDocument(OperationContext* opCtx,
-                          const BSONObj& keyPattern,
-                          const BSONObj& key,
-                          WorkingSet* ws,
-                          WorkingSetID id,
-                          unowned_ptr<SeekableRecordCursor> recordCursor)
-        : _opCtx(opCtx),
-          _recordCursor(recordCursor),
-          _keyPattern(keyPattern),
-          _key(key),
-          _ws(ws),
-          _id(id) {}
-
-    BSONObj toBSON() const {
-        return getObj();
-    }
-
-    ElementIterator* allocateIterator(const ElementPath* path) const final {
-        WorkingSetMember* member = _ws->get(_id);
-        if (!member->hasObj()) {
-            // Try to look in the key.
-            BSONObjIterator keyPatternIt(_keyPattern);
-            BSONObjIterator keyDataIt(_key);
-
-            while (keyPatternIt.more()) {
-                BSONElement keyPatternElt = keyPatternIt.next();
-                verify(keyDataIt.more());
-                BSONElement keyDataElt = keyDataIt.next();
-
-                if (path->fieldRef().equalsDottedField(keyPatternElt.fieldName())) {
-                    if (Array == keyDataElt.type()) {
-                        return new SimpleArrayElementIterator(keyDataElt, true);
-                    } else {
-                        return new SingleElementElementIterator(keyDataElt);
-                    }
-                }
-            }
-        }
-
-        // Go to the raw document, fetching if needed.
-        return new BSONElementIterator(path, getObj());
-    }
-
-    void releaseIterator(ElementIterator* iterator) const final {
-        delete iterator;
-    }
-
-    // Thrown if we detect that the document being matched was deleted.
-    class DocumentDeletedException {};
-
-private:
-    BSONObj getObj() const {
-        if (!WorkingSetCommon::fetchIfUnfetched(_opCtx, _ws, _id, _recordCursor))
-            throw DocumentDeletedException();
-
-        WorkingSetMember* member = _ws->get(_id);
-
-        // Make it owned since we are buffering results.
-        member->makeObjOwnedIfNeeded();
-        return member->obj.value();
-    }
-
-    OperationContext* _opCtx;
-    unowned_ptr<SeekableRecordCursor> _recordCursor;
-    BSONObj _keyPattern;
-    BSONObj _key;
-    WorkingSet* _ws;
-    WorkingSetID _id;
-};
-
 PlanStage::StageState TextOrStage::addTerm(WorkingSetID wsid, WorkingSetID* out) {
     WorkingSetMember* wsm = _ws->get(wsid);
     invariant(wsm->getState() == WorkingSetMember::RID_AND_IDX);
@@ -343,57 +273,29 @@ PlanStage::StageState TextOrStage::addTerm(WorkingSetID wsid, WorkingSetID* out)
     if (WorkingSet::INVALID_ID == textRecordData->wsid) {
         // We haven't seen this RecordId before.
         invariant(textRecordData->score == 0);
-        bool shouldKeep = true;
-        if (_filter) {
-            // We have not seen this document before and need to apply a filter.
-            bool wasDeleted = false;
-            try {
-                TextMatchableDocument tdoc(getOpCtx(),
-                                           newKeyData.indexKeyPattern,
-                                           newKeyData.keyData,
-                                           _ws,
-                                           wsid,
-                                           _recordCursor);
-                shouldKeep = _filter->matches(&tdoc);
-            } catch (const WriteConflictException&) {
-                // Ensure that the BSONObj underlying the WorkingSetMember is owned because it may
-                // be freed when we yield.
-                wsm->makeObjOwnedIfNeeded();
-                _idRetrying = wsid;
-                *out = WorkingSet::INVALID_ID;
-                return NEED_YIELD;
-            } catch (const TextMatchableDocument::DocumentDeletedException&) {
-                // We attempted to fetch the document but decided it should be excluded from the
-                // result set.
-                shouldKeep = false;
-                wasDeleted = true;
-            }
-
-            if (wasDeleted || wsm->hasObj()) {
-                ++_specificStats.fetches;
-            }
-        }
-
-        if (shouldKeep && !wsm->hasObj()) {
-            // Our parent expects RID_AND_OBJ members, so we fetch the document here if we haven't
-            // already.
-            try {
-                shouldKeep = WorkingSetCommon::fetch(getOpCtx(), _ws, wsid, _recordCursor);
-                ++_specificStats.fetches;
-            } catch (const WriteConflictException&) {
-                wsm->makeObjOwnedIfNeeded();
-                _idRetrying = wsid;
-                *out = WorkingSet::INVALID_ID;
-                return NEED_YIELD;
-            }
-        }
 
-        if (!shouldKeep) {
+        if (!Filter::passes(newKeyData.keyData, newKeyData.indexKeyPattern, _filter)) {
             _ws->free(wsid);
             textRecordData->score = -1;
             return NEED_TIME;
         }
 
+        // Our parent expects RID_AND_OBJ members, so we fetch the document here if we haven't
+        // already.
+        try {
+            if (!WorkingSetCommon::fetch(getOpCtx(), _ws, wsid, _recordCursor)) {
+                _ws->free(wsid);
+                textRecordData->score = -1;
+                return NEED_TIME;
+            }
+            ++_specificStats.fetches;
+        } catch (const WriteConflictException&) {
+            wsm->makeObjOwnedIfNeeded();
+            _idRetrying = wsid;
+            *out = WorkingSet::INVALID_ID;
+            return NEED_YIELD;
+        }
+
         textRecordData->wsid = wsid;
 
         // Ensure that the BSONObj underlying the WorkingSetMember is owned in case we yield.
diff --git a/src/mongo/db/exec/text_or.h b/src/mongo/db/exec/text_or.h
index b40c069cc18..a705d53cfef 100644
--- a/src/mongo/db/exec/text_or.h
+++ b/src/mongo/db/exec/text_or.h
@@ -81,6 +81,8 @@ public:
 
     void addChild(unique_ptr<PlanStage> child);
 
+    void addChildren(Children childrenToAdd);
+
     bool isEOF() final;
 
     StageState doWork(WorkingSetID* out) final;
diff --git a/src/mongo/db/query/parsed_projection.cpp b/src/mongo/db/query/parsed_projection.cpp
index 53e5f5a523e..fa2b10f4f8f 100644
--- a/src/mongo/db/query/parsed_projection.cpp
+++ b/src/mongo/db/query/parsed_projection.cpp
@@ -57,6 +57,7 @@ Status ParsedProjection::make(OperationContext* opCtx,
     bool requiresDocument = false;
     bool hasIndexKeyProjection = false;
 
+    bool wantTextScore = false;
     bool wantGeoNearPoint = false;
     bool wantGeoNearDistance = false;
     bool wantSortKey = false;
@@ -167,7 +168,9 @@ Status ParsedProjection::make(OperationContext* opCtx,
                 }
 
                 // This clobbers everything else.
-                if (e2.valuestr() == QueryRequest::metaIndexKey) {
+                if (e2.valuestr() == QueryRequest::metaTextScore) {
+                    wantTextScore = true;
+                } else if (e2.valuestr() == QueryRequest::metaIndexKey) {
                     hasIndexKeyProjection = true;
                 } else if (e2.valuestr() == QueryRequest::metaGeoNearDistance) {
                     wantGeoNearDistance = true;
@@ -268,6 +271,7 @@ Status ParsedProjection::make(OperationContext* opCtx,
     pp->_requiresDocument = requiresDocument;
 
     // Add meta-projections.
+    pp->_wantTextScore = wantTextScore;
     pp->_wantGeoNearPoint = wantGeoNearPoint;
     pp->_wantGeoNearDistance = wantGeoNearDistance;
     pp->_wantSortKey = wantSortKey;
diff --git a/src/mongo/db/query/parsed_projection.h b/src/mongo/db/query/parsed_projection.h
index 82544e82c47..cbf7ad6903b 100644
--- a/src/mongo/db/query/parsed_projection.h
+++ b/src/mongo/db/query/parsed_projection.h
@@ -82,6 +82,10 @@ public:
         return _source;
     }
 
+    bool wantTextScore() const {
+        return _wantTextScore;
+    }
+
     /**
      * Does the projection want geoNear metadata?  If so any geoNear stage should include them.
      */
@@ -180,6 +184,8 @@ private:
 
     BSONObj _source;
 
+    bool _wantTextScore = false;
+
     bool _wantGeoNearDistance = false;
 
     bool _wantGeoNearPoint = false;
diff --git a/src/mongo/db/query/stage_builder.cpp b/src/mongo/db/query/stage_builder.cpp
index dcd21b27f35..caf108aa019 100644
--- a/src/mongo/db/query/stage_builder.cpp
+++ b/src/mongo/db/query/stage_builder.cpp
@@ -285,6 +285,7 @@ PlanStage* buildStages(OperationContext* opCtx,
             // planning a query that contains "no-op" expressions. TODO: make StageBuilder::build()
             // fail in this case (this improvement is being tracked by SERVER-21510).
             params.query = static_cast<FTSQueryImpl&>(*node->ftsQuery);
+            params.wantTextScore = (cq.getProj() && cq.getProj()->wantTextScore());
             return new TextStage(opCtx, params, ws, node->filter.get());
         }
         case STAGE_SHARDING_FILTER: {