SERVER-11675 Agg text support

To enable this, Documents now can store "metadata" that gets carried though
the pipeline as long as the result is the same logical document. This includes
crossing the shard-to-merger boundary.

17 files changed, 315 insertions, 50 deletions

diff --git a/src/mongo/db/commands/pipeline_command.cpp b/src/mongo/db/commands/pipeline_command.cpp
index 5813afffb8e..24e52a38ac9 100644
--- a/src/mongo/db/commands/pipeline_command.cpp
+++ b/src/mongo/db/commands/pipeline_command.cpp
@@ -56,6 +56,7 @@ namespace {
     public:
         PipelineRunner(intrusive_ptr<Pipeline> pipeline)
             : _pipeline(pipeline)
+            , _includeMetaData(_pipeline->getContext()->inShard) // send metadata to merger
         {}
 
         virtual RunnerState getNext(BSONObj* objOut, DiskLoc* dlOut) {
@@ -68,8 +69,8 @@ namespace {
                 return RUNNER_ADVANCED;
             }
 
-            if (boost::optional<Document> next = _pipeline->output()->getNext()) {
-                *objOut = next->toBson();
+            if (boost::optional<BSONObj> next = getNextBson()) {
+                *objOut = *next;
                 return RUNNER_ADVANCED;
             }
 
@@ -79,8 +80,8 @@ namespace {
             if (!_stash.empty())
                 return false;
 
-            if (boost::optional<Document> next = _pipeline->output()->getNext()) {
-                _stash.push_back(next->toBson());
+            if (boost::optional<BSONObj> next = getNextBson()) {
+                _stash.push_back(*next);
                 return false;
             }
 
@@ -111,9 +112,23 @@ namespace {
         }
 
     private:
+        boost::optional<BSONObj> getNextBson() {
+            if (boost::optional<Document> next = _pipeline->output()->getNext()) {
+                if (_includeMetaData) {
+                    return next->toBsonWithMetaData();
+                }
+                else {
+                    return next->toBson();
+                }
+            }
+
+            return boost::none;
+        }
+
         // Things in the _stash sould be returned before pulling items from _pipeline.
-        intrusive_ptr<Pipeline> _pipeline;
+        const intrusive_ptr<Pipeline> _pipeline;
         vector<BSONObj> _stash;
+        const bool _includeMetaData;
     };
 }
 
diff --git a/src/mongo/db/pipeline/document.cpp b/src/mongo/db/pipeline/document.cpp
index f7e41fc6606..4f260591fe1 100644
--- a/src/mongo/db/pipeline/document.cpp
+++ b/src/mongo/db/pipeline/document.cpp
@@ -195,6 +195,8 @@ namespace mongo {
         out->_usedBytes = _usedBytes;
         out->_numFields = _numFields;
         out->_hashTabMask = _hashTabMask;
+        out->_hasTextScore = _hasTextScore;
+        out->_textScore = _textScore;
 
         // Tell values that they have been memcpyed (updates ref counts)
         for (DocumentStorageIterator it = out->iteratorAll(); !it.atEnd(); it.advance()) {
@@ -243,6 +245,35 @@ namespace mongo {
         return bb.obj();
     }
 
+    const StringData Document::metaFieldTextScore("$textScore", StringData::LiteralTag());
+
+    BSONObj Document::toBsonWithMetaData() const {
+        BSONObjBuilder bb;
+        toBson(&bb);
+        if (hasTextScore())
+            bb.append(metaFieldTextScore, getTextScore());
+        return bb.obj();
+    }
+
+    Document Document::fromBsonWithMetaData(const BSONObj& bson) {
+        MutableDocument md;
+
+        BSONObjIterator it(bson);
+        while(it.more()) {
+            BSONElement elem(it.next());
+            if (elem.fieldName()[0] == '$') {
+                if (elem.fieldNameStringData() == metaFieldTextScore) {
+                    md.setTextScore(elem.Double());
+                    continue;
+                }
+            }
+
+            // Note: this will not parse out metadata in embedded documents.
+            md.addField(elem.fieldNameStringData(), Value(elem));
+        }
+
+        return md.freeze();
+    }
 
     MutableDocument::MutableDocument(size_t expectedFields)
         : _storageHolder(NULL)
@@ -393,6 +424,14 @@ namespace mongo {
             buf.appendStr(it->nameSD(), /*NUL byte*/ true);
             it->val.serializeForSorter(buf);
         }
+
+        if (hasTextScore()) {
+            buf.appendNum(char(1));
+            buf.appendNum(getTextScore());
+        }
+        else {
+            buf.appendNum(char(0));
+        }
     }
 
     Document Document::deserializeForSorter(BufReader& buf, const SorterDeserializeSettings&) {
@@ -403,6 +442,10 @@ namespace mongo {
             doc.addField(name, Value::deserializeForSorter(buf,
                                                            Value::SorterDeserializeSettings()));
         }
+
+        if (buf.read<char>()) // hasTextScore
+            doc.setTextScore(buf.read<double>());
+
         return doc.freeze();
     }
 }
diff --git a/src/mongo/db/pipeline/document.h b/src/mongo/db/pipeline/document.h
index 7bb79e68db8..7f2fde8a26f 100644
--- a/src/mongo/db/pipeline/document.h
+++ b/src/mongo/db/pipeline/document.h
@@ -115,6 +115,8 @@ namespace mongo {
          *  as strings are compared, but comparing one field at a time instead
          *  of one character at a time.
          *
+         *  Note: This does not consider metadata when comparing documents.
+         *
          *  @returns an integer less than zero, zero, or an integer greater than
          *           zero, depending on whether lhs < rhs, lhs == rhs, or lhs > rhs
          *  Warning: may return values other than -1, 0, or 1
@@ -133,10 +135,26 @@ namespace mongo {
          */
         void hash_combine(size_t &seed) const;
 
-        /// Add this document to the BSONObj under construction with the given BSONObjBuilder.
+        /**
+         * Add this document to the BSONObj under construction with the given BSONObjBuilder.
+         * Does not include metadata.
+         */
         void toBson(BSONObjBuilder *pBsonObjBuilder) const;
         BSONObj toBson() const;
 
+        /**
+         * Like toBson, but includes metadata at the top-level.
+         * Output is parseable by fromBsonWithMetaData
+         */
+        BSONObj toBsonWithMetaData() const;
+
+        /**
+         * Like Document(BSONObj) but treats top-level fields with special names as metadata.
+         * Special field names are available as static constants on this class with names starting
+         * with metaField.
+         */
+        static Document fromBsonWithMetaData(const BSONObj& bson);
+
         // Support BSONObjBuilder and BSONArrayBuilder "stream" API
         friend BSONObjBuilder& operator << (BSONObjBuilderValueStream& builder, const Document& d);
 
@@ -155,6 +173,10 @@ namespace mongo {
          */
         Document clone() const { return Document(storage().clone().get()); }
 
+        static const StringData metaFieldTextScore; // "$textScore"
+        bool hasTextScore() const { return storage().hasTextScore(); }
+        double getTextScore() const { return storage().getTextScore(); }
+
         /// members for Sorter
         struct SorterDeserializeSettings {}; // unused
         void serializeForSorter(BufBuilder& buf) const;
@@ -339,6 +361,16 @@ namespace mongo {
             getNestedField(positions) = val;
         }
 
+        /**
+         * Copies all metadata from source if it has any.
+         * Note: does not clear metadata from this.
+         */
+        void copyMetaDataFrom(const Document& source) {
+            storage().copyMetaDataFrom(source.storage());
+        }
+
+        void setTextScore(double score) { storage().setTextScore(score); }
+
         /** Convert to a read-only document and release reference.
          *
          *  Call this to indicate that you are done with this Document and will
diff --git a/src/mongo/db/pipeline/document_internal.h b/src/mongo/db/pipeline/document_internal.h
index fe92c9f72bf..4ef2a0f0710 100644
--- a/src/mongo/db/pipeline/document_internal.h
+++ b/src/mongo/db/pipeline/document_internal.h
@@ -166,6 +166,8 @@ namespace mongo {
                           , _usedBytes(0)
                           , _numFields(0)
                           , _hashTabMask(0)
+                          , _hasTextScore(false)
+                          , _textScore(0)
         {}
         ~DocumentStorage();
 
@@ -237,6 +239,23 @@ namespace mongo {
             return !_buffer ? 0 : (_bufferEnd - _buffer + hashTabBytes());
         }
 
+        /**
+         * Copies all metadata from source if it has any.
+         * Note: does not clear metadata from this.
+         */
+        void copyMetaDataFrom(const DocumentStorage& source) {
+            if (source.hasTextScore()) {
+                setTextScore(source.getTextScore());
+            }
+        }
+
+        bool hasTextScore() const { return _hasTextScore; }
+        double getTextScore() const { return _textScore; }
+        void setTextScore(double score) {
+            _hasTextScore = true;
+            _textScore = score;
+        }
+
     private:
 
         /// Same as lastElement->next() or firstElement() if empty.
@@ -305,6 +324,9 @@ namespace mongo {
         unsigned _usedBytes; // position where next field would start
         unsigned _numFields; // this includes removed fields
         unsigned _hashTabMask; // equal to hashTabBuckets()-1 but used more often
+
+        bool _hasTextScore; // When adding more metadata fields, this should become a bitvector
+        double _textScore;
         // When adding a field, make sure to update clone() method
     };
 }
diff --git a/src/mongo/db/pipeline/document_source.cpp b/src/mongo/db/pipeline/document_source.cpp
index 148bf4134d2..f07739479e3 100644
--- a/src/mongo/db/pipeline/document_source.cpp
+++ b/src/mongo/db/pipeline/document_source.cpp
@@ -84,6 +84,15 @@ namespace mongo {
                 needId = true;
                 continue;
             }
+
+            if (str::startsWith(*it, '$')) {
+                if (*it == "$textScore") {
+                    // textScore metadata
+                    bb.append(Document::metaFieldTextScore, BSON("$meta" << "textScore"));
+                    continue;
+                }
+            }
+
             if (!last.empty() && str::startsWith(*it, last)) {
                 // we are including a parent of *it so we don't need to include this field
                 // explicitly. In fact, due to SERVER-6527 if we included this field, the parent
@@ -92,6 +101,7 @@ namespace mongo {
                 // prefixes.
                 continue;
             }
+
             last = *it + '.';
             bb.append(*it, 1);
         }
@@ -121,6 +131,12 @@ namespace mongo {
 
         string last;
         for (set<string>::const_iterator it(deps.begin()), end(deps.end()); it!=end; ++it) {
+            if (str::startsWith(*it, '$')) {
+                // documentFromBsonWithDeps doesn't handle meta data
+                if (*it == "$textScore")
+                    return ParsedDeps();
+            }
+
             if (!last.empty() && str::startsWith(*it, last)) {
                 // we are including a parent of *it so we don't need to include this field
                 // explicitly. In fact, if we included this field, the parent wouldn't be fully
diff --git a/src/mongo/db/pipeline/document_source.h b/src/mongo/db/pipeline/document_source.h
index f3b282182a0..604b3117a16 100644
--- a/src/mongo/db/pipeline/document_source.h
+++ b/src/mongo/db/pipeline/document_source.h
@@ -166,7 +166,7 @@ namespace mongo {
 
         /// Returns true if doesn't require an input source (most DocumentSources do).
         virtual bool isValidInitialSource() const { return false; }
-        
+
     protected:
         /**
            Base constructor.
@@ -403,7 +403,17 @@ namespace mongo {
          */
         void setSort(const BSONObj& sort) { _sort = sort; }
 
-        void setProjection(const BSONObj& projection, const ParsedDeps& deps);
+        /**
+         * Informs this object of projection and dependency information.
+         *
+         * @param projection A projection specification describing the fields needed by the rest of
+         *                   the pipeline.
+         * @param deps The output of DocumentSource::parseDeps.
+         * @param projectionInQuery True if the underlying cursor will handle the projection for us.
+         */
+        void setProjection(const BSONObj& projection,
+                           const ParsedDeps& deps,
+                           bool projectionInQuery);
 
         /// returns -1 for no limit
         long long getLimit() const;
@@ -424,6 +434,7 @@ namespace mongo {
         BSONObj _projection;
         bool _haveDeps;
         ParsedDeps _dependencies;
+        bool _projectionInQuery;
         intrusive_ptr<DocumentSourceLimit> _limit;
         long long _docsAddedToBatches; // for _limit enforcement
 
@@ -570,6 +581,7 @@ namespace mongo {
         virtual const char *getSourceName() const;
         virtual bool coalesce(const intrusive_ptr<DocumentSource>& nextSource);
         virtual Value serialize(bool explain = false) const;
+        virtual void setSource(DocumentSource* Source);
 
         /**
           Create a filter.
@@ -598,11 +610,15 @@ namespace mongo {
          */
         BSONObj redactSafePortion() const;
 
+        static bool isTextQuery(const BSONObj& query);
+        bool isTextQuery() const { return _isTextQuery; }
+
     private:
         DocumentSourceMatch(const BSONObj &query,
             const intrusive_ptr<ExpressionContext> &pExpCtx);
 
         scoped_ptr<Matcher> matcher;
+        bool _isTextQuery;
     };
 
     class DocumentSourceMergeCursors :
@@ -811,7 +827,7 @@ namespace mongo {
         void addKey(const string &fieldPath, bool ascending);
 
         /// Write out a Document whose contents are the sort key.
-        Document serializeSortKey() const;
+        Document serializeSortKey(bool explain) const;
 
         /**
           Create a sorting DocumentSource from BSON.
@@ -868,8 +884,8 @@ namespace mongo {
         void populateFromBsonArrays(const vector<BSONArray>& arrays);
 
         /* these two parallel each other */
-        typedef vector<intrusive_ptr<ExpressionFieldPath> > SortPaths;
-        SortPaths vSortKey;
+        typedef vector<intrusive_ptr<Expression> > SortKey;
+        SortKey vSortKey;
         vector<char> vAscending; // used like vector<bool> but without specialization
 
         /// Extracts the fields in vSortKey from the Document;
@@ -1056,7 +1072,7 @@ namespace mongo {
         // virtuals from DocumentSource
         virtual boost::optional<Document> getNext();
         virtual const char *getSourceName() const;
-        virtual void setSource(DocumentSource *pSource); // errors out since this must be first
+        virtual void setSource(DocumentSource *pSource);
         virtual bool coalesce(const intrusive_ptr<DocumentSource> &pNextSource);
         virtual bool isValidInitialSource() const { return true; }
         virtual Value serialize(bool explain = false) const;
diff --git a/src/mongo/db/pipeline/document_source_cursor.cpp b/src/mongo/db/pipeline/document_source_cursor.cpp
index bba5ab951fe..2235ce455d9 100644
--- a/src/mongo/db/pipeline/document_source_cursor.cpp
+++ b/src/mongo/db/pipeline/document_source_cursor.cpp
@@ -97,11 +97,11 @@ namespace mongo {
         BSONObj obj;
         Runner::RunnerState state;
         while ((state = runner->getNext(&obj, NULL)) == Runner::RUNNER_ADVANCED) {
-            if (_haveDeps) {
+            if (_haveDeps && !_projectionInQuery) {
                 _currentBatch.push_back(documentFromBsonWithDeps(obj, _dependencies));
             }
             else {
-                _currentBatch.push_back(Document(obj));
+                _currentBatch.push_back(Document::fromBsonWithMetaData(obj));
             }
 
             if (_limit) {
@@ -268,9 +268,13 @@ namespace {
         return new DocumentSourceCursor(ns, cursorId, pExpCtx);
     }
 
-    void DocumentSourceCursor::setProjection(const BSONObj& projection, const ParsedDeps& deps) {
+    void DocumentSourceCursor::setProjection(
+            const BSONObj& projection,
+            const ParsedDeps& deps,
+            bool projectionInQuery) {
         _projection = projection;
         _dependencies = deps;
+        _projectionInQuery = projectionInQuery;
         _haveDeps = true;
     }
 }
diff --git a/src/mongo/db/pipeline/document_source_match.cpp b/src/mongo/db/pipeline/document_source_match.cpp
index ce364064e77..5d3525181e0 100644
--- a/src/mongo/db/pipeline/document_source_match.cpp
+++ b/src/mongo/db/pipeline/document_source_match.cpp
@@ -52,6 +52,10 @@ namespace mongo {
     boost::optional<Document> DocumentSourceMatch::getNext() {
         pExpCtx->checkForInterrupt();
 
+        // The user facing error should have been generated earlier.
+        massert(17309, "Should never call getNext on a $match stage with $text clause",
+                !_isTextQuery);
+
         while (boost::optional<Document> next = pSource->getNext()) {
             // The matcher only takes BSON documents, so we have to make one.
             if (matcher->matches(next->toBson()))
@@ -67,6 +71,22 @@ namespace mongo {
         if (!otherMatch)
             return false;
 
+        if (otherMatch->_isTextQuery) {
+            // Non-initial text queries are disallowed (enforced by setSource below). This prevents
+            // "hiding" a non-initial text query by combining it with another match.
+            return false;
+
+            // The rest of this block is for once we support non-initial text queries.
+
+            if (_isTextQuery) {
+                // The score should only come from the last $match. We can't combine since then this
+                // match's score would impact otherMatch's.
+                return false;
+            }
+
+            _isTextQuery = true;
+        }
+
         // Replace our matcher with the $and of ours and theirs.
         matcher.reset(new Matcher(BSON("$and" << BSON_ARRAY(getQuery()
                                                          << otherMatch->getQuery()))));
@@ -271,6 +291,25 @@ namespace {
         return redactSafePortionTopLevel(getQuery()).toBson();
     }
 
+    void DocumentSourceMatch::setSource(DocumentSource* source) {
+        uassert(17313, "$match with $text is only allowed as the first pipeline stage",
+                !_isTextQuery);
+
+        DocumentSource::setSource(source);
+    }
+
+    bool DocumentSourceMatch::isTextQuery(const BSONObj& query) {
+        BSONForEach(e, query) {
+            const StringData fieldName = e.fieldNameStringData();
+            if (fieldName == StringData("$text", StringData::LiteralTag()))
+                return true;
+
+            if (e.isABSONObj() && isTextQuery(e.Obj()))
+                return true;
+        }
+        return false;
+    }
+
     static void uassertNoDisallowedClauses(BSONObj query) {
         BSONForEach(e, query) {
             // can't use the Matcher API because this would segfault the constructor
@@ -305,5 +344,6 @@ namespace {
                                              const intrusive_ptr<ExpressionContext> &pExpCtx)
         : DocumentSource(pExpCtx)
         , matcher(new Matcher(query.getOwned()))
+        , _isTextQuery(isTextQuery(query))
     {}
 }
diff --git a/src/mongo/db/pipeline/document_source_merge_cursors.cpp b/src/mongo/db/pipeline/document_source_merge_cursors.cpp
index fff86778e0a..c7cb53eccf1 100644
--- a/src/mongo/db/pipeline/document_source_merge_cursors.cpp
+++ b/src/mongo/db/pipeline/document_source_merge_cursors.cpp
@@ -156,7 +156,7 @@ namespace mongo {
         if (++_currentCursor == _cursors.end())
             _currentCursor = _cursors.begin();
 
-        return Document(next);
+        return Document::fromBsonWithMetaData(next);
     }
 
     void DocumentSourceMergeCursors::dispose() {
diff --git a/src/mongo/db/pipeline/document_source_project.cpp b/src/mongo/db/pipeline/document_source_project.cpp
index ff1a3a05a27..c6708524f45 100644
--- a/src/mongo/db/pipeline/document_source_project.cpp
+++ b/src/mongo/db/pipeline/document_source_project.cpp
@@ -58,6 +58,7 @@ namespace mongo {
         /* create the result document */
         const size_t sizeHint = pEO->getSizeHint();
         MutableDocument out (sizeHint);
+        out.copyMetaDataFrom(*input);
 
         /*
           Use the ExpressionObject to create the base result.
diff --git a/src/mongo/db/pipeline/document_source_redact.cpp b/src/mongo/db/pipeline/document_source_redact.cpp
index 6a6ebc6e1e8..e7a1548b1e4 100644
--- a/src/mongo/db/pipeline/document_source_redact.cpp
+++ b/src/mongo/db/pipeline/document_source_redact.cpp
@@ -111,8 +111,10 @@ namespace mongo {
             return boost::optional<Document>();
         }
         else if (expressionResult == descendVal) {
+            const Document in = _variables->getDocument(_currentId);
             MutableDocument out;
-            FieldIterator fields(_variables->getDocument(_currentId));
+            out.copyMetaDataFrom(in);
+            FieldIterator fields(in);
             while (fields.more()) {
                 const Document::FieldPair field(fields.next());
 
diff --git a/src/mongo/db/pipeline/document_source_sort.cpp b/src/mongo/db/pipeline/document_source_sort.cpp
index 77adeafaca8..c75feb629ce 100644
--- a/src/mongo/db/pipeline/document_source_sort.cpp
+++ b/src/mongo/db/pipeline/document_source_sort.cpp
@@ -58,12 +58,12 @@ namespace mongo {
     void DocumentSourceSort::serializeToArray(vector<Value>& array, bool explain) const {
         if (explain) { // always one Value for combined $sort + $limit
             array.push_back(Value(DOC(getSourceName() <<
-                DOC("sortKey" << serializeSortKey()
+                DOC("sortKey" << serializeSortKey(explain)
                  << "mergePresorted" << (_mergingPresorted ? Value(true) : Value())
                  << "limit" << (limitSrc ? Value(limitSrc->getLimit()) : Value())))));
         }
         else { // one Value for $sort and maybe a Value for $limit
-            MutableDocument inner (serializeSortKey());
+            MutableDocument inner (serializeSortKey(explain));
             if (_mergingPresorted)
                 inner["$mergePresorted"] = Value(true);
             array.push_back(Value(DOC(getSourceName() << inner.freeze())));
@@ -106,19 +106,25 @@ namespace mongo {
         vAscending.push_back(ascending);
     }
 
-    Document DocumentSourceSort::serializeSortKey() const {
+    Document DocumentSourceSort::serializeSortKey(bool explain) const {
         MutableDocument keyObj;
         // add the key fields
         const size_t n = vSortKey.size();
         for(size_t i = 0; i < n; ++i) {
-            // get the field name out of each ExpressionFieldPath
-            const FieldPath& withVariable = vSortKey[i]->getFieldPath();
-            verify(withVariable.getPathLength() > 1);
-            verify(withVariable.getFieldName(0) == "ROOT");
-            const string fieldPath = withVariable.tail().getPath(false);
-
-            // append a named integer based on the sort order
-            keyObj.setField(fieldPath, Value(vAscending[i] ? 1 : -1));
+            if (ExpressionFieldPath* efp = dynamic_cast<ExpressionFieldPath*>(vSortKey[i].get())) {
+                // ExpressionFieldPath gets special syntax that includes direction
+                const FieldPath& withVariable = efp->getFieldPath();
+                verify(withVariable.getPathLength() > 1);
+                verify(withVariable.getFieldName(0) == "ROOT");
+                const string fieldPath = withVariable.tail().getPath(false);
+
+                // append a named integer based on the sort order
+                keyObj.setField(fieldPath, Value(vAscending[i] ? 1 : -1));
+            }
+            else {
+                // other expressions use a made-up field name
+                keyObj[string(str::stream() << "$computed" << i)] = vSortKey[i]->serialize(explain);
+            }
         }
         return keyObj.freeze();
     }
@@ -150,7 +156,6 @@ namespace mongo {
         intrusive_ptr<DocumentSourceSort> pSort = new DocumentSourceSort(pExpCtx);
 
         /* check for then iterate over the sort object */
-        size_t sortKeys = 0;
         BSONForEach(keyField, sortOrder) {
             const char* fieldName = keyField.fieldName();
 
@@ -159,8 +164,19 @@ namespace mongo {
                 pSort->_mergingPresorted = true;
                 continue;
             }
+
+            if (keyField.type() == Object) {
+                // this restriction is due to needing to figure out sort direction
+                uassert(17312,
+                        "the only expression supported by $sort right now is {$meta: 'textScore'}",
+                        keyField.Obj() == BSON("$meta" << "textScore"));
+
+                pSort->vSortKey.push_back(new ExpressionMeta());
+                pSort->vAscending.push_back(false); // best scoring documents first
+                continue;
+            }
                 
-            uassert(15974, "$sort key ordering must be specified using a number",
+            uassert(15974, "$sort key ordering must be specified using a number or {$meta: 'text'}",
                     keyField.isNumber());
 
             int sortOrder = keyField.numberInt();
@@ -169,11 +185,10 @@ namespace mongo {
                     ((sortOrder == 1) || (sortOrder == -1)));
 
             pSort->addKey(fieldName, (sortOrder > 0));
-            ++sortKeys;
         }
 
-        uassert(15976, str::stream() << sortName <<
-                " must have at least one sort key", (sortKeys > 0));
+        uassert(15976, str::stream() << sortName << " must have at least one sort key",
+                !pSort->vSortKey.empty());
 
         if (limit > 0) {
             bool coalesced = pSort->coalesce(DocumentSourceLimit::create(pExpCtx, limit));
@@ -231,7 +246,7 @@ namespace mongo {
 
         bool more() { return _cursor->more(); }
         Data next() {
-            Document doc(_cursor->next());
+            const Document doc = Document::fromBsonWithMetaData(_cursor->next());
             return make_pair(_sorter->extractKey(doc), doc);
         }
     private:
diff --git a/src/mongo/db/pipeline/expression.cpp b/src/mongo/db/pipeline/expression.cpp
index 82f32dbe70b..449360ec36c 100644
--- a/src/mongo/db/pipeline/expression.cpp
+++ b/src/mongo/db/pipeline/expression.cpp
@@ -1505,6 +1505,36 @@ namespace {
         _each->addDependencies(deps);
     }
 
+    /* ------------------------- ExpressionMeta ----------------------------- */
+
+    REGISTER_EXPRESSION("$meta", ExpressionMeta::parse);
+    intrusive_ptr<Expression> ExpressionMeta::parse(
+            BSONElement expr,
+            const VariablesParseState& vpsIn) {
+
+        uassert(17307, "$meta only supports String arguments",
+                expr.type() == String);
+        uassert(17308, "Unsupported argument to $meta: " + expr.String(),
+                expr.String() == "textScore");
+
+        return new ExpressionMeta();
+    }
+
+    Value ExpressionMeta::serialize(bool explain) const {
+        return Value(DOC("$meta" << "textScore"));
+    }
+
+    Value ExpressionMeta::evaluateInternal(Variables* vars) const {
+        const Document& root = vars->getRoot();
+        return root.hasTextScore()
+                ? Value(root.getTextScore())
+                : Value();
+    }
+
+    void ExpressionMeta::addDependencies(set<string>& deps, vector<string>* path) const {
+        deps.insert("$textScore");
+    }
+
     /* ------------------------- ExpressionMillisecond ----------------------------- */
 
     Value ExpressionMillisecond::evaluateInternal(Variables* vars) const {
diff --git a/src/mongo/db/pipeline/expression.h b/src/mongo/db/pipeline/expression.h
index 36376bb04cb..99121ed1d55 100644
--- a/src/mongo/db/pipeline/expression.h
+++ b/src/mongo/db/pipeline/expression.h
@@ -161,7 +161,7 @@ namespace mongo {
 
           @returns the optimized Expression
          */
-        virtual intrusive_ptr<Expression> optimize() = 0;
+        virtual intrusive_ptr<Expression> optimize() { return this; }
 
         /**
          * Add this expression's field dependencies to the set
@@ -663,6 +663,18 @@ namespace mongo {
         intrusive_ptr<Expression> _each;
     };
 
+    class ExpressionMeta : public Expression {
+    public:
+        // virtuals from Expression
+        virtual Value serialize(bool explain) const;
+        virtual Value evaluateInternal(Variables* vars) const;
+        virtual void addDependencies(set<string>& deps, vector<string>* path=NULL) const;
+
+        static intrusive_ptr<Expression> parse(
+            BSONElement expr,
+            const VariablesParseState& vps);
+    };
+
     class ExpressionMillisecond : public ExpressionFixedArity<ExpressionMillisecond, 1> {
     public:
         // virtuals from ExpressionNary
diff --git a/src/mongo/db/pipeline/pipeline.cpp b/src/mongo/db/pipeline/pipeline.cpp
index 4106ab0938d..1a9f2ec7270 100644
--- a/src/mongo/db/pipeline/pipeline.cpp
+++ b/src/mongo/db/pipeline/pipeline.cpp
@@ -200,6 +200,8 @@ namespace mongo {
             verify(stage);
             sources.push_back(stage);
 
+            // TODO find a good general way to check stages that must be first syntactically
+
             if (dynamic_cast<DocumentSourceOut*>(stage.get())) {
                 uassert(16991, "$out can only be the final stage in the pipeline",
                         iStep == nSteps - 1);
@@ -218,10 +220,14 @@ namespace mongo {
     }
 
     void Pipeline::Optimizations::Local::moveMatchBeforeSort(Pipeline* pipeline) {
+        // TODO Keep moving matches across multiple sorts as moveLimitBeforeSkip does below.
+        // TODO Check sort for limit. Not an issue currently due to order optimizations are applied,
+        // but should be fixed.
         SourceContainer& sources = pipeline->sources;
         for (size_t srcn = sources.size(), srci = 1; srci < srcn; ++srci) {
             intrusive_ptr<DocumentSource> &pSource = sources[srci];
-            if (dynamic_cast<DocumentSourceMatch *>(pSource.get())) {
+            DocumentSourceMatch* match = dynamic_cast<DocumentSourceMatch *>(pSource.get());
+            if (match && !match->isTextQuery()) {
                 intrusive_ptr<DocumentSource> &pPrevious = sources[srci - 1];
                 if (dynamic_cast<DocumentSourceSort *>(pPrevious.get())) {
                     /* swap this item with the previous */
diff --git a/src/mongo/db/pipeline/pipeline_d.cpp b/src/mongo/db/pipeline/pipeline_d.cpp
index 130cd87d01a..b22429a7223 100644
--- a/src/mongo/db/pipeline/pipeline_d.cpp
+++ b/src/mongo/db/pipeline/pipeline_d.cpp
@@ -105,6 +105,7 @@ namespace {
          */
 
         bool haveProjection = false;
+        bool needQueryProjection = false; // true if we need to send the project to query system
         BSONObj projection;
         DocumentSource::ParsedDeps dependencies;
         {
@@ -120,6 +121,13 @@ namespace {
                 projection = DocumentSource::depsToProjection(deps);
                 dependencies = DocumentSource::parseDeps(deps);
                 haveProjection = true;
+                needQueryProjection = deps.count("$textScore");
+            }
+            else if (DocumentSourceMatch::isTextQuery(queryObj)) {
+                // doing a text query. assume we need score since we can't prove we don't.
+                projection = BSON(Document::metaFieldTextScore << BSON("$meta" << "textScore"));
+                haveProjection = true;
+                needQueryProjection = true;
             }
         }
 
@@ -136,7 +144,7 @@ namespace {
             sortStage = dynamic_cast<DocumentSourceSort*>(sources.front().get());
             if (sortStage) {
                 // build the sort key
-                sortObj = sortStage->serializeSortKey().toBson();
+                sortObj = sortStage->serializeSortKey(/*explain*/false).toBson();
             }
         }
 
@@ -186,11 +194,12 @@ namespace {
             CanonicalQuery* cq;
             // Passing an empty projection since it is faster to use documentFromBsonWithDeps.
             // This will need to change to support covering indexes (SERVER-12015).
-            uassertStatusOK(CanonicalQuery::canonicalize(pExpCtx->ns,
-                                                         queryObj,
-                                                         sortObj,
-                                                         BSONObj(), // projection
-                                                         &cq));
+            uassertStatusOK(
+                CanonicalQuery::canonicalize(pExpCtx->ns,
+                                             queryObj,
+                                             sortObj,
+                                             needQueryProjection ? projection : BSONObj(),
+                                             &cq));
             Runner* rawRunner;
             if (getRunner(cq, &rawRunner, runnerOptions).isOK()) {
                 // success: The Runner will handle sorting for us using an index.
@@ -208,11 +217,12 @@ namespace {
         if (!runner.get()) {
             const BSONObj noSort;
             CanonicalQuery* cq;
-            uassertStatusOK(CanonicalQuery::canonicalize(pExpCtx->ns,
-                                                         queryObj,
-                                                         noSort,
-                                                         BSONObj(), // projection
-                                                         &cq));
+            uassertStatusOK(
+                CanonicalQuery::canonicalize(pExpCtx->ns,
+                                             queryObj,
+                                             noSort,
+                                             needQueryProjection ? projection : BSONObj(),
+                                             &cq));
 
             Runner* rawRunner;
             uassertStatusOK(getRunner(cq, &rawRunner, runnerOptions));
@@ -240,7 +250,7 @@ namespace {
             pSource->setSort(sortObj);
 
         if (haveProjection) {
-            pSource->setProjection(projection, dependencies);
+            pSource->setProjection(projection, dependencies, needQueryProjection);
         }
 
         while (!sources.empty() && pSource->coalesce(sources.front())) {
diff --git a/src/mongo/db/pipeline/pipeline_optimizations.h b/src/mongo/db/pipeline/pipeline_optimizations.h
index 4dc8478d083..fe75ba9655a 100644
--- a/src/mongo/db/pipeline/pipeline_optimizations.h
+++ b/src/mongo/db/pipeline/pipeline_optimizations.h
@@ -46,8 +46,9 @@ namespace mongo {
         /** 
          * Moves matches before any adjacent sort phases.
          *
-         * This means we sort fewer items.  Neither changes the documents in
-         * the stream, so this transformation shouldn't affect the result.
+         * This means we sort fewer items.  Neither sorts, nor matches (excluding $text)
+         * change the documents in the stream, so this transformation shouldn't affect
+         * the result.
          */
         static void moveMatchBeforeSort(Pipeline* pipeline);