SERVER-20117 Allow queries with meta sortkey to be covered in mongos

15 files changed, 443 insertions, 139 deletions

diff --git a/src/mongo/db/exec/projection_exec.cpp b/src/mongo/db/exec/projection_exec.cpp
index 1d996d1c6eb..b5d83349cba 100644
--- a/src/mongo/db/exec/projection_exec.cpp
+++ b/src/mongo/db/exec/projection_exec.cpp
@@ -69,8 +69,6 @@ ProjectionExec::ProjectionExec()
       _skip(0),
       _limit(-1),
       _arrayOpType(ARRAY_OP_NORMAL),
-      _hasNonSimple(false),
-      _hasDottedField(false),
       _queryExpression(NULL),
       _hasReturnKey(false) {}
 
@@ -84,24 +82,16 @@ ProjectionExec::ProjectionExec(const BSONObj& spec,
       _skip(0),
       _limit(-1),
       _arrayOpType(ARRAY_OP_NORMAL),
-      _hasNonSimple(false),
-      _hasDottedField(false),
       _queryExpression(queryExpression),
       _hasReturnKey(false) {
-    // Are we including or excluding fields?
-    // -1 when we haven't initialized it.
-    // 1 when we're including
-    // 0 when we're excluding.
-    int include_exclude = -1;
+    // Whether we're including or excluding fields.
+    enum class IncludeExclude { kUninitialized, kInclude, kExclude };
+    IncludeExclude includeExclude = IncludeExclude::kUninitialized;
 
     BSONObjIterator it(_source);
     while (it.more()) {
         BSONElement e = it.next();
 
-        if (!e.isNumber() && !e.isBoolean()) {
-            _hasNonSimple = true;
-        }
-
         if (Object == e.type()) {
             BSONObj obj = e.embeddedObject();
             verify(1 == obj.nFields());
@@ -170,16 +160,10 @@ ProjectionExec::ProjectionExec(const BSONObj& spec,
         } else {
             add(e.fieldName(), e.trueValue());
 
-            // Projections of dotted fields aren't covered.
-            if (mongoutils::str::contains(e.fieldName(), '.')) {
-                _hasDottedField = true;
-            }
-
-            // Validate input.
-            if (include_exclude == -1) {
-                // If we haven't specified an include/exclude, initialize include_exclude.
-                // We expect further include/excludes to match it.
-                include_exclude = e.trueValue();
+            // If we haven't specified an include/exclude, initialize includeExclude.
+            if (includeExclude == IncludeExclude::kUninitialized) {
+                includeExclude =
+                    e.trueValue() ? IncludeExclude::kInclude : IncludeExclude::kExclude;
                 _include = !e.trueValue();
             }
         }
@@ -287,7 +271,7 @@ Status ProjectionExec::transform(WorkingSetMember* member) const {
             return projStatus;
         }
     } else {
-        verify(!requiresDocument());
+        invariant(!_include);
         // Go field by field.
         if (_includeID) {
             BSONElement elt;
@@ -304,6 +288,18 @@ Status ProjectionExec::transform(WorkingSetMember* member) const {
                 continue;
             }
 
+            // $meta sortKey is the only meta-projection which is allowed to operate on index keys
+            // rather than the full document.
+            auto metaIt = _meta.find(specElt.fieldName());
+            if (metaIt != _meta.end()) {
+                invariant(metaIt->second == META_SORT_KEY);
+                continue;
+            }
+
+            // $meta sortKey is also the only element with an Object value in the projection spec
+            // that can operate on index keys rather than the full document.
+            invariant(BSONType::Object != specElt.type());
+
             BSONElement keyElt;
             // We can project a field that doesn't exist.  We just ignore it.
             if (member->getFieldDotted(specElt.fieldName(), &keyElt) && !keyElt.eoo()) {
@@ -364,24 +360,6 @@ Status ProjectionExec::transform(WorkingSetMember* member) const {
     return Status::OK();
 }
 
-Status ProjectionExec::transform(const BSONObj& in, BSONObj* out) const {
-    // If it's a positional projection we need a MatchDetails.
-    MatchDetails matchDetails;
-    if (transformRequiresDetails()) {
-        matchDetails.requestElemMatchKey();
-        verify(NULL != _queryExpression);
-        verify(_queryExpression->matchesBSON(in, &matchDetails));
-    }
-
-    BSONObjBuilder bob;
-    Status s = transform(in, &bob, &matchDetails);
-    if (!s.isOK()) {
-        return s;
-    }
-    *out = bob.obj();
-    return Status::OK();
-}
-
 Status ProjectionExec::transform(const BSONObj& in,
                                  BSONObjBuilder* bob,
                                  const MatchDetails* details) const {
diff --git a/src/mongo/db/exec/projection_exec.h b/src/mongo/db/exec/projection_exec.h
index 906d2fe12d8..3248844b6b7 100644
--- a/src/mongo/db/exec/projection_exec.h
+++ b/src/mongo/db/exec/projection_exec.h
@@ -75,14 +75,6 @@ public:
      */
     Status transform(WorkingSetMember* member) const;
 
-    /**
-     * Apply this projection to the object 'in'.
-     *
-     * Upon success, 'out' is set to the new object and Status::OK() is returned.
-     * Otherwise, returns an error Status and *out is not mutated.
-     */
-    Status transform(const BSONObj& in, BSONObj* out) const;
-
 private:
     //
     // Initialization
@@ -125,13 +117,6 @@ private:
     }
 
     /**
-     * Is the full document required to compute this projection?
-     */
-    bool requiresDocument() const {
-        return _include || _hasNonSimple || _hasDottedField;
-    }
-
-    /**
      * Appends the element 'e' to the builder 'bob', possibly descending into sub-fields of 'e'
      * if needed.
      */
@@ -178,12 +163,6 @@ private:
 
     ArrayOpType _arrayOpType;
 
-    // Is there an slice, elemMatch or meta operator?
-    bool _hasNonSimple;
-
-    // Is there a projection over a dotted field or a $ positional operator?
-    bool _hasDottedField;
-
     // The full query expression.  Used when we need MatchDetails.
     const MatchExpression* _queryExpression;
 
diff --git a/src/mongo/db/exec/projection_exec_test.cpp b/src/mongo/db/exec/projection_exec_test.cpp
index febc30479e8..f8118b4f141 100644
--- a/src/mongo/db/exec/projection_exec_test.cpp
+++ b/src/mongo/db/exec/projection_exec_test.cpp
@@ -141,6 +141,34 @@ void testTransform(const char* specStr,
     testTransform(specStr, queryStr, objStr, NULL, expectedStatusOK, expectedObjStr);
 }
 
+/**
+ * Test function to verify the results of projecting the $meta sortKey while under a covered
+ * projection. In particular, it tests that ProjectionExec can take a WorkingSetMember in
+ * LOC_AND_IDX state and use the sortKey along with the index data to generate the final output
+ * document. For SERVER-20117.
+ *
+ * sortKey - The sort key in BSONObj form.
+ * projSpec - The JSON representation of the proj spec BSONObj.
+ * ikd - The data stored in the index.
+ *
+ * Returns the BSON representation of the actual output, to be checked against the expected output.
+ */
+BSONObj transformMetaSortKeyCovered(const BSONObj& sortKey,
+                                    const char* projSpec,
+                                    const IndexKeyDatum& ikd) {
+    WorkingSet ws;
+    WorkingSetID wsid = ws.allocate();
+    WorkingSetMember* wsm = ws.get(wsid);
+    wsm->keyData.push_back(ikd);
+    wsm->addComputed(new SortKeyComputedData(sortKey));
+    ws.transitionToLocAndIdx(wsid);
+
+    ProjectionExec projExec(fromjson(projSpec), nullptr);
+    ASSERT_OK(projExec.transform(wsm));
+
+    return wsm->obj.value();
+}
+
 //
 // position $
 //
@@ -241,4 +269,61 @@ TEST(ProjectionExecTest, TransformMetaSortKey) {
                   "{a: {'': 99}}");
 }
 
+TEST(ProjectionExecTest, TransformMetaSortKeyCoveredNormal) {
+    BSONObj actualOut =
+        transformMetaSortKeyCovered(BSON("" << 5),
+                                    "{_id: 0, a: 1, b: {$meta: 'sortKey'}}",
+                                    IndexKeyDatum(BSON("a" << 1), BSON("" << 5), nullptr));
+    BSONObj expectedOut = BSON("a" << 5 << "b" << BSON("" << 5));
+    ASSERT_EQ(actualOut, expectedOut);
+}
+
+TEST(ProjectionExecTest, TransformMetaSortKeyCoveredOverwrite) {
+    BSONObj actualOut =
+        transformMetaSortKeyCovered(BSON("" << 5),
+                                    "{_id: 0, a: 1, a: {$meta: 'sortKey'}}",
+                                    IndexKeyDatum(BSON("a" << 1), BSON("" << 5), nullptr));
+    BSONObj expectedOut = BSON("a" << BSON("" << 5));
+    ASSERT_EQ(actualOut, expectedOut);
+}
+
+TEST(ProjectionExecTest, TransformMetaSortKeyCoveredAdditionalData) {
+    BSONObj actualOut = transformMetaSortKeyCovered(
+        BSON("" << 5),
+        "{_id: 0, a: 1, b: {$meta: 'sortKey'}, c: 1}",
+        IndexKeyDatum(BSON("a" << 1 << "c" << 1), BSON("" << 5 << "" << 6), nullptr));
+    BSONObj expectedOut = BSON("a" << 5 << "c" << 6 << "b" << BSON("" << 5));
+    ASSERT_EQ(actualOut, expectedOut);
+}
+
+TEST(ProjectionExecTest, TransformMetaSortKeyCoveredCompound) {
+    BSONObj actualOut = transformMetaSortKeyCovered(
+        BSON("" << 5 << "" << 6),
+        "{_id: 0, a: 1, b: {$meta: 'sortKey'}}",
+        IndexKeyDatum(BSON("a" << 1 << "c" << 1), BSON("" << 5 << "" << 6), nullptr));
+    BSONObj expectedOut = BSON("a" << 5 << "b" << BSON("" << 5 << "" << 6));
+    ASSERT_EQ(actualOut, expectedOut);
+}
+
+TEST(ProjectionExecTest, TransformMetaSortKeyCoveredCompound2) {
+    BSONObj actualOut = transformMetaSortKeyCovered(
+        BSON("" << 5 << "" << 6),
+        "{_id: 0, a: 1, c: 1, b: {$meta: 'sortKey'}}",
+        IndexKeyDatum(
+            BSON("a" << 1 << "b" << 1 << "c" << 1), BSON("" << 5 << "" << 6 << "" << 4), nullptr));
+    BSONObj expectedOut = BSON("a" << 5 << "c" << 4 << "b" << BSON("" << 5 << "" << 6));
+    ASSERT_EQ(actualOut, expectedOut);
+}
+
+TEST(ProjectionExecTest, TransformMetaSortKeyCoveredCompound3) {
+    BSONObj actualOut = transformMetaSortKeyCovered(
+        BSON("" << 6 << "" << 4),
+        "{_id: 0, c: 1, d: 1, b: {$meta: 'sortKey'}}",
+        IndexKeyDatum(BSON("a" << 1 << "b" << 1 << "c" << 1 << "d" << 1),
+                      BSON("" << 5 << "" << 6 << "" << 4 << "" << 9000),
+                      nullptr));
+    BSONObj expectedOut = BSON("c" << 4 << "d" << 9000 << "b" << BSON("" << 6 << "" << 4));
+    ASSERT_EQ(actualOut, expectedOut);
+}
+
 }  // namespace
diff --git a/src/mongo/db/exec/sort_key_generator.cpp b/src/mongo/db/exec/sort_key_generator.cpp
index b9a02b17ed2..7c4be13bc0d 100644
--- a/src/mongo/db/exec/sort_key_generator.cpp
+++ b/src/mongo/db/exec/sort_key_generator.cpp
@@ -49,10 +49,7 @@ namespace mongo {
 // SortKeyGenerator
 //
 
-SortKeyGenerator::SortKeyGenerator(const Collection* collection,
-                                   const BSONObj& sortSpec,
-                                   const BSONObj& queryObj) {
-    _collection = collection;
+SortKeyGenerator::SortKeyGenerator(const BSONObj& sortSpec, const BSONObj& queryObj) {
     _hasBounds = false;
     _sortHasMeta = false;
     _rawSortSpec = sortSpec;
@@ -109,15 +106,19 @@ SortKeyGenerator::SortKeyGenerator(const Collection* collection,
 }
 
 Status SortKeyGenerator::getSortKey(const WorkingSetMember& member, BSONObj* objOut) const {
-    BSONObj btreeKeyToUse;
+    StatusWith<BSONObj> sortKey = BSONObj();
 
-    Status btreeStatus = getBtreeKey(member.obj.value(), &btreeKeyToUse);
-    if (!btreeStatus.isOK()) {
-        return btreeStatus;
+    if (member.hasObj()) {
+        sortKey = getSortKeyFromObject(member);
+    } else {
+        sortKey = getSortKeyFromIndexKey(member);
+    }
+    if (!sortKey.isOK()) {
+        return sortKey.getStatus();
     }
 
     if (!_sortHasMeta) {
-        *objOut = btreeKeyToUse;
+        *objOut = sortKey.getValue();
         return Status::OK();
     }
 
@@ -125,12 +126,12 @@ Status SortKeyGenerator::getSortKey(const WorkingSetMember& member, BSONObj* obj
 
     // Merge metadata into the key.
     BSONObjIterator it(_rawSortSpec);
-    BSONObjIterator btreeIt(btreeKeyToUse);
+    BSONObjIterator sortKeyIt(sortKey.getValue());
     while (it.more()) {
         BSONElement elt = it.next();
         if (elt.isNumber()) {
             // Merge btree key elt.
-            mergedKeyBob.append(btreeIt.next());
+            mergedKeyBob.append(sortKeyIt.next());
         } else if (LiteParsedQuery::isTextScoreMeta(elt)) {
             // Add text score metadata
             double score = 0.0;
@@ -147,11 +148,25 @@ Status SortKeyGenerator::getSortKey(const WorkingSetMember& member, BSONObj* obj
     return Status::OK();
 }
 
-Status SortKeyGenerator::getBtreeKey(const BSONObj& memberObj, BSONObj* objOut) const {
+StatusWith<BSONObj> SortKeyGenerator::getSortKeyFromIndexKey(const WorkingSetMember& member) const {
+    invariant(member.getState() == WorkingSetMember::LOC_AND_IDX);
+    invariant(!_sortHasMeta);
+
+    BSONObjBuilder sortKeyObj;
+    for (BSONElement specElt : _rawSortSpec) {
+        invariant(specElt.isNumber());
+        BSONElement sortKeyElt;
+        invariant(member.getFieldDotted(specElt.fieldName(), &sortKeyElt));
+        sortKeyObj.appendAs(sortKeyElt, "");
+    }
+
+    return sortKeyObj.obj();
+}
+
+StatusWith<BSONObj> SortKeyGenerator::getSortKeyFromObject(const WorkingSetMember& member) const {
     // Not sorting by anything in the key, just bail out early.
     if (_btreeObj.isEmpty()) {
-        *objOut = BSONObj();
-        return Status::OK();
+        return BSONObj();
     }
 
     // We will sort '_data' in the same order an index over '_pattern' would have.  This is
@@ -161,7 +176,7 @@ Status SortKeyGenerator::getBtreeKey(const BSONObj& memberObj, BSONObj* objOut)
     BSONObjSet keys(patternCmp);
 
     try {
-        _keyGen->getKeys(memberObj, &keys);
+        _keyGen->getKeys(member.obj.value(), &keys);
     } catch (const UserException& e) {
         // Probably a parallel array.
         if (BtreeKeyGenerator::ParallelArraysCode == e.getCode()) {
@@ -179,8 +194,7 @@ Status SortKeyGenerator::getBtreeKey(const BSONObj& memberObj, BSONObj* objOut)
     // No bounds?  No problem!  Use the first key.
     if (!_hasBounds) {
         // Note that we sort 'keys' according to the pattern '_btreeObj'.
-        *objOut = *keys.begin();
-        return Status::OK();
+        return *keys.begin();
     }
 
     // To decide which key to use in sorting, we must consider not only the sort pattern but
@@ -192,15 +206,13 @@ Status SortKeyGenerator::getBtreeKey(const BSONObj& memberObj, BSONObj* objOut)
     verify(NULL != _boundsChecker.get());
     for (BSONObjSet::const_iterator it = keys.begin(); it != keys.end(); ++it) {
         if (_boundsChecker->isValidKey(*it)) {
-            *objOut = *it;
-            return Status::OK();
+            return *it;
         }
     }
 
     // No key is in our bounds.
     // TODO: will this ever happen?  don't think it should.
-    *objOut = *keys.begin();
-    return Status::OK();
+    return *keys.begin();
 }
 
 void SortKeyGenerator::getBoundsForSort(const BSONObj& queryObj, const BSONObj& sortObj) {
@@ -258,14 +270,9 @@ const char* SortKeyGeneratorStage::kStageType = "SORT_KEY_GENERATOR";
 SortKeyGeneratorStage::SortKeyGeneratorStage(OperationContext* opCtx,
                                              PlanStage* child,
                                              WorkingSet* ws,
-                                             const Collection* collection,
                                              const BSONObj& sortSpecObj,
                                              const BSONObj& queryObj)
-    : PlanStage(kStageType, opCtx),
-      _ws(ws),
-      _collection(collection),
-      _sortSpec(sortSpecObj),
-      _query(queryObj) {
+    : PlanStage(kStageType, opCtx), _ws(ws), _sortSpec(sortSpecObj), _query(queryObj) {
     _children.emplace_back(child);
 }
 
@@ -280,7 +287,7 @@ PlanStage::StageState SortKeyGeneratorStage::work(WorkingSetID* out) {
     ScopedTimer timer(&_commonStats.executionTimeMillis);
 
     if (!_sortKeyGen) {
-        _sortKeyGen = stdx::make_unique<SortKeyGenerator>(_collection, _sortSpec, _query);
+        _sortKeyGen = stdx::make_unique<SortKeyGenerator>(_sortSpec, _query);
         ++_commonStats.needTime;
         return PlanStage::NEED_TIME;
     }
diff --git a/src/mongo/db/exec/sort_key_generator.h b/src/mongo/db/exec/sort_key_generator.h
index 15bf709fc56..555d6b8ce6b 100644
--- a/src/mongo/db/exec/sort_key_generator.h
+++ b/src/mongo/db/exec/sort_key_generator.h
@@ -53,17 +53,20 @@ public:
      * ensure that the value we select to sort by is within bounds generated by
      * executing 'queryObj' using the virtual index with key pattern 'sortSpec'.
      */
-    SortKeyGenerator(const Collection* collection,
-                     const BSONObj& sortSpec,
-                     const BSONObj& queryObj);
+    SortKeyGenerator(const BSONObj& sortSpec, const BSONObj& queryObj);
 
     /**
-     * Returns the key used to sort 'member'.
+     * Returns the key used to sort 'member'. If the member is in LOC_AND_IDX state, it must not
+     * contain a $meta textScore in its sort spec, and this function will use the index key data
+     * stored in 'member' to extract the sort key. Otherwise, if the member is in LOC_AND_OBJ or
+     * OWNED_OBJ state, this function will use the object data stored in 'member' to extract the
+     * sort key.
      */
     Status getSortKey(const WorkingSetMember& member, BSONObj* objOut) const;
 
 private:
-    Status getBtreeKey(const BSONObj& memberObj, BSONObj* objOut) const;
+    StatusWith<BSONObj> getSortKeyFromIndexKey(const WorkingSetMember& member) const;
+    StatusWith<BSONObj> getSortKeyFromObject(const WorkingSetMember& member) const;
 
     /**
      * In order to emulate the existing sort behavior we must make unindexed sort behavior as
@@ -75,9 +78,6 @@ private:
      */
     void getBoundsForSort(const BSONObj& queryObj, const BSONObj& sortObj);
 
-    // Not owned by us
-    const Collection* _collection;
-
     // The raw object in .sort()
     BSONObj _rawSortSpec;
 
@@ -109,7 +109,6 @@ public:
     SortKeyGeneratorStage(OperationContext* opCtx,
                           PlanStage* child,
                           WorkingSet* ws,
-                          const Collection* collection,
                           const BSONObj& sortSpecObj,
                           const BSONObj& queryObj);
 
@@ -130,8 +129,6 @@ public:
 private:
     WorkingSet* const _ws;
 
-    const Collection* const _collection;
-
     // The raw sort pattern as expressed by the user.
     const BSONObj _sortSpec;
 
diff --git a/src/mongo/db/exec/sort_test.cpp b/src/mongo/db/exec/sort_test.cpp
index 552ec12a69f..8f152709ea4 100644
--- a/src/mongo/db/exec/sort_test.cpp
+++ b/src/mongo/db/exec/sort_test.cpp
@@ -47,7 +47,7 @@ TEST(SortStageTest, SortEmptyWorkingSet) {
     // QueuedDataStage will be owned by SortStage.
     auto queuedDataStage = stdx::make_unique<QueuedDataStage>(nullptr, &ws);
     auto sortKeyGen = stdx::make_unique<SortKeyGeneratorStage>(
-        nullptr, queuedDataStage.release(), &ws, nullptr, BSONObj(), BSONObj());
+        nullptr, queuedDataStage.release(), &ws, BSONObj(), BSONObj());
     SortStageParams params;
     SortStage sort(nullptr, params, &ws, sortKeyGen.release());
 
@@ -113,7 +113,7 @@ void testWork(const char* patternStr,
     params.limit = limit;
 
     auto sortKeyGen = stdx::make_unique<SortKeyGeneratorStage>(
-        nullptr, queuedDataStage.release(), &ws, nullptr, params.pattern, fromjson(queryStr));
+        nullptr, queuedDataStage.release(), &ws, params.pattern, fromjson(queryStr));
 
     SortStage sort(nullptr, params, &ws, sortKeyGen.release());
 
diff --git a/src/mongo/db/query/get_executor.cpp b/src/mongo/db/query/get_executor.cpp
index 03a2872efc9..908ca6c5ef3 100644
--- a/src/mongo/db/query/get_executor.cpp
+++ b/src/mongo/db/query/get_executor.cpp
@@ -273,7 +273,6 @@ Status prepareExecution(OperationContext* opCtx,
                 *rootOut = new SortKeyGeneratorStage(opCtx,
                                                      *rootOut,
                                                      ws,
-                                                     collection,
                                                      canonicalQuery->getParsed().getSort(),
                                                      canonicalQuery->getParsed().getFilter());
             }
diff --git a/src/mongo/db/query/parsed_projection.cpp b/src/mongo/db/query/parsed_projection.cpp
index 4f6a48c2f9e..92c33bb14db 100644
--- a/src/mongo/db/query/parsed_projection.cpp
+++ b/src/mongo/db/query/parsed_projection.cpp
@@ -49,16 +49,11 @@ Status ParsedProjection::make(const BSONObj& spec,
                               const MatchExpression* const query,
                               ParsedProjection** out,
                               const MatchExpressionParser::WhereCallback& whereCallback) {
-    // Are we including or excluding fields?  Values:
-    // -1 when we haven't initialized it.
-    // 1 when we're including
-    // 0 when we're excluding.
-    int include_exclude = -1;
+    // Whether we're including or excluding fields.
+    enum class IncludeExclude { kUninitialized, kInclude, kExclude };
+    IncludeExclude includeExclude = IncludeExclude::kUninitialized;
 
-    // If any of these are 'true' the projection isn't covered.
-    bool include = true;
-    bool hasNonSimple = false;
-    bool hasDottedField = false;
+    bool requiresDocument = false;
 
     bool includeID = true;
 
@@ -75,10 +70,6 @@ Status ParsedProjection::make(const BSONObj& spec,
     while (it.more()) {
         BSONElement e = it.next();
 
-        if (!e.isNumber() && !e.isBoolean()) {
-            hasNonSimple = true;
-        }
-
         if (Object == e.type()) {
             BSONObj obj = e.embeddedObject();
             if (1 != obj.nFields()) {
@@ -106,6 +97,9 @@ Status ParsedProjection::make(const BSONObj& spec,
                     return Status(ErrorCodes::BadValue,
                                   "$slice only supports numbers and [skip, limit] arrays");
                 }
+
+                // Projections with $slice aren't covered.
+                requiresDocument = true;
             } else if (mongoutils::str::equals(e2.fieldName(), "$elemMatch")) {
                 // Validate $elemMatch arguments and dependencies.
                 if (Object != e2.type()) {
@@ -135,6 +129,9 @@ Status ParsedProjection::make(const BSONObj& spec,
                 if (!statusWithMatcher.isOK()) {
                     return statusWithMatcher.getStatus();
                 }
+
+                // Projections with $elemMatch aren't covered.
+                requiresDocument = true;
             } else if (mongoutils::str::equals(e2.fieldName(), "$meta")) {
                 // Field for meta must be top level.  We can relax this at some point.
                 if (mongoutils::str::contains(e.fieldName(), '.')) {
@@ -166,6 +163,11 @@ Status ParsedProjection::make(const BSONObj& spec,
                 } else if (e2.valuestr() == LiteParsedQuery::metaSortKey) {
                     wantSortKey = true;
                 }
+
+                // Of the $meta projections, only sortKey can be covered.
+                if (e2.valuestr() != LiteParsedQuery::metaSortKey) {
+                    requiresDocument = true;
+                }
             } else {
                 return Status(ErrorCodes::BadValue,
                               string("Unsupported projection option: ") + e.toString());
@@ -175,23 +177,21 @@ Status ParsedProjection::make(const BSONObj& spec,
         } else {
             // Projections of dotted fields aren't covered.
             if (mongoutils::str::contains(e.fieldName(), '.')) {
-                hasDottedField = true;
+                requiresDocument = true;
             }
 
-            // Validate input.
-            if (include_exclude == -1) {
-                // If we haven't specified an include/exclude, initialize include_exclude.
-                // We expect further include/excludes to match it.
-                include_exclude = e.trueValue();
-                include = !e.trueValue();
-            } else if (static_cast<bool>(include_exclude) != e.trueValue()) {
-                // Make sure that the incl./excl. matches the previous.
+            // If we haven't specified an include/exclude, initialize includeExclude. We expect
+            // further include/excludes to match it.
+            if (includeExclude == IncludeExclude::kUninitialized) {
+                includeExclude =
+                    e.trueValue() ? IncludeExclude::kInclude : IncludeExclude::kExclude;
+            } else if ((includeExclude == IncludeExclude::kInclude && !e.trueValue()) ||
+                       (includeExclude == IncludeExclude::kExclude && e.trueValue())) {
                 return Status(ErrorCodes::BadValue,
                               "Projection cannot have a mix of inclusion and exclusion.");
             }
         }
 
-
         if (_isPositionalOperator(e.fieldName())) {
             // Validate the positional op.
             if (!e.trueValue()) {
@@ -229,6 +229,13 @@ Status ParsedProjection::make(const BSONObj& spec,
         }
     }
 
+    // If includeExclude is uninitialized or set to exclude fields, then we can't use an index
+    // because we don't know what fields we're missing.
+    if (includeExclude == IncludeExclude::kUninitialized ||
+        includeExclude == IncludeExclude::kExclude) {
+        requiresDocument = true;
+    }
+
     // Fill out the returned obj.
     unique_ptr<ParsedProjection> pp(new ParsedProjection());
 
@@ -240,11 +247,7 @@ Status ParsedProjection::make(const BSONObj& spec,
     verify(spec.isOwned());
     pp->_source = spec;
     pp->_returnKey = hasIndexKeyProjection;
-
-    // Dotted fields aren't covered, non-simple require match details, and as for include, "if
-    // we default to including then we can't use an index because we don't know what we're
-    // missing."
-    pp->_requiresDocument = include || hasNonSimple || hasDottedField;
+    pp->_requiresDocument = requiresDocument;
 
     // Add meta-projections.
     pp->_wantGeoNearPoint = wantGeoNearPoint;
@@ -258,8 +261,9 @@ Status ParsedProjection::make(const BSONObj& spec,
             pp->_requiredFields.push_back("_id");
         }
 
-        // The only way we could be here is if spec is only simple non-dotted-field projections.
-        // Therefore we can iterate over spec to get the fields required.
+        // The only way we could be here is if spec is only simple non-dotted-field inclusions or
+        // the $meta sortKey projection. Therefore we can iterate over spec to get the fields
+        // required.
         BSONObjIterator srcIt(spec);
         while (srcIt.more()) {
             BSONElement elt = srcIt.next();
@@ -267,6 +271,13 @@ Status ParsedProjection::make(const BSONObj& spec,
             if (includeID && mongoutils::str::equals(elt.fieldName(), "_id")) {
                 continue;
             }
+            // $meta sortKey should not be checked as a part of _requiredFields, since it can
+            // potentially produce a covered projection as long as the sort key is covered.
+            if (BSONType::Object == elt.type()) {
+                dassert(elt.Obj() == BSON("$meta"
+                                          << "sortKey"));
+                continue;
+            }
             if (elt.trueValue()) {
                 pp->_requiredFields.push_back(elt.fieldName());
             }
diff --git a/src/mongo/db/query/parsed_projection_test.cpp b/src/mongo/db/query/parsed_projection_test.cpp
index f850221b5fe..f235f32a15e 100644
--- a/src/mongo/db/query/parsed_projection_test.cpp
+++ b/src/mongo/db/query/parsed_projection_test.cpp
@@ -215,6 +215,62 @@ TEST(ParsedProjectionTest, SortKeyMetaProjection) {
     ASSERT_FALSE(parsedProjection->wantIndexKey());
 }
 
+TEST(ParsedProjectionTest, SortKeyMetaProjectionCovered) {
+    auto parsedProjection = createParsedProjection("{}", "{a: 1, foo: {$meta: 'sortKey'}, _id: 0}");
+
+    ASSERT_EQ(parsedProjection->getProjObj(), fromjson("{a: 1, foo: {$meta: 'sortKey'}, _id: 0}"));
+    ASSERT_TRUE(parsedProjection->wantSortKey());
+
+    ASSERT_FALSE(parsedProjection->requiresDocument());
+    ASSERT_FALSE(parsedProjection->requiresMatchDetails());
+    ASSERT_FALSE(parsedProjection->wantGeoNearDistance());
+    ASSERT_FALSE(parsedProjection->wantGeoNearPoint());
+    ASSERT_FALSE(parsedProjection->wantIndexKey());
+}
+
+TEST(ParsedProjectionTest, SortKeyMetaAndSlice) {
+    auto parsedProjection =
+        createParsedProjection("{}", "{a: 1, foo: {$meta: 'sortKey'}, _id: 0, b: {$slice: 1}}");
+
+    ASSERT_EQ(parsedProjection->getProjObj(),
+              fromjson("{a: 1, foo: {$meta: 'sortKey'}, _id: 0, b: {$slice: 1}}"));
+    ASSERT_TRUE(parsedProjection->wantSortKey());
+    ASSERT_TRUE(parsedProjection->requiresDocument());
+
+    ASSERT_FALSE(parsedProjection->requiresMatchDetails());
+    ASSERT_FALSE(parsedProjection->wantGeoNearDistance());
+    ASSERT_FALSE(parsedProjection->wantGeoNearPoint());
+    ASSERT_FALSE(parsedProjection->wantIndexKey());
+}
+
+TEST(ParsedProjectionTest, SortKeyMetaAndElemMatch) {
+    auto parsedProjection = createParsedProjection(
+        "{}", "{a: 1, foo: {$meta: 'sortKey'}, _id: 0, b: {$elemMatch: {a: 1}}}");
+
+    ASSERT_EQ(parsedProjection->getProjObj(),
+              fromjson("{a: 1, foo: {$meta: 'sortKey'}, _id: 0, b: {$elemMatch: {a: 1}}}"));
+    ASSERT_TRUE(parsedProjection->wantSortKey());
+    ASSERT_TRUE(parsedProjection->requiresDocument());
+
+    ASSERT_FALSE(parsedProjection->requiresMatchDetails());
+    ASSERT_FALSE(parsedProjection->wantGeoNearDistance());
+    ASSERT_FALSE(parsedProjection->wantGeoNearPoint());
+    ASSERT_FALSE(parsedProjection->wantIndexKey());
+}
+
+TEST(ParsedProjectionTest, SortKeyMetaAndExclusion) {
+    auto parsedProjection = createParsedProjection("{}", "{a: 0, foo: {$meta: 'sortKey'}, _id: 0}");
+
+    ASSERT_EQ(parsedProjection->getProjObj(), fromjson("{a: 0, foo: {$meta: 'sortKey'}, _id: 0}"));
+    ASSERT_TRUE(parsedProjection->wantSortKey());
+    ASSERT_TRUE(parsedProjection->requiresDocument());
+
+    ASSERT_FALSE(parsedProjection->requiresMatchDetails());
+    ASSERT_FALSE(parsedProjection->wantGeoNearDistance());
+    ASSERT_FALSE(parsedProjection->wantGeoNearPoint());
+    ASSERT_FALSE(parsedProjection->wantIndexKey());
+}
+
 //
 // DBRef projections
 //
diff --git a/src/mongo/db/query/planner_analysis.cpp b/src/mongo/db/query/planner_analysis.cpp
index 1ba667ec16b..9a9ec01fd8c 100644
--- a/src/mongo/db/query/planner_analysis.cpp
+++ b/src/mongo/db/query/planner_analysis.cpp
@@ -765,6 +765,13 @@ QuerySolution* QueryPlannerAnalysis::analyzeDataAccess(const CanonicalQuery& que
                     }
                 }
             }
+
+            // If we have a $meta sortKey, just use the project default path, as currently the
+            // project fast paths cannot handle $meta sortKey projections.
+            if (query.getProj()->wantSortKey()) {
+                projType = ProjectionNode::DEFAULT;
+                LOG(5) << "PROJECTION: needs $meta sortKey, using DEFAULT path instead";
+            }
         }
         // If we don't have a covered project, and we're not allowed to put an uncovered one in,
         // bail out.
diff --git a/src/mongo/db/query/query_planner_test.cpp b/src/mongo/db/query/query_planner_test.cpp
index e36991f2b86..4a5ca6591c8 100644
--- a/src/mongo/db/query/query_planner_test.cpp
+++ b/src/mongo/db/query/query_planner_test.cpp
@@ -3995,6 +3995,24 @@ TEST_F(QueryPlannerTest, SortKeyMetaProjection) {
         "{ixscan: {pattern: {a: 1}}}}}}}}}");
 }
 
+TEST_F(QueryPlannerTest, SortKeyMetaProjectionCovered) {
+    addIndex(BSON("a" << 1));
+
+    runQuerySortProj(
+        BSONObj(), fromjson("{a: 1}"), fromjson("{_id: 0, a: 1, b: {$meta: 'sortKey'}}"));
+
+    assertNumSolutions(2U);
+    assertSolutionExists(
+        "{proj: {spec: {_id: 0, a: 1, b: {$meta: 'sortKey'}}, node: "
+        "{sort: {limit: 0, pattern: {a: 1}, node: "
+        "{sortKeyGen: {node: "
+        "{cscan: {dir: 1}}}}}}}}");
+    assertSolutionExists(
+        "{proj: {spec: {_id: 0, a: 1, b: {$meta: 'sortKey'}}, node: "
+        "{sortKeyGen: {node: "
+        "{ixscan: {pattern: {a: 1}}}}}}}");
+}
+
 //
 // Test bad input to query planner helpers.
 //
diff --git a/src/mongo/db/query/stage_builder.cpp b/src/mongo/db/query/stage_builder.cpp
index aec122b537c..b971c44288a 100644
--- a/src/mongo/db/query/stage_builder.cpp
+++ b/src/mongo/db/query/stage_builder.cpp
@@ -125,7 +125,7 @@ PlanStage* buildStages(OperationContext* txn,
             return NULL;
         }
         return new SortKeyGeneratorStage(
-            txn, childStage, ws, collection, keyGenNode->sortSpec, keyGenNode->queryObj);
+            txn, childStage, ws, keyGenNode->sortSpec, keyGenNode->queryObj);
     } else if (STAGE_PROJECTION == root->getType()) {
         const ProjectionNode* pn = static_cast<const ProjectionNode*>(root);
         PlanStage* childStage = buildStages(txn, collection, qsol, pn->children[0], ws);
diff --git a/src/mongo/dbtests/SConscript b/src/mongo/dbtests/SConscript
index 8d5ee4ff09b..2f63251efae 100644
--- a/src/mongo/dbtests/SConscript
+++ b/src/mongo/dbtests/SConscript
@@ -109,6 +109,7 @@ dbtest = env.Program(
         'repltests.cpp',
         'rollbacktests.cpp',
         'socktests.cpp',
+        'sort_key_generator_test.cpp',
         'threadedtests.cpp',
         'updatetests.cpp',
         'validate_tests.cpp',
diff --git a/src/mongo/dbtests/query_stage_sort.cpp b/src/mongo/dbtests/query_stage_sort.cpp
index 04f3496c6ba..64db92f4e22 100644
--- a/src/mongo/dbtests/query_stage_sort.cpp
+++ b/src/mongo/dbtests/query_stage_sort.cpp
@@ -114,7 +114,7 @@ public:
         params.limit = limit();
 
         auto keyGenStage = make_unique<SortKeyGeneratorStage>(
-            &_txn, queuedDataStage.release(), ws.get(), coll, params.pattern, BSONObj());
+            &_txn, queuedDataStage.release(), ws.get(), params.pattern, BSONObj());
 
         auto ss = make_unique<SortStage>(&_txn, params, ws.get(), keyGenStage.release());
 
@@ -153,7 +153,7 @@ public:
         params.limit = limit();
 
         auto keyGenStage = make_unique<SortKeyGeneratorStage>(
-            &_txn, queuedDataStage.release(), ws.get(), coll, params.pattern, BSONObj());
+            &_txn, queuedDataStage.release(), ws.get(), params.pattern, BSONObj());
 
         auto sortStage = make_unique<SortStage>(&_txn, params, ws.get(), keyGenStage.release());
 
@@ -556,7 +556,7 @@ public:
         params.limit = 0;
 
         auto keyGenStage = make_unique<SortKeyGeneratorStage>(
-            &_txn, queuedDataStage.release(), ws.get(), coll, params.pattern, BSONObj());
+            &_txn, queuedDataStage.release(), ws.get(), params.pattern, BSONObj());
 
         auto sortStage = make_unique<SortStage>(&_txn, params, ws.get(), keyGenStage.release());
 
diff --git a/src/mongo/dbtests/sort_key_generator_test.cpp b/src/mongo/dbtests/sort_key_generator_test.cpp
new file mode 100644
index 00000000000..26263c06119
--- /dev/null
+++ b/src/mongo/dbtests/sort_key_generator_test.cpp
@@ -0,0 +1,166 @@
+/**
+ *    Copyright (C) 2015 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/exec/sort_key_generator.h"
+#include "mongo/db/json.h"
+#include "mongo/stdx/memory.h"
+#include "mongo/unittest/unittest.h"
+
+namespace mongo {
+
+namespace {
+
+/**
+ * Test function to verify that the SortKeyGenerator can generate a sortKey from a fetched document.
+ *
+ * sortSpec - The JSON representation of the sort spec BSONObj.
+ * doc - The JSON representation of the BSON document.
+ *
+ * Returns the BSON representation of the sort key, to be checked against the expected sort key.
+ */
+BSONObj extractSortKey(const char* sortSpec, const char* doc) {
+    WorkingSetMember wsm;
+    wsm.obj = Snapshotted<BSONObj>(SnapshotId(), fromjson(doc));
+    wsm.transitionToOwnedObj();
+
+    BSONObj sortKey;
+    auto sortKeyGen = stdx::make_unique<SortKeyGenerator>(fromjson(sortSpec), BSONObj());
+    ASSERT_OK(sortKeyGen->getSortKey(wsm, &sortKey));
+
+    return sortKey;
+}
+
+/**
+ * Test function to verify that the SortKeyGenerator can generate a sortKey while using only index
+ * data (that is, the document will not be fetched). For SERVER-20117.
+ *
+ * sortSpec - The JSON representation of the sort spec BSONObj.
+ * ikd - The data stored in the index.
+ *
+ * Returns the BSON representation of the sort key, to be checked against the expected sort key.
+ */
+BSONObj extractSortKeyCovered(const char* sortSpec, const IndexKeyDatum& ikd) {
+    WorkingSet ws;
+    WorkingSetID wsid = ws.allocate();
+    WorkingSetMember* wsm = ws.get(wsid);
+    wsm->keyData.push_back(ikd);
+    ws.transitionToLocAndIdx(wsid);
+
+    BSONObj sortKey;
+    auto sortKeyGen = stdx::make_unique<SortKeyGenerator>(fromjson(sortSpec), BSONObj());
+    ASSERT_OK(sortKeyGen->getSortKey(*wsm, &sortKey));
+
+    return sortKey;
+}
+
+TEST(SortKeyGeneratorTest, SortKeyNormal) {
+    BSONObj actualOut = extractSortKey("{a: 1}", "{_id: 0, a: 5}");
+    BSONObj expectedOut = BSON("" << 5);
+    ASSERT_EQ(actualOut, expectedOut);
+}
+
+TEST(SortKeyGeneratorTest, SortKeyNormal2) {
+    BSONObj actualOut = extractSortKey("{a: 1}", "{_id: 0, z: 10, a: 6, b: 16}");
+    BSONObj expectedOut = BSON("" << 6);
+    ASSERT_EQ(actualOut, expectedOut);
+}
+
+TEST(SortKeyGeneratorTest, SortKeyString) {
+    BSONObj actualOut = extractSortKey("{a: 1}", "{_id: 0, z: 'thing1', a: 'thing2', b: 16}");
+    BSONObj expectedOut = BSON(""
+                               << "thing2");
+    ASSERT_EQ(actualOut, expectedOut);
+}
+
+TEST(SortKeyGeneratorTest, SortKeyCompound) {
+    BSONObj actualOut =
+        extractSortKey("{a: 1, b: 1}", "{_id: 0, z: 'thing1', a: 99, c: {a: 4}, b: 16}");
+    BSONObj expectedOut = BSON("" << 99 << "" << 16);
+    ASSERT_EQ(actualOut, expectedOut);
+}
+
+TEST(SortKeyGeneratorTest, SortKeyEmbedded) {
+    BSONObj actualOut =
+        extractSortKey("{'c.a': 1, b: 1}", "{_id: 0, z: 'thing1', a: 99, c: {a: 4}, b: 16}");
+    BSONObj expectedOut = BSON("" << 4 << "" << 16);
+    ASSERT_EQ(actualOut, expectedOut);
+}
+
+TEST(SortKeyGeneratorTest, SortKeyArray) {
+    BSONObj actualOut =
+        extractSortKey("{'c': 1, b: 1}", "{_id: 0, z: 'thing1', a: 99, c: [2, 4, 1], b: 16}");
+    BSONObj expectedOut = BSON("" << 1 << "" << 16);
+    ASSERT_EQ(actualOut, expectedOut);
+}
+
+TEST(SortKeyGeneratorTest, SortKeyCoveredNormal) {
+    BSONObj actualOut =
+        extractSortKeyCovered("{a: 1}", IndexKeyDatum(BSON("a" << 1), BSON("" << 5), nullptr));
+    BSONObj expectedOut = BSON("" << 5);
+    ASSERT_EQ(actualOut, expectedOut);
+}
+
+TEST(SortKeyGeneratorTest, SortKeyCoveredEmbedded) {
+    BSONObj actualOut = extractSortKeyCovered(
+        "{'a.c': 1}",
+        IndexKeyDatum(BSON("a.c" << 1 << "c" << 1), BSON("" << 5 << "" << 6), nullptr));
+    BSONObj expectedOut = BSON("" << 5);
+    ASSERT_EQ(actualOut, expectedOut);
+}
+
+TEST(SortKeyGeneratorTest, SortKeyCoveredCompound) {
+    BSONObj actualOut = extractSortKeyCovered(
+        "{a: 1, c: 1}",
+        IndexKeyDatum(BSON("a" << 1 << "c" << 1), BSON("" << 5 << "" << 6), nullptr));
+    BSONObj expectedOut = BSON("" << 5 << "" << 6);
+    ASSERT_EQ(actualOut, expectedOut);
+}
+
+TEST(SortKeyGeneratorTest, SortKeyCoveredCompound2) {
+    BSONObj actualOut = extractSortKeyCovered("{a: 1, b: 1}",
+                                              IndexKeyDatum(BSON("a" << 1 << "b" << 1 << "c" << 1),
+                                                            BSON("" << 5 << "" << 6 << "" << 4),
+                                                            nullptr));
+    BSONObj expectedOut = BSON("" << 5 << "" << 6);
+    ASSERT_EQ(actualOut, expectedOut);
+}
+
+TEST(SortKeyGeneratorTest, SortKeyCoveredCompound3) {
+    BSONObj actualOut =
+        extractSortKeyCovered("{b: 1, c: 1}",
+                              IndexKeyDatum(BSON("a" << 1 << "b" << 1 << "c" << 1 << "d" << 1),
+                                            BSON("" << 5 << "" << 6 << "" << 4 << "" << 9000),
+                                            nullptr));
+    BSONObj expectedOut = BSON("" << 6 << "" << 4);
+    ASSERT_EQ(actualOut, expectedOut);
+}
+
+}  // namespace
+}  // namespace mongo