SERVER-37446 Change PlanStages which don't require a collection to avoid keeping a Collection*.

In order to prevent COUNT stage from requiring a Collection*, splits
fast count into a new RECORD_STORE_FAST_COUNT stage.

15 files changed, 188 insertions, 151 deletions

diff --git a/src/mongo/db/exec/and_hash.cpp b/src/mongo/db/exec/and_hash.cpp
index 338a4abdd31..fdce9a79066 100644
--- a/src/mongo/db/exec/and_hash.cpp
+++ b/src/mongo/db/exec/and_hash.cpp
@@ -56,21 +56,16 @@ const size_t AndHashStage::kLookAheadWorks = 10;
 // static
 const char* AndHashStage::kStageType = "AND_HASH";
 
-AndHashStage::AndHashStage(OperationContext* opCtx, WorkingSet* ws, const Collection* collection)
+AndHashStage::AndHashStage(OperationContext* opCtx, WorkingSet* ws)
     : PlanStage(kStageType, opCtx),
-      _collection(collection),
       _ws(ws),
       _hashingChildren(true),
       _currentChild(0),
       _memUsage(0),
       _maxMemUsage(kDefaultMaxMemUsageBytes) {}
 
-AndHashStage::AndHashStage(OperationContext* opCtx,
-                           WorkingSet* ws,
-                           const Collection* collection,
-                           size_t maxMemUsage)
+AndHashStage::AndHashStage(OperationContext* opCtx, WorkingSet* ws, size_t maxMemUsage)
     : PlanStage(kStageType, opCtx),
-      _collection(collection),
       _ws(ws),
       _hashingChildren(true),
       _currentChild(0),
diff --git a/src/mongo/db/exec/and_hash.h b/src/mongo/db/exec/and_hash.h
index facdf11c49e..67a41a743f7 100644
--- a/src/mongo/db/exec/and_hash.h
+++ b/src/mongo/db/exec/and_hash.h
@@ -49,15 +49,12 @@ namespace mongo {
  */
 class AndHashStage final : public PlanStage {
 public:
-    AndHashStage(OperationContext* opCtx, WorkingSet* ws, const Collection* collection);
+    AndHashStage(OperationContext* opCtx, WorkingSet* ws);
 
     /**
      * For testing only. Allows tests to set memory usage threshold.
      */
-    AndHashStage(OperationContext* opCtx,
-                 WorkingSet* ws,
-                 const Collection* collection,
-                 size_t maxMemUsage);
+    AndHashStage(OperationContext* opCtx, WorkingSet* ws, size_t maxMemUsage);
 
     void addChild(PlanStage* child);
 
@@ -88,9 +85,6 @@ private:
     StageState workChild(size_t childNo, WorkingSetID* out);
 
     // Not owned by us.
-    const Collection* _collection;
-
-    // Not owned by us.
     WorkingSet* _ws;
 
     // We want to see if any of our children are EOF immediately.  This requires working them a
diff --git a/src/mongo/db/exec/and_sorted.cpp b/src/mongo/db/exec/and_sorted.cpp
index c2ab3ec657e..a9e57bf8ea7 100644
--- a/src/mongo/db/exec/and_sorted.cpp
+++ b/src/mongo/db/exec/and_sorted.cpp
@@ -46,11 +46,8 @@ using stdx::make_unique;
 // static
 const char* AndSortedStage::kStageType = "AND_SORTED";
 
-AndSortedStage::AndSortedStage(OperationContext* opCtx,
-                               WorkingSet* ws,
-                               const Collection* collection)
+AndSortedStage::AndSortedStage(OperationContext* opCtx, WorkingSet* ws)
     : PlanStage(kStageType, opCtx),
-      _collection(collection),
       _ws(ws),
       _targetNode(numeric_limits<size_t>::max()),
       _targetId(WorkingSet::INVALID_ID),
diff --git a/src/mongo/db/exec/and_sorted.h b/src/mongo/db/exec/and_sorted.h
index 0a23de04617..784da1f1e0a 100644
--- a/src/mongo/db/exec/and_sorted.h
+++ b/src/mongo/db/exec/and_sorted.h
@@ -48,7 +48,7 @@ namespace mongo {
  */
 class AndSortedStage final : public PlanStage {
 public:
-    AndSortedStage(OperationContext* opCtx, WorkingSet* ws, const Collection* collection);
+    AndSortedStage(OperationContext* opCtx, WorkingSet* ws);
 
     void addChild(PlanStage* child);
 
@@ -74,9 +74,6 @@ private:
     PlanStage::StageState moveTowardTargetRecordId(WorkingSetID* out);
 
     // Not owned by us.
-    const Collection* _collection;
-
-    // Not owned by us.
     WorkingSet* _ws;
 
     // The current node we're AND-ing against.
diff --git a/src/mongo/db/exec/count.cpp b/src/mongo/db/exec/count.cpp
index 3fa907b177e..15d6ec5baff 100644
--- a/src/mongo/db/exec/count.cpp
+++ b/src/mongo/db/exec/count.cpp
@@ -51,62 +51,23 @@ CountStage::CountStage(OperationContext* opCtx,
                        CountStageParams params,
                        WorkingSet* ws,
                        PlanStage* child)
-    : PlanStage(kStageType, opCtx),
-      _collection(collection),
-      _params(std::move(params)),
-      _leftToSkip(_params.skip),
-      _ws(ws) {
-    if (child)
-        _children.emplace_back(child);
+    : PlanStage(kStageType, opCtx), _params(std::move(params)), _leftToSkip(_params.skip), _ws(ws) {
+    invariant(child);
+    _children.emplace_back(child);
 }
 
 bool CountStage::isEOF() {
-    if (_specificStats.recordStoreCount) {
-        return true;
-    }
-
     if (_params.limit > 0 && _specificStats.nCounted >= _params.limit) {
         return true;
     }
 
-    return !_children.empty() && child()->isEOF();
-}
-
-void CountStage::recordStoreCount() {
-    invariant(_collection);
-    long long nCounted = _collection->numRecords(getOpCtx());
-
-    if (0 != _params.skip) {
-        nCounted -= _params.skip;
-        if (nCounted < 0) {
-            nCounted = 0;
-        }
-    }
-
-    long long limit = _params.limit;
-    if (limit < 0) {
-        limit = -limit;
-    }
-
-    if (limit < nCounted && 0 != limit) {
-        nCounted = limit;
-    }
-
-    _specificStats.nCounted = nCounted;
-    _specificStats.nSkipped = _params.skip;
-    _specificStats.recordStoreCount = true;
+    return child()->isEOF();
 }
 
 PlanStage::StageState CountStage::doWork(WorkingSetID* out) {
     // This stage never returns a working set member.
     *out = WorkingSet::INVALID_ID;
 
-    if (_params.useRecordStoreCount) {
-        invariant(_collection);
-        recordStoreCount();
-        return PlanStage::IS_EOF;
-    }
-
     if (isEOF()) {
         _commonStats.isEOF = true;
         return PlanStage::IS_EOF;
diff --git a/src/mongo/db/exec/count.h b/src/mongo/db/exec/count.h
index 23e513e33e3..791a8254213 100644
--- a/src/mongo/db/exec/count.h
+++ b/src/mongo/db/exec/count.h
@@ -30,18 +30,14 @@
 
 #pragma once
 
-
 #include "mongo/db/exec/plan_stage.h"
 #include "mongo/db/query/count_request.h"
 
 namespace mongo {
 
 struct CountStageParams {
-    CountStageParams(const CountRequest& request, bool useRecordStoreCount)
-        : nss(request.getNs()),
-          limit(request.getLimit()),
-          skip(request.getSkip()),
-          useRecordStoreCount(useRecordStoreCount) {}
+    CountStageParams(const CountRequest& request)
+        : nss(request.getNs()), limit(request.getLimit()), skip(request.getSkip()) {}
 
     // Namespace to operate on (e.g. "foo.bar").
     NamespaceString nss;
@@ -51,25 +47,18 @@ struct CountStageParams {
 
     // An integer indicating to not include the first n documents in the count. 0 means no skip.
     long long skip;
-
-    // True if this count stage should just ask the record store for a count instead of computing
-    // one itself.
-    //
-    // Note: This strategy can lead to inaccurate counts on certain storage engines (including
-    // WiredTiger).
-    bool useRecordStoreCount;
 };
 
 /**
- * Stage used by the count command. This stage sits at the root of a plan tree
- * and counts the number of results returned by its child stage.
+ * Stage used by the count command. This stage sits at the root of a plan tree and counts the number
+ * of results returned by its child stage.
  *
- * This should not be confused with the CountScan stage. CountScan is a special
- * index access stage which can optimize index access for count operations in
- * some cases. On the other hand, *every* count op has a CountStage at its root.
+ * This should not be confused with the CountScan stage. CountScan is a special index access stage
+ * which can optimize index access for count operations in some cases. On the other hand, *every*
+ * count op has a CountStage at its root.
  *
- * Only returns NEED_TIME until hitting EOF. The count result can be obtained by examining
- * the specific stats.
+ * Only returns NEED_TIME until hitting EOF. The count result can be obtained by examining the
+ * specific stats.
  */
 class CountStage final : public PlanStage {
 public:
@@ -93,17 +82,6 @@ public:
     static const char* kStageType;
 
 private:
-    /**
-     * Asks the record store for the count, applying the skip and limit if necessary. The result is
-     * stored in '_specificStats'.
-     *
-     * This is only valid if the query and hint are both empty.
-     */
-    void recordStoreCount();
-
-    // The collection over which we are counting.
-    Collection* _collection;
-
     CountStageParams _params;
 
     // The number of documents that we still need to skip.
diff --git a/src/mongo/db/exec/merge_sort.cpp b/src/mongo/db/exec/merge_sort.cpp
index 753461a5e13..79c37bb9449 100644
--- a/src/mongo/db/exec/merge_sort.cpp
+++ b/src/mongo/db/exec/merge_sort.cpp
@@ -50,10 +50,8 @@ const char* MergeSortStage::kStageType = "SORT_MERGE";
 
 MergeSortStage::MergeSortStage(OperationContext* opCtx,
                                const MergeSortStageParams& params,
-                               WorkingSet* ws,
-                               const Collection* collection)
+                               WorkingSet* ws)
     : PlanStage(kStageType, opCtx),
-      _collection(collection),
       _ws(ws),
       _pattern(params.pattern),
       _collator(params.collator),
diff --git a/src/mongo/db/exec/merge_sort.h b/src/mongo/db/exec/merge_sort.h
index d5683cc76b0..127e87b4877 100644
--- a/src/mongo/db/exec/merge_sort.h
+++ b/src/mongo/db/exec/merge_sort.h
@@ -58,10 +58,7 @@ class MergeSortStageParams;
  */
 class MergeSortStage final : public PlanStage {
 public:
-    MergeSortStage(OperationContext* opCtx,
-                   const MergeSortStageParams& params,
-                   WorkingSet* ws,
-                   const Collection* collection);
+    MergeSortStage(OperationContext* opCtx, const MergeSortStageParams& params, WorkingSet* ws);
 
     void addChild(PlanStage* child);
 
@@ -107,9 +104,6 @@ private:
     };
 
     // Not owned by us.
-    const Collection* _collection;
-
-    // Not owned by us.
     WorkingSet* _ws;
 
     // The pattern that we're sorting by.
diff --git a/src/mongo/db/exec/pipeline_proxy.h b/src/mongo/db/exec/pipeline_proxy.h
index 0cef0826526..9520b2139eb 100644
--- a/src/mongo/db/exec/pipeline_proxy.h
+++ b/src/mongo/db/exec/pipeline_proxy.h
@@ -33,7 +33,6 @@
 #include <boost/intrusive_ptr.hpp>
 #include <boost/optional/optional.hpp>
 
-#include "mongo/db/catalog/collection.h"
 #include "mongo/db/exec/plan_stage.h"
 #include "mongo/db/exec/plan_stats.h"
 #include "mongo/db/pipeline/pipeline.h"
diff --git a/src/mongo/db/exec/plan_stats.h b/src/mongo/db/exec/plan_stats.h
index ff03efa5e61..e0be0212b0b 100644
--- a/src/mongo/db/exec/plan_stats.h
+++ b/src/mongo/db/exec/plan_stats.h
@@ -204,7 +204,7 @@ struct CollectionScanStats : public SpecificStats {
 };
 
 struct CountStats : public SpecificStats {
-    CountStats() : nCounted(0), nSkipped(0), recordStoreCount(false) {}
+    CountStats() : nCounted(0), nSkipped(0) {}
 
     SpecificStats* clone() const final {
         CountStats* specific = new CountStats(*this);
@@ -216,9 +216,6 @@ struct CountStats : public SpecificStats {
 
     // The number of results we skipped over.
     long long nSkipped;
-
-    // True if we computed the count via Collection::numRecords().
-    bool recordStoreCount;
 };
 
 struct CountScanStats : public SpecificStats {
@@ -345,18 +342,6 @@ struct FetchStats : public SpecificStats {
     size_t docsExamined = 0u;
 };
 
-struct GroupStats : public SpecificStats {
-    GroupStats() : nGroups(0) {}
-
-    SpecificStats* clone() const final {
-        GroupStats* specific = new GroupStats(*this);
-        return specific;
-    }
-
-    // The total number of groups.
-    size_t nGroups;
-};
-
 struct IDHackStats : public SpecificStats {
     IDHackStats() : keysExamined(0), docsExamined(0) {}
 
diff --git a/src/mongo/db/exec/record_store_fast_count.cpp b/src/mongo/db/exec/record_store_fast_count.cpp
new file mode 100644
index 00000000000..392a8e09033
--- /dev/null
+++ b/src/mongo/db/exec/record_store_fast_count.cpp
@@ -0,0 +1,82 @@
+/**
+ *    Copyright (C) 2018-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/exec/record_store_fast_count.h"
+
+namespace mongo {
+
+const char* RecordStoreFastCountStage::kStageType = "RECORD_STORE_FAST_COUNT";
+
+RecordStoreFastCountStage::RecordStoreFastCountStage(OperationContext* opCtx,
+                                                     Collection* collection,
+                                                     long long skip,
+                                                     long long limit)
+    : RequiresCollectionStage(kStageType, opCtx, collection), _skip(skip), _limit(limit) {
+    invariant(_skip >= 0);
+    invariant(_limit >= 0);
+}
+
+std::unique_ptr<PlanStageStats> RecordStoreFastCountStage::getStats() {
+    auto planStats = std::make_unique<PlanStageStats>(_commonStats, STAGE_RECORD_STORE_FAST_COUNT);
+    planStats->specific = std::make_unique<CountStats>(_specificStats);
+    return planStats;
+}
+
+PlanStage::StageState RecordStoreFastCountStage::doWork(WorkingSetID* out) {
+    // This stage never returns a working set member.
+    *out = WorkingSet::INVALID_ID;
+
+    long long nCounted = collection()->numRecords(getOpCtx());
+
+    if (_skip) {
+        nCounted -= _skip;
+        if (nCounted < 0) {
+            nCounted = 0;
+        }
+    }
+
+    long long limit = _limit;
+    if (limit < 0) {
+        limit = -limit;
+    }
+
+    if (limit < nCounted && 0 != limit) {
+        nCounted = limit;
+    }
+
+    _specificStats.nCounted = nCounted;
+    _specificStats.nSkipped = _skip;
+    _commonStats.isEOF = true;
+
+    return PlanStage::IS_EOF;
+}
+
+}  // namespace mongo
diff --git a/src/mongo/db/exec/record_store_fast_count.h b/src/mongo/db/exec/record_store_fast_count.h
new file mode 100644
index 00000000000..b3578ae5f30
--- /dev/null
+++ b/src/mongo/db/exec/record_store_fast_count.h
@@ -0,0 +1,78 @@
+/**
+ *    Copyright (C) 2018-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/db/exec/requires_collection_stage.h"
+
+namespace mongo {
+
+/**
+ * Implements "fast count" by asking the underlying RecordStore for its number of records, applying
+ * the skip and limit it necessary. The result is stored in '_specificStats'. Only used to answer
+ * count commands when both the query and hint are empty.
+ */
+class RecordStoreFastCountStage final : public RequiresCollectionStage {
+public:
+    static const char* kStageType;
+
+    RecordStoreFastCountStage(OperationContext* opCtx,
+                              Collection* collection,
+                              long long skip,
+                              long long limit);
+
+    bool isEOF() override {
+        return _commonStats.isEOF;
+    }
+
+    StageState doWork(WorkingSetID* out) override;
+
+    StageType stageType() const override {
+        return StageType::STAGE_RECORD_STORE_FAST_COUNT;
+    }
+
+    std::unique_ptr<PlanStageStats> getStats() override;
+
+    const SpecificStats* getSpecificStats() const override {
+        return &_specificStats;
+    }
+
+protected:
+    void saveState(RequiresCollTag) override {}
+
+    void restoreState(RequiresCollTag) override {}
+
+private:
+    long long _skip = 0;
+    long long _limit = 0;
+
+    CountStats _specificStats;
+};
+
+}  // namepace mongo
diff --git a/src/mongo/db/exec/sort.cpp b/src/mongo/db/exec/sort.cpp
index 16b41f7b5ea..918f2a873a1 100644
--- a/src/mongo/db/exec/sort.cpp
+++ b/src/mongo/db/exec/sort.cpp
@@ -74,7 +74,6 @@ SortStage::SortStage(OperationContext* opCtx,
                      WorkingSet* ws,
                      PlanStage* child)
     : PlanStage(kStageType, opCtx),
-      _collection(params.collection),
       _ws(ws),
       _pattern(params.pattern),
       _limit(params.limit),
diff --git a/src/mongo/db/exec/sort.h b/src/mongo/db/exec/sort.h
index e58f8567447..f8bf3c4943d 100644
--- a/src/mongo/db/exec/sort.h
+++ b/src/mongo/db/exec/sort.h
@@ -48,16 +48,13 @@ class BtreeKeyGenerator;
 // Parameters that must be provided to a SortStage
 class SortStageParams {
 public:
-    SortStageParams() : collection(NULL), limit(0) {}
-
-    // Used for resolving RecordIds to BSON
-    const Collection* collection;
+    SortStageParams() = default;
 
     // How we're sorting.
     BSONObj pattern;
 
     // Equal to 0 for no limit.
-    size_t limit;
+    size_t limit = 0;
 };
 
 /**
@@ -95,9 +92,6 @@ private:
     //
 
     // Not owned by us.
-    const Collection* _collection;
-
-    // Not owned by us.
     WorkingSet* _ws;
 
     // The raw sort _pattern as expressed by the user
diff --git a/src/mongo/db/exec/stagedebug_cmd.cpp b/src/mongo/db/exec/stagedebug_cmd.cpp
index 2cb843ea293..df13e4a5985 100644
--- a/src/mongo/db/exec/stagedebug_cmd.cpp
+++ b/src/mongo/db/exec/stagedebug_cmd.cpp
@@ -316,7 +316,7 @@ public:
             uassert(
                 16921, "Nodes argument must be provided to AND", nodeArgs["nodes"].isABSONObj());
 
-            auto andStage = make_unique<AndHashStage>(opCtx, workingSet, collection);
+            auto andStage = make_unique<AndHashStage>(opCtx, workingSet);
 
             int nodesAdded = 0;
             BSONObjIterator it(nodeArgs["nodes"].Obj());
@@ -339,7 +339,7 @@ public:
             uassert(
                 16924, "Nodes argument must be provided to AND", nodeArgs["nodes"].isABSONObj());
 
-            auto andStage = make_unique<AndSortedStage>(opCtx, workingSet, collection);
+            auto andStage = make_unique<AndSortedStage>(opCtx, workingSet);
 
             int nodesAdded = 0;
             BSONObjIterator it(nodeArgs["nodes"].Obj());
@@ -424,21 +424,7 @@ public:
             }
 
             return new CollectionScan(opCtx, collection, params, workingSet, matcher);
-        }
-// sort is disabled for now.
-#if 0
-            else if ("sort" == nodeName) {
-                uassert(16969, "Node argument must be provided to sort",
-                        nodeArgs["node"].isABSONObj());
-                uassert(16970, "Pattern argument must be provided to sort",
-                        nodeArgs["pattern"].isABSONObj());
-                PlanStage* subNode = parseQuery(opCtx, db, nodeArgs["node"].Obj(), workingSet, exprs);
-                SortStageParams params;
-                params.pattern = nodeArgs["pattern"].Obj();
-                return new SortStage(params, workingSet, subNode);
-            }
-#endif
-        else if ("mergeSort" == nodeName) {
+        } else if ("mergeSort" == nodeName) {
             uassert(
                 16971, "Nodes argument must be provided to sort", nodeArgs["nodes"].isABSONObj());
             uassert(16972,
@@ -449,7 +435,7 @@ public:
             params.pattern = nodeArgs["pattern"].Obj();
             // Dedup is true by default.
 
-            auto mergeStage = make_unique<MergeSortStage>(opCtx, params, workingSet, collection);
+            auto mergeStage = make_unique<MergeSortStage>(opCtx, params, workingSet);
 
             BSONObjIterator it(nodeArgs["nodes"].Obj());
             while (it.more()) {