SERVER-66583 Re-enable SBE as the default execution engine and fall back to classic if no SBE compatible $group or $lookup exists

(cherry picked from commit aa40f5e)

16 files changed, 80 insertions, 156 deletions

diff --git a/jstests/noPassthrough/external_sort_find.js b/jstests/noPassthrough/external_sort_find.js
index a1505f129a3..befac93c5dc 100644
--- a/jstests/noPassthrough/external_sort_find.js
+++ b/jstests/noPassthrough/external_sort_find.js
@@ -21,7 +21,7 @@ assert.neq(null, conn, "mongod was unable to start up with options: " + tojson(o
 
 const testDb = conn.getDB("test");
 const collection = testDb.external_sort_find;
-const isSBEEnabled = checkSBEEnabled(testDb);
+const isSBEEnabled = checkSBEEnabled(testDb, ["featureFlagSbeFull"]);
 
 // Construct a document that is just over 1 kB.
 const charToRepeat = "-";
diff --git a/jstests/noPassthrough/log_format_slowms_samplerate_loglevel.js b/jstests/noPassthrough/log_format_slowms_samplerate_loglevel.js
index 856e752f627..5f932287b5d 100644
--- a/jstests/noPassthrough/log_format_slowms_samplerate_loglevel.js
+++ b/jstests/noPassthrough/log_format_slowms_samplerate_loglevel.js
@@ -86,7 +86,7 @@ function runLoggingTests({db, slowMs, logLevel, sampleRate}) {
     assert.commandWorked(db.setLogLevel(logLevel, "command"));
     assert.commandWorked(db.setLogLevel(logLevel, "write"));
 
-    const isSBEEnabled = checkSBEEnabled(db);
+    const isSBEEnabled = checkSBEEnabled(db, ["featureFlagSbeFull"]);
 
     // Certain fields in the log lines on mongoD are not applicable in their counterparts on
     // mongoS, and vice-versa. Ignore these fields when examining the logs of an instance on
diff --git a/jstests/noPassthrough/lookup_pushdown.js b/jstests/noPassthrough/lookup_pushdown.js
index 475ab5669f7..ffa5701b50f 100644
--- a/jstests/noPassthrough/lookup_pushdown.js
+++ b/jstests/noPassthrough/lookup_pushdown.js
@@ -294,6 +294,48 @@ function setLookupPushdownDisabled(value) {
             {allowDiskUse: true});
 }());
 
+// Verify that SBE is only used when a $lookup or a $group is present.
+(function testLookupGroupIsRequiredForPushdown() {
+    // Don't execute this test case if SBE is fully enabled.
+    if (checkSBEEnabled(db, ["featureFlagSbeFull"])) {
+        jsTestLog("Skipping test case because we are supporting SBE beyond $group and $lookup" +
+                  " pushdown");
+        return;
+    }
+
+    const assertEngineUsed = function(pipeline, isSBE) {
+        const explain = coll.explain().aggregate(pipeline);
+        assert(explain.hasOwnProperty("explainVersion"), explain);
+        if (isSBE) {
+            assert.eq(explain.explainVersion, 2, explain);
+        } else {
+            assert.eq(explain.explainVersion, 1, explain);
+        }
+    };
+
+    const lookup = {$lookup: {from: "coll", localField: "a", foreignField: "b", as: "out"}};
+    const group = {
+        $group: {
+            _id: "$a",
+            out: {$min: "$b"},
+        }
+    };
+    const match = {$match: {a: 1}};
+
+    // $lookup and $group should each run in SBE.
+    assertEngineUsed([lookup], true /* isSBE */);
+    assertEngineUsed([group], true /* isSBE */);
+    assertEngineUsed([lookup, group], true /* isSBE */);
+
+    // $match on its own won't use SBE, nor will an empty pipeline.
+    assertEngineUsed([match], false /* isSBE */);
+    assertEngineUsed([], false /* isSBE */);
+
+    // $match will use SBE if followed by either a $group or a $lookup.
+    assertEngineUsed([match, lookup], true /* isSBE */);
+    assertEngineUsed([match, group], true /* isSBE */);
+})();
+
 // Build an index on the foreign collection that matches the foreignField. This should cause us
 // to choose an indexed nested loop join.
 (function testIndexNestedLoopJoinRegularIndex() {
diff --git a/jstests/noPassthrough/plan_cache_index_create.js b/jstests/noPassthrough/plan_cache_index_create.js
index 3ce1731b847..862a46b7c64 100644
--- a/jstests/noPassthrough/plan_cache_index_create.js
+++ b/jstests/noPassthrough/plan_cache_index_create.js
@@ -178,7 +178,7 @@ rst.initiate();
 const primaryDB = rst.getPrimary().getDB(dbName);
 const secondaryDB = rst.getSecondary().getDB(dbName);
 
-if (checkSBEEnabled(primaryDB, ["featureFlagSbePlanCache"])) {
+if (checkSBEEnabled(primaryDB, ["featureFlagSbePlanCache", "featureFlagSbeFull"])) {
     jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
     rst.stopSet();
     return;
diff --git a/jstests/noPassthrough/plan_cache_list_failed_plans.js b/jstests/noPassthrough/plan_cache_list_failed_plans.js
index 696237a3248..853680f1872 100644
--- a/jstests/noPassthrough/plan_cache_list_failed_plans.js
+++ b/jstests/noPassthrough/plan_cache_list_failed_plans.js
@@ -8,9 +8,9 @@ const conn = MongoRunner.runMongod();
 assert.neq(null, conn, "mongod was unable to start up");
 const testDB = conn.getDB("jstests_plan_cache_list_failed_plans");
 const coll = testDB.test;
-const isSBEEnabled = checkSBEEnabled(testDB);
+const isSBEEnabled = checkSBEEnabled(testDB, ["featureFlagSbeFull"]);
 
-if (checkSBEEnabled(testDB, ["featureFlagSbePlanCache"])) {
+if (checkSBEEnabled(testDB, ["featureFlagSbePlanCache", "featureFlagSbeFull"])) {
     jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
     MongoRunner.stopMongod(conn);
     return;
diff --git a/jstests/noPassthrough/plan_cache_memory_debug_info.js b/jstests/noPassthrough/plan_cache_memory_debug_info.js
index 5f4b9b20678..e7c36ac4fff 100644
--- a/jstests/noPassthrough/plan_cache_memory_debug_info.js
+++ b/jstests/noPassthrough/plan_cache_memory_debug_info.js
@@ -79,7 +79,7 @@ assert.neq(conn, null, "mongod failed to start");
 const db = conn.getDB("test");
 const coll = db.plan_cache_memory_debug_info;
 
-if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache", "featureFlagSbeFull"])) {
     jsTest.log("Skipping test because SBE and SBE plan cache are both enabled.");
     MongoRunner.stopMongod(conn);
     return;
diff --git a/jstests/noPassthrough/plan_cache_replan_sort.js b/jstests/noPassthrough/plan_cache_replan_sort.js
index 55398c422ea..e905c1a4425 100644
--- a/jstests/noPassthrough/plan_cache_replan_sort.js
+++ b/jstests/noPassthrough/plan_cache_replan_sort.js
@@ -44,7 +44,7 @@ assert.eq(1, cachedPlans.length, cachedPlans);
 assert.eq(true, cachedPlans[0].isActive, cachedPlans);
 const cachedPlan = getCachedPlan(cachedPlans[0].cachedPlan);
 const cachedPlanVersion = cachedPlans[0].version;
-if (checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+if (checkSBEEnabled(db, ["featureFlagSbePlanCache", "featureFlagSbeFull"])) {
     // If the SBE plan cache is on, then the cached plan has a different format.
     assert.eq(cachedPlanVersion, "2", cachedPlans);
     assert(cachedPlan.stages.includes("sort"), cachedPlans);
diff --git a/jstests/noPassthrough/plan_cache_stats_agg_source.js b/jstests/noPassthrough/plan_cache_stats_agg_source.js
index 250a8a7d307..c2cef0bf538 100644
--- a/jstests/noPassthrough/plan_cache_stats_agg_source.js
+++ b/jstests/noPassthrough/plan_cache_stats_agg_source.js
@@ -16,7 +16,7 @@ const coll = testDb.plan_cache_stats_agg_source;
 // Note that the "getParameter" command is expected to fail in versions of mongod that do not yet
 // include the slot-based execution engine. When that happens, however, 'isSBEEnabled' still
 // correctly evaluates to false.
-const isSBEEnabled = checkSBEEnabled(testDb);
+const isSBEEnabled = checkSBEEnabled(testDb, ["featureFlagSbeFull"]);
 
 function makeMatchForFilteringByShape(query) {
     const keyHash = getPlanCacheKeyFromShape({query: query, collection: coll, db: testDb});
diff --git a/jstests/noPassthrough/query_engine_stats.js b/jstests/noPassthrough/query_engine_stats.js
index 492b4792efe..250943806d1 100644
--- a/jstests/noPassthrough/query_engine_stats.js
+++ b/jstests/noPassthrough/query_engine_stats.js
@@ -8,11 +8,20 @@
 
 // For 'getLatestProfilerEntry()'.
 load("jstests/libs/profiler.js");
+load("jstests/libs/sbe_util.js");  // For 'checkSBEEnabled()'.
 
 const conn = MongoRunner.runMongod({});
 assert.neq(null, conn, "mongod was unable to start up");
 
 const db = conn.getDB(jsTestName());
+
+// This test assumes that SBE is being used for most queries.
+if (!checkSBEEnabled(db, ["featureFlagSbeFull"])) {
+    jsTestLog("Skipping test because SBE is not fully enabled");
+    MongoRunner.stopMongod(conn);
+    return;
+}
+
 assert.commandWorked(db.dropDatabase());
 
 const coll = db.collection;
diff --git a/jstests/noPassthrough/query_oplogreplay.js b/jstests/noPassthrough/query_oplogreplay.js
index f59874e0c29..4fba7c108b7 100644
--- a/jstests/noPassthrough/query_oplogreplay.js
+++ b/jstests/noPassthrough/query_oplogreplay.js
@@ -6,7 +6,6 @@
 
 load("jstests/libs/analyze_plan.js");
 load("jstests/libs/storage_engine_utils.js");
-load("jstests/libs/sbe_util.js");  // For checkSBEEnabled.
 
 let replSet = new ReplSetTest({nodes: 1});
 replSet.startSet();
diff --git a/jstests/noPassthrough/sbe_multiplanner_trial_termination.js b/jstests/noPassthrough/sbe_multiplanner_trial_termination.js
index 75739587a9f..e429ae865ad 100644
--- a/jstests/noPassthrough/sbe_multiplanner_trial_termination.js
+++ b/jstests/noPassthrough/sbe_multiplanner_trial_termination.js
@@ -6,6 +6,8 @@
 (function() {
 "use strict";
 
+load("jstests/libs/sbe_util.js");  // For 'checkSBEEnabled()'.
+
 const numDocs = 1000;
 const dbName = "sbe_multiplanner_db";
 const collName = "sbe_multiplanner_coll";
@@ -21,6 +23,13 @@ const trialLengthFromWorksKnob = 0.1 * numDocs;
 const conn = MongoRunner.runMongod({});
 assert.neq(conn, null, "mongod failed to start");
 const db = conn.getDB(dbName);
+
+// This test assumes that SBE is being used for most queries.
+if (!checkSBEEnabled(db, ["featureFlagSbeFull"])) {
+    jsTestLog("Skipping test because SBE is not fully enabled");
+    MongoRunner.stopMongod(conn);
+    return;
+}
 const coll = db[collName];
 
 // Gets the "allPlansExecution" section from the explain of a query that has zero results, but for
diff --git a/jstests/noPassthrough/sbe_pipeline_plan_cache_key_reporting.js b/jstests/noPassthrough/sbe_pipeline_plan_cache_key_reporting.js
deleted file mode 100644
index 5c3a0aa5401..00000000000
--- a/jstests/noPassthrough/sbe_pipeline_plan_cache_key_reporting.js
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Confirms that 'planCacheKey' and 'queryHash' are correctly reported when the query has $lookup
- * and $query stages with enabled and disabled SBE Plan Cache.
- */
-
-(function() {
-"use strict";
-
-load("jstests/libs/sbe_util.js");  // For checkSBEEnabled.
-
-const databaseName = "pipeline_plan_cache_key_reporting";
-
-function isSBEEnabled() {
-    const conn = MongoRunner.runMongod({});
-    try {
-        const db = conn.getDB(databaseName);
-        return checkSBEEnabled(db);
-    } finally {
-        MongoRunner.stopMongod(conn);
-    }
-}
-
-if (!isSBEEnabled()) {
-    jsTest.log("Skipping test because SBE is not enabled.");
-    return;
-}
-
-/**
- * Driver function that creates mongod instances with specified parameters and run the given test
- * cases.
- * @param {*} params to be passed to mongod in format like { setParameter:
- *     "featureFlagSbePlanCache=true"}
- * @param {*} testCases a list of test cases where each test case is an object with 'setup(db)' and
- *     'run(db, assertMessage)' functions.
- * @returns results from 'testCase.run(db, assertMessage)'
- */
-function runTests(params, testCases) {
-    let results = [];
-    const conn = MongoRunner.runMongod(params);
-    const db = conn.getDB(databaseName);
-
-    const assertMessage = `${tojson(params)}`;
-    try {
-        for (let testCase of testCases) {
-            testCase.setup(db);
-            results.push(testCase.run(db, assertMessage));
-        }
-    } finally {
-        MongoRunner.stopMongod(conn);
-    }
-    return results;
-}
-
-/**
- * This function validates given explain and return and object with extracted and validated
- * PlanCacheKey and QueryHash.
- * @returns {planCacheKey, queryHash, explain}
- */
-function processAndValidateExplain(explain, assertMessage) {
-    assert.neq(explain, null);
-    assert.eq(explain.explainVersion,
-              "2",
-              `[${assertMessage}] invalid explain version ${tojson(explain)}`);
-
-    const planCacheKey = explain.queryPlanner.planCacheKey;
-    validateKey(planCacheKey, `[${assertMessage}] Invalid planCacheKey: ${tojson(explain)}`);
-
-    const queryHash = explain.queryPlanner.queryHash;
-    validateKey(queryHash, `[${assertMessage}] Invalid queryHash: ${tojson(explain)}`);
-
-    return {planCacheKey, queryHash, explain};
-}
-
-/**
- * Validates given 'key' (PlanCacheKey or QueryHash).
- */
-function validateKey(key, assertMessage) {
-    assert.eq(typeof key, "string", assertMessage);
-    assert.gt(key.length, 0, assertMessage);
-}
-
-// 1. Create test cases for $lookup and $group stages.
-const lookupTestCase = {
-    setup: db => {
-        db.coll.drop();
-        assert.commandWorked(db.coll.createIndexes([{a: 1}, {a: 1, b: 1}]));
-
-        db.lookupColl.drop();
-        assert.commandWorked(db.lookupColl.createIndex({b: 1}));
-    },
-
-    run: (db, assertMessage) => {
-        const pipeline = [
-            {$lookup: {from: db.lookupColl.getName(), localField: "a", foreignField: "b", as: "w"}}
-        ];
-        const explain = db.coll.explain().aggregate(pipeline);
-        return processAndValidateExplain(explain, assertMessage);
-    },
-};
-
-const groupTestCase = {
-    setup: db => {
-        db.coll.drop();
-        assert.commandWorked(db.coll.insertOne({a: 1}));
-    },
-
-    run: (db, assertMessage) => {
-        const pipeline = [{
-            $group: {
-                _id: "$b",
-            }
-        }];
-        const explain = db.coll.explain().aggregate(pipeline);
-        return processAndValidateExplain(explain, assertMessage);
-    },
-};
-
-const testCases = [lookupTestCase, groupTestCase];
-
-// 2. Run the test cases with SBE Plan Cache Enabled.
-const sbeParams = {
-    setParameter: "featureFlagSbePlanCache=true"
-};
-const sbeKeys = runTests(sbeParams, testCases);
-assert.eq(testCases.length, sbeKeys.length);
-
-// 3. Run the test cases with SBE Plan Cache disabled.
-const classicParams = {
-    setParameter: "featureFlagSbePlanCache=false"
-};
-const classicKeys = runTests(classicParams, testCases);
-assert.eq(testCases.length, classicKeys.length);
-
-// 4. Validate that PlanCacheKeys and QueryHash are equal. They should be different once
-// SERVER-61507 is completed.
-for (let i = 0; i < sbeKeys.length; ++i) {
-    const sbe = sbeKeys[i];
-    const classic = classicKeys[i];
-
-    const message = `sbe=${tojson(sbe.explain)}, classic=${tojson(classic.explain)}`;
-    assert.eq(sbe.planCacheKey, classic.planCacheKey, message);
-    assert.eq(sbe.queryHash, classic.queryHash, message);
-}
-})();
diff --git a/jstests/noPassthrough/sbe_plan_cache_clear_on_param_change.js b/jstests/noPassthrough/sbe_plan_cache_clear_on_param_change.js
index 21ff98a1ae1..eecea2a2349 100644
--- a/jstests/noPassthrough/sbe_plan_cache_clear_on_param_change.js
+++ b/jstests/noPassthrough/sbe_plan_cache_clear_on_param_change.js
@@ -50,7 +50,7 @@ const db = conn.getDB(dbName);
 
 // This test is specifically verifying the behavior of the SBE plan cache. So if either the SBE plan
 // cache or SBE itself are disabled, bail out.
-if (!checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+if (!checkSBEEnabled(db, ["featureFlagSbePlanCache", "featureFlagSbeFull"])) {
     jsTestLog("Skipping test because either SBE engine or SBE plan cache are disabled");
     MongoRunner.stopMongod(conn);
     return;
diff --git a/jstests/noPassthrough/sbe_plan_cache_memory_debug_info.js b/jstests/noPassthrough/sbe_plan_cache_memory_debug_info.js
index df3ac0d3fd9..6a50b4db853 100644
--- a/jstests/noPassthrough/sbe_plan_cache_memory_debug_info.js
+++ b/jstests/noPassthrough/sbe_plan_cache_memory_debug_info.js
@@ -16,7 +16,7 @@ const conn = MongoRunner.runMongod({});
 assert.neq(conn, null, "mongod failed to start");
 const db = conn.getDB("sbe_plan_cache_memory_debug_info");
 
-if (!checkSBEEnabled(db, ["featureFlagSbePlanCache"])) {
+if (!checkSBEEnabled(db, ["featureFlagSbePlanCache", "featureFlagSbeFull"])) {
     jsTest.log("Skipping test because SBE plan cache is not enabled.");
     MongoRunner.stopMongod(conn);
     return;
diff --git a/jstests/noPassthrough/sbe_plan_cache_size_metric.js b/jstests/noPassthrough/sbe_plan_cache_size_metric.js
index 4cec6ba7f78..ebfe8d5f0bb 100644
--- a/jstests/noPassthrough/sbe_plan_cache_size_metric.js
+++ b/jstests/noPassthrough/sbe_plan_cache_size_metric.js
@@ -44,7 +44,8 @@ function assertQueryInPlanCache(coll, query) {
     assert.eq(1, planCacheEntries.length, planCacheEntries);
 }
 
-const isSbePlanCacheEnabled = checkSBEEnabled(db, ["featureFlagSbePlanCache"]);
+const isSbePlanCacheEnabled =
+    checkSBEEnabled(db, ["featureFlagSbePlanCache", "featureFlagSbeFull"]);
 if (isSbePlanCacheEnabled) {
     const collectionName = "plan_cache_sbe";
     const coll = db[collectionName];
diff --git a/jstests/noPassthrough/server_status_multiplanner.js b/jstests/noPassthrough/server_status_multiplanner.js
index f185cdb7841..42a7062e34c 100644
--- a/jstests/noPassthrough/server_status_multiplanner.js
+++ b/jstests/noPassthrough/server_status_multiplanner.js
@@ -13,6 +13,7 @@ function sumHistogramBucketCounts(histogram) {
 }
 
 load("jstests/libs/ftdc.js");
+load("jstests/libs/sbe_util.js");  // For 'checkSBEEnabled()'.
 
 const collName = jsTestName();
 const dbName = jsTestName();
@@ -22,6 +23,13 @@ const conn = MongoRunner.runMongod({});
 assert.neq(conn, null, "mongod failed to start");
 const db = conn.getDB(dbName);
 
+// This test assumes that SBE is being used for most queries.
+if (!checkSBEEnabled(db, ["featureFlagSbeFull"])) {
+    jsTestLog("Skipping test because SBE is not fully enabled");
+    MongoRunner.stopMongod(conn);
+    return;
+}
+
 let coll = db.getCollection(collName);
 coll.drop();