From 6afa8dd2bb3435e3632375a545d8a2c76a1cd620 Mon Sep 17 00:00:00 2001 From: David Storch Date: Thu, 30 Mar 2023 20:09:21 +0000 Subject: SERVER-72486 Enable all features supported in SBE by default 'featureFlagSbeFull' remains off by default but after this change has no function. We plan to re-use it in upcoming projects to flag-guard extensions to the SBE engine while they are under development. --- jstests/aggregation/expressions/concat_arrays.js | 2 +- jstests/aggregation/optimize_away_pipeline.js | 74 +++++----- jstests/auth/sbe_plan_cache_user_roles.js | 6 +- .../column_scan_skip_row_store_projection.js | 3 +- .../columnstore/column_store_index_compression.js | 3 +- .../core/columnstore/column_store_projection.js | 3 +- .../core/columnstore/columnstore_eligibility.js | 26 +--- jstests/core/columnstore/columnstore_index.js | 3 +- .../columnstore/columnstore_index_correctness.js | 3 +- .../columnstore_index_per_path_filters.js | 3 +- .../columnstore_large_array_index_correctness.js | 3 +- jstests/core/columnstore/columnstore_validindex.js | 1 - jstests/core/query/explode_for_sort_plan_cache.js | 8 +- .../sbe_plan_cache_autoparameterize_collscan.js | 58 ++++---- jstests/noPassthrough/analyze_command.js | 4 +- jstests/noPassthrough/cluster_analyze_command.js | 4 +- jstests/noPassthrough/column_scan_slow_logs.js | 1 - jstests/noPassthrough/column_store_index_load.js | 1 - .../noPassthrough/columnstore_index_persistence.js | 3 +- .../noPassthroughWithMongod/column_scan_explain.js | 3 +- .../columnstore_planning_heuristics.js | 3 +- jstests/noPassthroughWithMongod/group_pushdown.js | 54 +++----- jstests/noPassthroughWithMongod/now_variable.js | 2 +- .../noPassthroughWithMongod/sbe_agg_pushdown.js | 5 +- .../sbe_query_eligibility.js | 56 ++++++-- jstests/sharding/query/views.js | 29 ++-- src/mongo/db/query/get_executor.cpp | 150 +++++---------------- src/mongo/db/query/query_feature_flags.idl | 3 +- src/mongo/db/query/query_utils.cpp | 1 + 29 files changed, 206 insertions(+), 309 deletions(-) diff --git a/jstests/aggregation/expressions/concat_arrays.js b/jstests/aggregation/expressions/concat_arrays.js index a45637c1251..3c52f31cd08 100644 --- a/jstests/aggregation/expressions/concat_arrays.js +++ b/jstests/aggregation/expressions/concat_arrays.js @@ -135,7 +135,7 @@ runAndAssertThrows(["$dbl_arr", "$dbl_val"]); // Confirm edge case where if invalid input precedes null or missing inputs, the command fails. // Note that when the SBE engine is enabled, null will be returned before invalid input because // we check if any values are null before checking whether all values are arrays. -let evalFn = checkSBEEnabled(db, ["featureFlagSbeFull"]) ? runAndAssertNull : runAndAssertThrows; +let evalFn = checkSBEEnabled(db) ? runAndAssertNull : runAndAssertThrows; evalFn(["$int_arr", "$dbl_val", "$null_val"]); evalFn(["$int_arr", "some_string_value", "$null_val"]); evalFn(["$dbl_val", "$null_val"]); diff --git a/jstests/aggregation/optimize_away_pipeline.js b/jstests/aggregation/optimize_away_pipeline.js index 61d4c35f636..9a6cbe8c09e 100644 --- a/jstests/aggregation/optimize_away_pipeline.js +++ b/jstests/aggregation/optimize_away_pipeline.js @@ -657,47 +657,41 @@ pipeline = [{$group: {_id: "$a.b", s: {$first: "$a"}}}]; // assertProjectionCanBeRemovedBeforeGroup(pipeline, "PROJECTION_SIMPLE"); assertProjectionIsNotRemoved(pipeline); -// Though $group is generally eligible for pushdown into SBE, such a pushdown may be inhibited by -// dotted as well as computed projections. As such we only run the test cases below if SBE is fully -// enabled. -const sbeFull = checkSBEEnabled(db, ["featureFlagSbeFull"], true /* checkAllNodes */); -if (sbeFull) { - assertProjectionCanBeRemovedBeforeGroup( - [{$project: {'a.b': 1, 'b.c': 1}}, {$group: {_id: "$a.b", s: {$sum: "$b.c"}}}], - "PROJECTION_DEFAULT"); - - // Test that a computed projection at the front of the pipeline is pushed down, even if there's - // no finite dependency set. - pipeline = [{$project: {x: {$add: ["$a", 1]}}}]; - assertPipelineDoesNotUseAggregation( - {pipeline: pipeline, expectedStages: ["COLLSCAN", "PROJECTION_DEFAULT"]}); - - // The projections below are not removed because they fail to include the $group's dependencies. - assertProjectionIsNotRemoved([{$project: {'a.b': 1}}, {$group: {_id: "$a.b", s: {$sum: "$b"}}}], - "PROJECTION_DEFAULT"); - assertProjectionIsNotRemoved( - [{$project: {'a.b': 1}}, {$group: {_id: "$a.b", s: {$sum: "$a.c"}}}], "PROJECTION_DEFAULT"); - - pipeline = [{$project: {a: {$add: ["$a", 1]}}}, {$group: {_id: "$a", s: {$sum: "$b"}}}]; - assertPipelineIfGroupPushdown( - // Test that a computed projection at the front of the pipeline is pushed down when there's - // a finite dependency set. Additionally, the group pushdown shouldn't erase the computed - // projection. - function() { - explain = coll.explain().aggregate(pipeline); - assertPipelineDoesNotUseAggregation( - {pipeline: pipeline, expectedStages: ["COLLSCAN", "PROJECTION_DEFAULT", "GROUP"]}); - }, - // Test that a computed projection at the front of the pipeline is pushed down when there's - // a finite dependency set. - function() { - explain = coll.explain().aggregate(pipeline); - assertPipelineUsesAggregation({ - pipeline: pipeline, - expectedStages: ["COLLSCAN", "PROJECTION_DEFAULT", "$group"], - }); +assertProjectionCanBeRemovedBeforeGroup( + [{$project: {'a.b': 1, 'b.c': 1}}, {$group: {_id: "$a.b", s: {$sum: "$b.c"}}}], + "PROJECTION_DEFAULT"); + +// Test that a computed projection at the front of the pipeline is pushed down, even if there's no +// finite dependency set. +pipeline = [{$project: {x: {$add: ["$a", 1]}}}]; +assertPipelineDoesNotUseAggregation( + {pipeline: pipeline, expectedStages: ["COLLSCAN", "PROJECTION_DEFAULT"]}); + +// The projections below are not removed because they fail to include the $group's dependencies. +assertProjectionIsNotRemoved([{$project: {'a.b': 1}}, {$group: {_id: "$a.b", s: {$sum: "$b"}}}], + "PROJECTION_DEFAULT"); +assertProjectionIsNotRemoved([{$project: {'a.b': 1}}, {$group: {_id: "$a.b", s: {$sum: "$a.c"}}}], + "PROJECTION_DEFAULT"); + +pipeline = [{$project: {a: {$add: ["$a", 1]}}}, {$group: {_id: "$a", s: {$sum: "$b"}}}]; +assertPipelineIfGroupPushdown( + // Test that a computed projection at the front of the pipeline is pushed down when there's a + // finite dependency set. Additionally, the group pushdown shouldn't erase the computed + // projection. + function() { + explain = coll.explain().aggregate(pipeline); + assertPipelineDoesNotUseAggregation( + {pipeline: pipeline, expectedStages: ["COLLSCAN", "PROJECTION_DEFAULT", "GROUP"]}); + }, + // Test that a computed projection at the front of the pipeline is pushed down when there's a + // finite dependency set. + function() { + explain = coll.explain().aggregate(pipeline); + assertPipelineUsesAggregation({ + pipeline: pipeline, + expectedStages: ["COLLSCAN", "PROJECTION_DEFAULT", "$group"], }); -} + }); // We generate a projection stage from dependency analysis, even if the pipeline begins with an // exclusion projection. diff --git a/jstests/auth/sbe_plan_cache_user_roles.js b/jstests/auth/sbe_plan_cache_user_roles.js index a42e36bf372..8c4313a4ee5 100644 --- a/jstests/auth/sbe_plan_cache_user_roles.js +++ b/jstests/auth/sbe_plan_cache_user_roles.js @@ -15,6 +15,8 @@ const mongod = MongoRunner.runMongod(); const dbName = "test"; const db = mongod.getDB(dbName); +const sbeEnabled = checkSBEEnabled(db); + // Create two users, each with different roles. assert.commandWorked( db.runCommand({createUser: "user1", pwd: "pwd", roles: [{role: "read", db: dbName}]})); @@ -25,7 +27,7 @@ const coll = db.sbe_plan_cache_user_roles; coll.drop(); const verifyPlanCache = function(role) { - if (checkSBEEnabled(db, ["featureFlagSbeFull"])) { + if (sbeEnabled) { const caches = coll.getPlanCache().list(); assert.eq(1, caches.length, caches); assert.eq(caches[0].cachedPlan.stages.includes(role), false, caches); @@ -53,4 +55,4 @@ verifyPlanCache("test.readWrite"); db.logout(); MongoRunner.stopMongod(mongod); -})(); \ No newline at end of file +})(); diff --git a/jstests/core/columnstore/column_scan_skip_row_store_projection.js b/jstests/core/columnstore/column_scan_skip_row_store_projection.js index 29d1268d571..8cfee05c075 100644 --- a/jstests/core/columnstore/column_scan_skip_row_store_projection.js +++ b/jstests/core/columnstore/column_scan_skip_row_store_projection.js @@ -3,9 +3,8 @@ * above a columnscan stage. * * @tags: [ - * # Column store indexes are still under a feature flag and require full SBE. + * # Column store indexes are still under a feature flag. * featureFlagColumnstoreIndexes, - * featureFlagSbeFull, * # explain is not supported in transactions * does_not_support_transactions, * requires_pipeline_optimization, diff --git a/jstests/core/columnstore/column_store_index_compression.js b/jstests/core/columnstore/column_store_index_compression.js index 0e0b4358d40..f59a78d5696 100644 --- a/jstests/core/columnstore/column_store_index_compression.js +++ b/jstests/core/columnstore/column_store_index_compression.js @@ -5,9 +5,8 @@ * does_not_support_transactions, * requires_collstats, * - * # Column store indexes are still under a feature flag and require full SBE. + * # Column store indexes are still under a feature flag. * featureFlagColumnstoreIndexes, - * featureFlagSbeFull, * * # In passthrough suites, this test makes direct connections to mongod instances that compose * # the passthrough fixture in order to perform additional validation. Tenant migration, diff --git a/jstests/core/columnstore/column_store_projection.js b/jstests/core/columnstore/column_store_projection.js index c7cb1a0b2a7..65a3f78388f 100644 --- a/jstests/core/columnstore/column_store_projection.js +++ b/jstests/core/columnstore/column_store_projection.js @@ -2,9 +2,8 @@ * Test column stores indexes that use a "columnstoreProjection" or "prefix.$**" notation to limit * indexed data to a subset of the document namespace. * @tags: [ - * # Column store indexes are still under a feature flag and require full SBE. + * # Column store indexes are still under a feature flag. * featureFlagColumnstoreIndexes, - * featureFlagSbeFull, * # Runs explain on an aggregate command which is only compatible with readConcern local. * assumes_read_concern_unchanged, * # Columnstore tests set server parameters to disable columnstore query planning heuristics - diff --git a/jstests/core/columnstore/columnstore_eligibility.js b/jstests/core/columnstore/columnstore_eligibility.js index 19ff6a3ea57..0ae6e9b0e01 100644 --- a/jstests/core/columnstore/columnstore_eligibility.js +++ b/jstests/core/columnstore/columnstore_eligibility.js @@ -1,9 +1,8 @@ /** * Tests the eligibility of certain queries to use a columnstore index. * @tags: [ - * # Column store indexes are still under a feature flag and require full SBE. + * # Column store indexes are still under a feature flag. * featureFlagColumnstoreIndexes, - * featureFlagSbeFull, * # Refusing to run a test that issues an aggregation command with explain because it may return * # incomplete results if interrupted by a stepdown. * does_not_support_stepdowns, @@ -13,6 +12,9 @@ * # server parameters are stored in-memory only so are not transferred onto the recipient. * tenant_migration_incompatible, * not_allowed_with_security_token, + * # Logic for when a COLUMN_SCAN plan is generated changed slightly as part of enabling more + * # queries in SBE in the 7.0 release. + * requires_fcv_70, * ] */ (function() { @@ -21,7 +23,6 @@ load("jstests/libs/analyze_plan.js"); load("jstests/libs/columnstore_util.js"); // For setUpServerForColumnStoreIndexTest. load("jstests/libs/fixture_helpers.js"); // For FixtureHelpers.isMongos. -load("jstests/libs/sbe_util.js"); // For checkSBEEnabled. if (!setUpServerForColumnStoreIndexTest(db)) { return; @@ -69,33 +70,18 @@ explain = coll.find({$or: [{a: 2}, {b: 2}]}, {_id: 0, a: 1}).explain(); // COLUMN_SCAN is used for top-level $or queries. assert(planHasStage(db, explain, "COLUMN_SCAN"), explain); -// COLUMN_SCAN is only used for for certain top-level $or queries when sbeFull is also enabled due -// to a quirk in the engine selection logic. -const sbeFull = checkSBEEnabled(db, ["featureFlagSbeFull"]); explain = coll.explain().aggregate([ {$match: {$or: [{a: {$gt: 0}}, {b: {$gt: 0}}]}}, {$project: {_id: 0, computedField: {$add: ["$a", "$b"]}}}, ]); -let planHasColumnScan = planHasStage(db, explain, "COLUMN_SCAN"); - -if (sbeFull) { - assert(planHasColumnScan, explain); -} else { - assert(!planHasColumnScan, explain); -} +assert(planHasStage(db, explain, "COLUMN_SCAN"), explain); explain = coll.explain().aggregate([ {$match: {$or: [{a: {$gt: 0}}, {b: {$gt: 0}}]}}, {$project: {_id: 0, computedField: {$add: ["$a", "$b"]}}}, {$group: {_id: "$computedField"}} ]); -planHasColumnScan = planHasStage(db, explain, "COLUMN_SCAN"); - -if (sbeFull) { - assert(planHasColumnScan, explain); -} else { - assert(!planHasColumnScan, explain); -} +assert(planHasStage(db, explain, "COLUMN_SCAN"), explain); // Simplest case: just scan "a" column. explain = coll.find({a: {$exists: true}}, {_id: 0, a: 1}).explain(); diff --git a/jstests/core/columnstore/columnstore_index.js b/jstests/core/columnstore/columnstore_index.js index 8d68700593c..895f93ae93f 100644 --- a/jstests/core/columnstore/columnstore_index.js +++ b/jstests/core/columnstore/columnstore_index.js @@ -1,9 +1,8 @@ /** * Tests some basic use cases and functionality of a columnstore index. * @tags: [ - * # Column store indexes are still under a feature flag and require full SBE. + * # Column store indexes are still under a feature flag. * featureFlagColumnstoreIndexes, - * featureFlagSbeFull, * * # Uses $indexStats which is not supported inside a transaction. * does_not_support_transactions, diff --git a/jstests/core/columnstore/columnstore_index_correctness.js b/jstests/core/columnstore/columnstore_index_correctness.js index 66398ea26f5..f6fc1621fb3 100644 --- a/jstests/core/columnstore/columnstore_index_correctness.js +++ b/jstests/core/columnstore/columnstore_index_correctness.js @@ -1,9 +1,8 @@ /** * Testing of just the query layer's integration for columnar index. * @tags: [ - * # Column store indexes are still under a feature flag and require full SBE. + * # Column store indexes are still under a feature flag. * featureFlagColumnstoreIndexes, - * featureFlagSbeFull, * # Runs explain on an aggregate command which is only compatible with readConcern local. * assumes_read_concern_unchanged, * # Columnstore tests set server parameters to disable columnstore query planning heuristics - diff --git a/jstests/core/columnstore/columnstore_index_per_path_filters.js b/jstests/core/columnstore/columnstore_index_per_path_filters.js index c01556b797e..32b457c6b6a 100644 --- a/jstests/core/columnstore/columnstore_index_per_path_filters.js +++ b/jstests/core/columnstore/columnstore_index_per_path_filters.js @@ -3,9 +3,8 @@ * might be pushed down into the column scan stage. * * @tags: [ - * # Column store indexes are still under a feature flag and require full SBE. + * # Column store indexes are still under a feature flag. * featureFlagColumnstoreIndexes, - * featureFlagSbeFull, * # Runs explain on an aggregate command which is only compatible with readConcern local. * assumes_read_concern_unchanged, * # Columnstore tests set server parameters to disable columnstore query planning heuristics - diff --git a/jstests/core/columnstore/columnstore_large_array_index_correctness.js b/jstests/core/columnstore/columnstore_large_array_index_correctness.js index 5a0f06984d5..3c15b62e247 100644 --- a/jstests/core/columnstore/columnstore_large_array_index_correctness.js +++ b/jstests/core/columnstore/columnstore_large_array_index_correctness.js @@ -1,9 +1,8 @@ /** * Testing of just the query layer's integration for columnar indexes that encode large arrays. * @tags: [ - * # Column store indexes are still under a feature flag and require full SBE. + * # Column store indexes are still under a feature flag. * featureFlagColumnstoreIndexes, - * featureFlagSbeFull, * # Columnstore tests set server parameters to disable columnstore query planning heuristics - * # 1) server parameters are stored in-memory only so are not transferred onto the recipient, * # 2) server parameters may not be set in stepdown passthroughs because it is a command that may diff --git a/jstests/core/columnstore/columnstore_validindex.js b/jstests/core/columnstore/columnstore_validindex.js index 7b86e97edda..421ea4a6edc 100644 --- a/jstests/core/columnstore/columnstore_validindex.js +++ b/jstests/core/columnstore/columnstore_validindex.js @@ -2,7 +2,6 @@ * Tests parsing and validation of columnstore indexes. * @tags: [ * featureFlagColumnstoreIndexes, - * featureFlagSbeFull, * # Uses index building in background. * requires_background_index, * # Columnstore tests set server parameters to disable columnstore query planning heuristics - diff --git a/jstests/core/query/explode_for_sort_plan_cache.js b/jstests/core/query/explode_for_sort_plan_cache.js index 65bf1a3707e..53d343b35d8 100644 --- a/jstests/core/query/explode_for_sort_plan_cache.js +++ b/jstests/core/query/explode_for_sort_plan_cache.js @@ -29,7 +29,6 @@ load("jstests/libs/analyze_plan.js"); load("jstests/libs/sbe_util.js"); const isSBEEnabled = checkSBEEnabled(db); -const isSBEFullEnabled = checkSBEEnabled(db, ["featureFlagSbeFull"]); const coll = db.explode_for_sort_plan_cache; coll.drop(); @@ -147,9 +146,8 @@ assertExplodeForSortCacheParameterizedCorrectly({ newQuery: {$and: [{$expr: {$eq: ["$a", 2]}}, {b: {$in: [1, 2]}}]}, newQueryCount: 6, // The plan cache entry is always reused for the classic engine but never reused for the SBE - // engine. Whether this query uses SBE currently depends on the value of 'featureFlagSbeFull', - // since $expr is not yet enabled by default in SBE. - reuseEntry: !isSBEFullEnabled, + // engine. + reuseEntry: !isSBEEnabled, }); // Rewriting the $in predicate with $or should reuse the plan cache and gives correct results. @@ -218,4 +216,4 @@ assertExplodeForSortCacheParameterizedCorrectly({ newQueryCount: 0, reuseEntry: false, }); -}()); \ No newline at end of file +}()); diff --git a/jstests/core/sbe_plan_cache_autoparameterize_collscan.js b/jstests/core/sbe_plan_cache_autoparameterize_collscan.js index c7b47f077a3..c1aaea782be 100644 --- a/jstests/core/sbe_plan_cache_autoparameterize_collscan.js +++ b/jstests/core/sbe_plan_cache_autoparameterize_collscan.js @@ -7,8 +7,8 @@ * assumes_read_preference_unchanged, * assumes_unsharded_collection, * does_not_support_stepdowns, - * # The SBE plan cache was enabled by default in 6.3, some of the $in tests behave differently - * # in 7.0. + * # Auto-parameterization behavior observed by this test changed in 7.0 as a result of enabling + * # additional scenarios in SBE. * requires_fcv_70, * # Plan cache state is node-local and will not get migrated alongside tenant data. * tenant_migration_incompatible, @@ -306,35 +306,31 @@ runTest({query: {a: {$exists: true}}, projection: {_id: 1}}, false); // Test that comparisons expressed as $expr are not auto-parameterized. -if (checkSBEEnabled(db, ["featureFlagSbeFull"])) { - runTest({query: {$expr: {$eq: ["$a", 3]}}, projection: {_id: 1}}, - [{_id: 2}], - {query: {$expr: {$eq: ["$a", 4]}}, projection: {_id: 1}}, - [{_id: 3}, {_id: 4}], - false); - runTest({query: {$expr: {$lt: ["$a", 3]}, a: {$type: "number"}}, projection: {_id: 1}}, - [{_id: 0}, {_id: 1}], - {query: {$expr: {$lt: ["$a", 4]}, a: {$type: "number"}}, projection: {_id: 1}}, - [{_id: 0}, {_id: 1}, {_id: 2}], - false); - runTest({query: {$expr: {$lte: ["$a", 3]}, a: {$type: "number"}}, projection: {_id: 1}}, - [{_id: 0}, {_id: 1}, {_id: 2}], - {query: {$expr: {$lte: ["$a", 4]}, a: {$type: "number"}}, projection: {_id: 1}}, - [{_id: 0}, {_id: 1}, {_id: 2}, {_id: 3}, {_id: 4}], - false); - runTest({query: {$expr: {$gt: ["$a", 2]}, a: {$type: "number"}}, projection: {_id: 1}}, - [{_id: 2}, {_id: 3}, {_id: 4}, {_id: 5}, {_id: 6}], - {query: {$expr: {$gt: ["$a", 3]}, a: {$type: "number"}}, projection: {_id: 1}}, - [{_id: 3}, {_id: 4}, {_id: 5}, {_id: 6}], - false); - runTest({query: {$expr: {$gte: ["$a", 2]}, a: {$type: "number"}}, projection: {_id: 1}}, - [{_id: 1}, {_id: 2}, {_id: 3}, {_id: 4}, {_id: 5}, {_id: 6}], - {query: {$expr: {$gte: ["$a", 3]}, a: {$type: "number"}}, projection: {_id: 1}}, - [{_id: 2}, {_id: 3}, {_id: 4}, {_id: 5}, {_id: 6}], - false); -} else { - jsTestLog("Skipping $expr test cases because SBE is not fully enabled"); -} +runTest({query: {$expr: {$eq: ["$a", 3]}}, projection: {_id: 1}}, + [{_id: 2}], + {query: {$expr: {$eq: ["$a", 4]}}, projection: {_id: 1}}, + [{_id: 3}, {_id: 4}], + false); +runTest({query: {$expr: {$lt: ["$a", 3]}, a: {$type: "number"}}, projection: {_id: 1}}, + [{_id: 0}, {_id: 1}], + {query: {$expr: {$lt: ["$a", 4]}, a: {$type: "number"}}, projection: {_id: 1}}, + [{_id: 0}, {_id: 1}, {_id: 2}], + false); +runTest({query: {$expr: {$lte: ["$a", 3]}, a: {$type: "number"}}, projection: {_id: 1}}, + [{_id: 0}, {_id: 1}, {_id: 2}], + {query: {$expr: {$lte: ["$a", 4]}, a: {$type: "number"}}, projection: {_id: 1}}, + [{_id: 0}, {_id: 1}, {_id: 2}, {_id: 3}, {_id: 4}], + false); +runTest({query: {$expr: {$gt: ["$a", 2]}, a: {$type: "number"}}, projection: {_id: 1}}, + [{_id: 2}, {_id: 3}, {_id: 4}, {_id: 5}, {_id: 6}], + {query: {$expr: {$gt: ["$a", 3]}, a: {$type: "number"}}, projection: {_id: 1}}, + [{_id: 3}, {_id: 4}, {_id: 5}, {_id: 6}], + false); +runTest({query: {$expr: {$gte: ["$a", 2]}, a: {$type: "number"}}, projection: {_id: 1}}, + [{_id: 1}, {_id: 2}, {_id: 3}, {_id: 4}, {_id: 5}, {_id: 6}], + {query: {$expr: {$gte: ["$a", 3]}, a: {$type: "number"}}, projection: {_id: 1}}, + [{_id: 2}, {_id: 3}, {_id: 4}, {_id: 5}, {_id: 6}], + false); // Test that the same length of $in list generates the same plan cache key. runTest({query: {a: {$in: [1, 2]}}, projection: {_id: 1}}, diff --git a/jstests/noPassthrough/analyze_command.js b/jstests/noPassthrough/analyze_command.js index a6e8e7e6c29..122bca9fc13 100644 --- a/jstests/noPassthrough/analyze_command.js +++ b/jstests/noPassthrough/analyze_command.js @@ -8,8 +8,8 @@ assert.neq(null, conn, "mongod was unable to start up"); const db = conn.getDB(jsTestName()); -if (checkSBEEnabled(db, ["featureFlagSbeFull"], true)) { - jsTestLog("Skipping the test because it doesn't work in Full SBE..."); +if (!checkSBEEnabled(db)) { + jsTestLog("Skipping test because SBE is not enabled"); MongoRunner.stopMongod(conn); return; } diff --git a/jstests/noPassthrough/cluster_analyze_command.js b/jstests/noPassthrough/cluster_analyze_command.js index 463a4f2c0ac..8dea667053b 100644 --- a/jstests/noPassthrough/cluster_analyze_command.js +++ b/jstests/noPassthrough/cluster_analyze_command.js @@ -14,8 +14,8 @@ const st = new ShardingTest({ const db = st.getDB("test"); -if (checkSBEEnabled(db, ["featureFlagSbeFull"], true)) { - jsTestLog("Skipping the test because it doesn't work in Full SBE..."); +if (!checkSBEEnabled(db)) { + jsTestLog("Skipping test because SBE is not enabled"); st.stop(); return; } diff --git a/jstests/noPassthrough/column_scan_slow_logs.js b/jstests/noPassthrough/column_scan_slow_logs.js index eeaa8fe3618..b4fda3d4200 100644 --- a/jstests/noPassthrough/column_scan_slow_logs.js +++ b/jstests/noPassthrough/column_scan_slow_logs.js @@ -3,7 +3,6 @@ * the columstore index is the winning plan. * @tags: [ * featureFlagColumnstoreIndexes, - * featureFlagSbeFull, * ] */ (function() { diff --git a/jstests/noPassthrough/column_store_index_load.js b/jstests/noPassthrough/column_store_index_load.js index e37f84ec3d5..d73d2f97a06 100644 --- a/jstests/noPassthrough/column_store_index_load.js +++ b/jstests/noPassthrough/column_store_index_load.js @@ -8,7 +8,6 @@ * # yet implemented. * does_not_support_stepdowns, * featureFlagColumnstoreIndexes, - * featureFlagSbeFull, * ] */ (function() { diff --git a/jstests/noPassthrough/columnstore_index_persistence.js b/jstests/noPassthrough/columnstore_index_persistence.js index afdb7a235b3..e8e587ee280 100644 --- a/jstests/noPassthrough/columnstore_index_persistence.js +++ b/jstests/noPassthrough/columnstore_index_persistence.js @@ -6,9 +6,8 @@ * @tags: [ * requires_persistence, * requires_replication, - * # column store indexes are still under a feature flag and require full sbe + * # column store indexes are still under a feature flag. * featureFlagColumnstoreIndexes, - * featureFlagSbeFull, * ] */ diff --git a/jstests/noPassthroughWithMongod/column_scan_explain.js b/jstests/noPassthroughWithMongod/column_scan_explain.js index fe3e78190aa..43b7ee11a00 100644 --- a/jstests/noPassthroughWithMongod/column_scan_explain.js +++ b/jstests/noPassthroughWithMongod/column_scan_explain.js @@ -1,9 +1,8 @@ /** * Tests the explain support for the COLUMN_SCAN stage. * @tags: [ - * # Column store indexes are still under a feature flag and require full sbe. + * # Column store indexes are still under a feature flag. * featureFlagColumnstoreIndexes, - * featureFlagSbeFull, * ] */ (function() { diff --git a/jstests/noPassthroughWithMongod/columnstore_planning_heuristics.js b/jstests/noPassthroughWithMongod/columnstore_planning_heuristics.js index d2f704d4729..dd671fa72d0 100644 --- a/jstests/noPassthroughWithMongod/columnstore_planning_heuristics.js +++ b/jstests/noPassthroughWithMongod/columnstore_planning_heuristics.js @@ -2,9 +2,8 @@ * Testing of the query planner heuristics for determining whether a collection is eligible for * column scan. * @tags: [ - * # Column store indexes are still under a feature flag and require full sbe. + * # Column store indexes are still under a feature flag. * featureFlagColumnstoreIndexes, - * featureFlagSbeFull, * ] */ (function() { diff --git a/jstests/noPassthroughWithMongod/group_pushdown.js b/jstests/noPassthroughWithMongod/group_pushdown.js index 63bbc29de15..40289e0360f 100644 --- a/jstests/noPassthroughWithMongod/group_pushdown.js +++ b/jstests/noPassthroughWithMongod/group_pushdown.js @@ -185,38 +185,28 @@ assertResultsMatchWithAndWithoutPushdown( ], 1); -// Computed projections are only eligible for pushdown into SBE when SBE is fully enabled. -// Additionally, $group stages with dotted fields may only be eligible for pushdown when SBE is -// fully enabled as dependancy analysis may produce a dotted projection, which are not currently -// supported in mainline SBE. -const sbeFull = checkSBEEnabled(db, ["featureFlagSbeFull"]); -if (sbeFull) { - // The $group stage refers to two existing sub-fields. - assertResultsMatchWithAndWithoutPushdown( - coll, - [ - { - $project: - {item: 1, price: 1, quantity: 1, dateParts: {$dateToParts: {date: "$date"}}} - }, - { - $group: { - _id: "$item", - hs: {$sum: - {$add: ["$dateParts.hour", "$dateParts.hour", "$dateParts.minute"]}} - } - }, - ], - [{"_id": "a", "hs": 39}, {"_id": "b", "hs": 34}, {"_id": "c", "hs": 23}], - 1); - - // The $group stage refers to a non-existing sub-field twice. - assertResultsMatchWithAndWithoutPushdown( - coll, - [{$group: {_id: "$item", hs: {$sum: {$add: ["$date.hour", "$date.hour"]}}}}], - [{"_id": "a", "hs": 0}, {"_id": "b", "hs": 0}, {"_id": "c", "hs": 0}], - 1); -} +// The $group stage refers to two existing sub-fields. +assertResultsMatchWithAndWithoutPushdown( + coll, + [ + {$project: {item: 1, price: 1, quantity: 1, dateParts: {$dateToParts: {date: "$date"}}}}, + { + $group: { + _id: "$item", + hs: {$sum: {$add: ["$dateParts.hour", "$dateParts.hour", "$dateParts.minute"]}} + } + }, + ], + [{"_id": "a", "hs": 39}, {"_id": "b", "hs": 34}, {"_id": "c", "hs": 23}], + 1); + +// The $group stage refers to a non-existing sub-field twice. +assertResultsMatchWithAndWithoutPushdown( + coll, + [{$group: {_id: "$item", hs: {$sum: {$add: ["$date.hour", "$date.hour"]}}}}], + [{"_id": "a", "hs": 0}, {"_id": "b", "hs": 0}, {"_id": "c", "hs": 0}], + 1); + // Two group stages both get pushed down and the second $group stage refers to only existing // top-level fields of the first $group. The field name may be one of "result" / "recordId" / // "returnKey" / "snapshotId" / "indexId" / "indexKey" / "indexKeyPattern" which are reserved names diff --git a/jstests/noPassthroughWithMongod/now_variable.js b/jstests/noPassthroughWithMongod/now_variable.js index 341959956cb..c20558102fb 100644 --- a/jstests/noPassthroughWithMongod/now_variable.js +++ b/jstests/noPassthroughWithMongod/now_variable.js @@ -156,7 +156,7 @@ runTestsExpectFailure(baseCollectionClusterTimeAgg); runTestsExpectFailure(fromViewWithClusterTime); runTestsExpectFailure(withExprClusterTime); -if (checkSBEEnabled(db, ["featureFlagSbeFull"])) { +if (checkSBEEnabled(db)) { function verifyPlanCacheSize(query) { coll.getPlanCache().clear(); diff --git a/jstests/noPassthroughWithMongod/sbe_agg_pushdown.js b/jstests/noPassthroughWithMongod/sbe_agg_pushdown.js index 2e2e08fa587..78f998ad4f7 100644 --- a/jstests/noPassthroughWithMongod/sbe_agg_pushdown.js +++ b/jstests/noPassthroughWithMongod/sbe_agg_pushdown.js @@ -13,9 +13,8 @@ const kUnsupportedExpression = { $toBool: {date: "$b"} }; -const isSBEEnabled = checkSBEEnabled(db, ["featureFlagSbeFull"]); -if (!isSBEEnabled) { - jsTestLog("Skipping test because the SBE feature flag is disabled"); +if (!checkSBEEnabled(db)) { + jsTestLog("Skipping test because SBE is not enabled"); return; } diff --git a/jstests/noPassthroughWithMongod/sbe_query_eligibility.js b/jstests/noPassthroughWithMongod/sbe_query_eligibility.js index f9bc9fcb181..e229c8269a2 100644 --- a/jstests/noPassthroughWithMongod/sbe_query_eligibility.js +++ b/jstests/noPassthroughWithMongod/sbe_query_eligibility.js @@ -8,11 +8,19 @@ load("jstests/libs/analyze_plan.js"); load("jstests/libs/sbe_util.js"); // For 'checkSBEEnabled'. /** - * Utility which asserts whether, when aggregating over 'collection' with 'pipeline', whether - * explain reports that SBE or classic was used, based on the value of 'isSBE'. + * Utility which asserts that when running the given 'query' over 'collection', explain's reported + * use of SBE versus classic agrees with the given 'isSBE' value. + * + * If 'query' is an array, assumes that it is an aggregation pipeline. Otherwise, treats it as an + * arbitrary explainable command. */ -function assertEngineUsed(collection, pipeline, isSBE) { - const explain = collection.explain().aggregate(pipeline); +function assertEngineUsed(collection, query, isSBE) { + let explain; + if (Array.isArray(query)) { + explain = collection.explain().aggregate(query); + } else { + explain = db.runCommand({explain: query}); + } const expectedExplainVersion = isSBE ? "2" : "1"; assert(explain.hasOwnProperty("explainVersion"), explain); assert.eq(explain.explainVersion, expectedExplainVersion, explain); @@ -29,7 +37,7 @@ coll.drop(); assert.commandWorked(coll.insert({})); assert.eq(coll.find().itcount(), 1); -// Simple eligible cases. +// Queries which should use SBE. const expectedSbeCases = [ // Non-$expr match filters. [{$match: {a: 1}}], @@ -76,15 +84,10 @@ const expectedSbeCases = [ {$group: {_id: "$a", out: {$sum: 1}}}, {$match: {$expr: {$eq: ["$a", "$b"]}}} ], -]; -for (const pipeline of expectedSbeCases) { - assertEngineUsed(coll, pipeline, true /* isSBE */); -} + // A find command which uses $slice projection. + {find: coll.getName(), projection: {a: {$slice: 3}}}, -// The cases below are expected to use SBE only if 'featureFlagSbeFull' is on. -const isSbeFullyEnabled = checkSBEEnabled(db, ["featureFlagSbeFull"]); -const sbeFullCases = [ // Match filters with $expr. [{$match: {$expr: {$eq: ["$a", "$b"]}}}], [{$match: {$and: [{$expr: {$eq: ["$a", "$b"]}}, {c: 1}]}}], @@ -186,7 +189,32 @@ const sbeFullCases = [ ], ]; -for (const pipeline of sbeFullCases) { - assertEngineUsed(coll, pipeline, isSbeFullyEnabled /* isSBE */); +for (const query of expectedSbeCases) { + assertEngineUsed(coll, query, true /* isSBE */); +} + +// Queries that are ineligible for SBE and will fall back to the classic engine. +const fallbackToClassicCases = [ + // Uses an unsupported expression. + [{$project: {fieldA: {$getField: "a"}}}], + + // Sort with numeric path component. + [{$sort: {"a.0": 1}}], + + // An idhack eligible query. + [{$match: {_id: 1}}], + + // A find command which $elemMatch projection. + {find: coll.getName(), projection: {a: {$elemMatch: {b: 1}}}}, + + // A find command which uses positional ($) projection. + {find: coll.getName(), filter: {a: 1}, projection: {"a.$": 1}}, + + // A find command which uses 'showRecordId'. + {find: coll.getName(), filter: {a: 1}, showRecordId: true}, +]; + +for (const query of fallbackToClassicCases) { + assertEngineUsed(coll, query, false /* isSBE */); } })(); diff --git a/jstests/sharding/query/views.js b/jstests/sharding/query/views.js index 86568d8e12b..0dde8f06736 100644 --- a/jstests/sharding/query/views.js +++ b/jstests/sharding/query/views.js @@ -53,7 +53,6 @@ let config = mongos.getDB("config"); let db = mongos.getDB(jsTestName()); db.dropDatabase(); -const sbeEnabled = checkSBEEnabled(db); const isClustered = ClusteredCollectionUtil.areAllCollectionsClustered(st.rs0.getPrimary()); let coll = db.getCollection("coll"); @@ -117,21 +116,25 @@ for (let verbosity of explainVerbosities) { // assert.eq(5, view.count({a: {$lte: 8}})); -result = db.runCommand({explain: {count: "view", query: {a: {$lte: 8}}}}); -// Allow success whether or not the pipeline is optimized away, as it differs based on test -// environment and execution engine used. -verifyExplainResult({ - shardedExplain: result, - verbosity: "allPlansExecution", - optimizedAwayPipeline: sbeEnabled && !isClustered -}); -for (let verbosity of explainVerbosities) { - result = db.runCommand({explain: {count: "view", query: {a: {$lte: 8}}}, verbosity: verbosity}); +// If SBE is enabled on all nodes, then validate the explain results for a count command. We could +// validate the explain results when the classic engine is enabled, but doing so is complicated for +// multiversion scenarios (e.g. in the multiversion passthrough on the classic engine build variant +// only some shards have SBE enabled, so the expected results differ across shards). +if (checkSBEEnabled(db, [], true /*checkAllNodes*/)) { + result = db.runCommand({explain: {count: "view", query: {a: {$lte: 8}}}}); + // Allow success whether or not the pipeline is optimized away, as it differs based on test + // environment and execution engine used. verifyExplainResult({ shardedExplain: result, - verbosity: verbosity, - optimizedAwayPipeline: sbeEnabled && !isClustered + verbosity: "allPlansExecution", + optimizedAwayPipeline: !isClustered }); + for (let verbosity of explainVerbosities) { + result = + db.runCommand({explain: {count: "view", query: {a: {$lte: 8}}}, verbosity: verbosity}); + verifyExplainResult( + {shardedExplain: result, verbosity: verbosity, optimizedAwayPipeline: !isClustered}); + } } // diff --git a/src/mongo/db/query/get_executor.cpp b/src/mongo/db/query/get_executor.cpp index cf90c42d51e..2dd4704b7c0 100644 --- a/src/mongo/db/query/get_executor.cpp +++ b/src/mongo/db/query/get_executor.cpp @@ -838,13 +838,6 @@ public: return buildMultiPlan(std::move(solutions)); } - /** - * Returns the planner params if initialized, otherwise returns nullptr. - */ - const QueryPlannerParams* plannerParams() const { - return _plannerParamsInitialized ? &_plannerParams : nullptr; - } - protected: static constexpr bool ShouldDeferExecutionTreeGeneration = DeferExecutionTreeGeneration; @@ -1416,88 +1409,30 @@ StatusWith> getSlotBasedExe } /** - * Checks if the result of query planning is SBE compatible. In this function, 'sbeFull' indicates - * whether the full set of features supported by SBE is enabled, while 'canUseRegularSbe' indicates - * whether the query is compatible with the subset of SBE enabled by default. + * Checks if the result of query planning is SBE compatible. If any of the query solutions in + * 'planningResult' cannot currently be compiled to an SBE plan via the SBE stage builders, then we + * will fall back to the classic engine. */ -bool shouldPlanningResultUseSbe(bool sbeFull, - bool canUseRegularSbe, - bool columnIndexPresent, - const SlotBasedPrepareExecutionResult& planningResult) { +bool shouldPlanningResultUseSbe(const SlotBasedPrepareExecutionResult& planningResult) { // If we have an entry in the SBE plan cache, then we can use SBE. if (planningResult.isRecoveredFromPlanCache()) { return true; } - // For now this function assumes one of these is true. If all are false, we should not use - // SBE. - tassert(6164401, - "Expected sbeFull, or a regular SBE compatiable query, or a CSI present", - sbeFull || canUseRegularSbe || columnIndexPresent); - const auto& solutions = planningResult.solutions(); if (solutions.empty()) { - // Query needs subplanning (plans are generated later, we don't have access yet). + // Query needs subplanning (plans are generated later, we don't have access yet). We can + // proceed with using SBE in this case. invariant(planningResult.needsSubplanning()); - - // Use SBE for rooted $or queries if SBE is fully enabled or the query is SBE compatible to - // begin with. - return sbeFull || canUseRegularSbe; - } - - // Check that the query solution is SBE compatible. - const bool allStagesCompatible = - std::all_of(solutions.begin(), solutions.end(), [](const auto& solution) { - // We must have a solution, otherwise we would have early exited. - invariant(solution->root()); - return isQueryPlanSbeCompatible(solution.get()); - }); - - if (!allStagesCompatible) { - return false; - } - - if (sbeFull || canUseRegularSbe) { return true; } - // Return true if we have a column scan plan, and false otherwise. - return solutions.size() == 1 && - solutions.front()->root()->hasNode(StageType::STAGE_COLUMN_SCAN); -} - -/** - * Returns true if the query *may* be eligible for column scan. This requires three conditions: - * - * 1) apiStrict is false - * 2) query has a column scan-eligible projection, i.e. an inclusion projection or is count-like - * 3) there is a column store index present on the main collection - * - * These checks are an optimization in cases where we can determine without query planning that a - * query doesn't meet other SBE requirements, and definitely can't use column scan. - */ -bool maybeQueryIsColumnScanEligible(OperationContext* opCtx, - const MultipleCollectionAccessor& collections, - const CanonicalQuery* cq) { - if (APIParameters::get(opCtx).getAPIStrict().value_or(false)) { - // Column scan is not allowed with apiStrict: true. - return false; - } - - if (!cq->isCountLike() && (!cq->getProj() || cq->getProj()->isExclusionOnly())) { - // The query's projection makes it automatically ineligible for column scan. - return false; - } - - if (const auto& mainColl = collections.getMainCollection()) { - std::vector csiIndexes; - mainColl->getIndexCatalog()->findIndexByType(opCtx, IndexNames::COLUMN, csiIndexes); - return std::any_of(csiIndexes.begin(), csiIndexes.end(), [](const auto* descriptor) { - return !descriptor->hidden(); - }); - } - - return false; + // Check that all query solutions are SBE compatible. + return std::all_of(solutions.begin(), solutions.end(), [](const auto& solution) { + // We must have a solution, otherwise we would have early exited. + invariant(solution->root()); + return isQueryPlanSbeCompatible(solution.get()); + }); } /** @@ -1505,35 +1440,16 @@ bool maybeQueryIsColumnScanEligible(OperationContext* opCtx, * 'featureFlagSbeFull' being set; false otherwise. */ bool shouldUseRegularSbe(const CanonicalQuery& cq) { - if (cq.getExpCtx()->sbeCompatibility != SbeCompatibility::fullyCompatible) { - return false; - } - - const auto* proj = cq.getProj(); - - // Disallow projections which use expressions. - if (proj && proj->hasExpressions()) { - return false; - } - - // Disallow projections which have dotted paths. - if (proj && proj->hasDottedPaths()) { - return false; - } - - // Disallow filters which feature $expr. - if (cq.countNodes(cq.root(), MatchExpression::MatchType::EXPRESSION) > 0) { - return false; - } - - const auto& sortPattern = cq.getSortPattern(); - - // Disallow sorts which have a common prefix. - if (sortPattern && sortPatternHasPartsWithCommonPrefix(*sortPattern)) { - return false; - } - - return true; + auto sbeCompatLevel = cq.getExpCtx()->sbeCompatibility; + // We shouldn't get here if there are expressions in the query which are completely unsupported + // by SBE. + tassert(7248600, + "Unexpected SBE compatibility value", + sbeCompatLevel != SbeCompatibility::notCompatible); + // The 'ExpressionContext' may indicate that there are expressions which are only supported in + // SBE when 'featureFlagSbeFull' is set, or fully supported regardless of the value of the + // feature flag. This function should only return true in the latter case. + return cq.getExpCtx()->sbeCompatibility == SbeCompatibility::fullyCompatible; } /** @@ -1564,14 +1480,14 @@ attemptToGetSlotBasedExecutor( const bool sbeFull = feature_flags::gFeatureFlagSbeFull.isEnabledAndIgnoreFCV(); const bool canUseRegularSbe = shouldUseRegularSbe(*canonicalQuery); - // Attempt to use SBE if the query may be eligible for column scan, if the currently supported - // subset of SBE is being used, or if SBE is fully enabled. Otherwise, fallback to the classic - // engine right away. - if (sbeFull || canUseRegularSbe || - maybeQueryIsColumnScanEligible(opCtx, collections, canonicalQuery.get())) { - // Create the SBE prepare execution helper and initialize the params for the planner. Our - // decision about using SBE will depend on whether there is a column index present. - + // If 'canUseRegularSbe' is true, then only the subset of SBE which is currently on by default + // is used by the query. If 'sbeFull' is true, then the server is configured to run any + // SBE-compatible query using SBE, even if the query uses features that are not on in SBE by + // default. Either way, try to construct an SBE plan executor. + if (canUseRegularSbe || sbeFull) { + // Create the SBE prepare execution helper and initialize the params for the planner. If + // planning results in any 'QuerySolution' which cannot be handled by the SBE stage builder, + // then we will fall back to the classic engine. auto sbeYieldPolicy = makeSbeYieldPolicy( opCtx, yieldPolicy, &collections.getMainCollection(), canonicalQuery->nss()); SlotBasedPrepareExecutionHelper helper{ @@ -1581,11 +1497,7 @@ attemptToGetSlotBasedExecutor( return planningResultWithStatus.getStatus(); } - const bool csiPresent = - helper.plannerParams() && !helper.plannerParams()->columnStoreIndexes.empty(); - - if (shouldPlanningResultUseSbe( - sbeFull, canUseRegularSbe, csiPresent, *planningResultWithStatus.getValue())) { + if (shouldPlanningResultUseSbe(*planningResultWithStatus.getValue())) { if (extractAndAttachPipelineStages) { // We know now that we will use SBE, so we need to remove the pushed-down stages // from the original pipeline object. diff --git a/src/mongo/db/query/query_feature_flags.idl b/src/mongo/db/query/query_feature_flags.idl index 2b8532dca4c..1274432404f 100644 --- a/src/mongo/db/query/query_feature_flags.idl +++ b/src/mongo/db/query/query_feature_flags.idl @@ -76,7 +76,8 @@ feature_flags: default: false featureFlagSbeFull: - description: "Feature flag for enabling SBE for a much larger class of queries than what is exposed by default" + description: "Feature flag for SBE behaviors, features, or extensions that are not yet enabled + by default" cpp_varname: gFeatureFlagSbeFull default: false diff --git a/src/mongo/db/query/query_utils.cpp b/src/mongo/db/query/query_utils.cpp index 835aec9e088..573f1d53d05 100644 --- a/src/mongo/db/query/query_utils.cpp +++ b/src/mongo/db/query/query_utils.cpp @@ -101,6 +101,7 @@ bool isQuerySbeCompatible(const CollectionPtr* collection, const CanonicalQuery* bool isQueryPlanSbeCompatible(const QuerySolution* root) { tassert(7061701, "Expected QuerySolution pointer to not be nullptr", root); + // TODO SERVER-52958: Add support in the SBE stage builders for the COUNT_SCAN stage. const bool isNotCountScan = !root->hasNode(StageType::STAGE_COUNT_SCAN); return isNotCountScan; -- cgit v1.2.1