From 8d9e3c1ba0760e75b05181e59330e8fd7a79c6b1 Mon Sep 17 00:00:00 2001 From: Alexander Ignatyev Date: Tue, 29 Nov 2022 11:29:58 +0000 Subject: SERVER-69456 Set new cost model as default --- jstests/cqf/basic_agg.js | 10 +- jstests/cqf/basic_find.js | 10 +- jstests/cqf/index_intersect.js | 29 ++-- jstests/cqf/sort_project.js | 6 +- jstests/cqf_parallel/groupby.js | 7 +- jstests/libs/optimizer_utils.js | 16 ++ src/mongo/db/exec/sbe/abt/sbe_abt_test.cpp | 1 + src/mongo/db/exec/sbe/abt/sbe_abt_test_util.cpp | 7 +- src/mongo/db/query/cost_model/cost_model.idl | 3 +- .../db/query/cost_model/cost_model_manager.cpp | 88 +++++------ src/mongo/db/query/cost_model/cost_model_test.cpp | 20 +-- .../query/optimizer/interval_intersection_test.cpp | 1 + .../optimizer/logical_rewriter_optimizer_test.cpp | 23 +++ .../db/query/optimizer/optimizer_failure_test.cpp | 5 + .../optimizer/physical_rewriter_optimizer_test.cpp | 170 +++++++++++++++++---- .../optimizer/utils/unit_test_pipeline_utils.cpp | 65 +++++++- .../db/query/optimizer/utils/unit_test_utils.cpp | 61 ++++---- .../db/query/optimizer/utils/unit_test_utils.h | 48 +++--- 18 files changed, 391 insertions(+), 179 deletions(-) diff --git a/jstests/cqf/basic_agg.js b/jstests/cqf/basic_agg.js index 66768c7255d..678d896c290 100644 --- a/jstests/cqf/basic_agg.js +++ b/jstests/cqf/basic_agg.js @@ -10,14 +10,16 @@ if (!checkCascadesOptimizerEnabled(db)) { const coll = db.cqf_basic_index; coll.drop(); -assert.commandWorked( - coll.insert([{a: {b: 1}}, {a: {b: 2}}, {a: {b: 3}}, {a: {b: 4}}, {a: {b: 5}}])); +const documents = [{a: {b: 1}}, {a: {b: 2}}, {a: {b: 3}}, {a: {b: 4}}, {a: {b: 5}}]; -const extraDocCount = 50; +const extraDocCount = 500; // Add extra docs to make sure indexes can be picked. for (let i = 0; i < extraDocCount; i++) { - assert.commandWorked(coll.insert({a: {b: i + 10}})); + documents.push({a: {b: i + 10}}); } + +assert.commandWorked(coll.insertMany(documents)); + assert.commandWorked(coll.createIndex({'a.b': 1})); let res = coll.explain("executionStats").aggregate([{$match: {'a.b': 2}}]); diff --git a/jstests/cqf/basic_find.js b/jstests/cqf/basic_find.js index e1489de45b2..f41e4b1b9f7 100644 --- a/jstests/cqf/basic_find.js +++ b/jstests/cqf/basic_find.js @@ -10,14 +10,16 @@ if (!checkCascadesOptimizerEnabled(db)) { const coll = db.cqf_basic_find; coll.drop(); -assert.commandWorked( - coll.insert([{a: {b: 1}}, {a: {b: 2}}, {a: {b: 3}}, {a: {b: 4}}, {a: {b: 5}}, {'': 3}])); +const docs = [{a: {b: 1}}, {a: {b: 2}}, {a: {b: 3}}, {a: {b: 4}}, {a: {b: 5}}, {'': 3}]; -const extraDocCount = 50; +const extraDocCount = 500; // Add extra docs to make sure indexes can be picked. for (let i = 0; i < extraDocCount; i++) { - assert.commandWorked(coll.insert({a: {b: i + 10}})); + docs.push({a: {b: i + 10}}); } + +assert.commandWorked(coll.insertMany(docs)); + assert.commandWorked(coll.createIndex({'a.b': 1})); let res = coll.explain("executionStats").find({'a.b': 2}).finish(); diff --git a/jstests/cqf/index_intersect.js b/jstests/cqf/index_intersect.js index 991de7b8ef3..3c6010f4cae 100644 --- a/jstests/cqf/index_intersect.js +++ b/jstests/cqf/index_intersect.js @@ -10,24 +10,28 @@ if (!checkCascadesOptimizerEnabled(db)) { const t = db.cqf_index_intersect; t.drop(); -const nMatches = 60; - -assert.commandWorked(t.insert({a: 1, b: 1, c: 1})); -assert.commandWorked(t.insert({a: 3, b: 2, c: 1})); +const documents = [{a: 1, b: 1, c: 1}, {a: 3, b: 2, c: 1}]; +const nMatches = 300; for (let i = 0; i < nMatches; i++) { - assert.commandWorked(t.insert({a: 3, b: 3, c: i})); + documents.push({a: 3, b: 3, c: i}); } -assert.commandWorked(t.insert({a: 4, b: 3, c: 2})); -assert.commandWorked(t.insert({a: 5, b: 5, c: 2})); +documents.push({a: 4, b: 3, c: 2}); +documents.push({a: 5, b: 5, c: 2}); -for (let i = 1; i < nMatches + 100; i++) { - assert.commandWorked(t.insert({a: i + nMatches, b: i + nMatches, c: i + nMatches})); +for (let i = 1; i < nMatches + 500; i++) { + documents.push({a: i + nMatches, b: i + nMatches, c: i + nMatches}); } +assert.commandWorked(t.insertMany(documents)); + assert.commandWorked(t.createIndex({'a': 1})); assert.commandWorked(t.createIndex({'b': 1})); -let res = t.explain("executionStats").aggregate([{$match: {'a': 3, 'b': 3}}]); +// TODO SERVER-71553 The Cost Model is overriden to preserve MergeJoin plan. +// In majority of cases it works well without Cost Model override, but in some rare cases it fails. +let res = runCommandWithCostModel( + () => t.explain("executionStats").aggregate([{$match: {'a': 3, 'b': 3}}]), + {"mergeJoinStartupCost": 1e-9, "mergeJoinIncrementalCost": 1e-9}); assert.eq(nMatches, res.executionStats.nReturned); // Verify we can place a MergeJoin @@ -37,7 +41,10 @@ assertValueOnPath("IndexScan", joinNode, "leftChild.nodeType"); assertValueOnPath("IndexScan", joinNode, "rightChild.children.0.child.nodeType"); // One side is not equality, and we use a HashJoin. -res = t.explain("executionStats").aggregate([{$match: {'a': {$lte: 3}, 'b': 3}}]); +// TODO SERVER-71553 The Cost Model is overriden to preserve HashJoin plan. +res = runCommandWithCostModel( + () => t.explain("executionStats").aggregate([{$match: {'a': {$lte: 3}, 'b': 3}}]), + {"hashJoinIncrementalCost": 1e-9}); assert.eq(nMatches, res.executionStats.nReturned); joinNode = navigateToPlanPath(res, "child.leftChild"); diff --git a/jstests/cqf/sort_project.js b/jstests/cqf/sort_project.js index 0e953f91f30..5c071490222 100644 --- a/jstests/cqf/sort_project.js +++ b/jstests/cqf/sort_project.js @@ -34,9 +34,11 @@ try { { // Covered plan. Also an index scan on all fields is cheaper than a collection scan. - const res = coll.explain("executionStats").aggregate([ + // TODO SERVER-71553 The Cost Model is overriden to preserve IndexScan plan. + const res = runCommandWithCostModel(() => coll.explain("executionStats").aggregate([ {'$project': {_id: 0, f_0: 1, f_1: 1, f_2: 1, f_3: 1, f_4: 1}} - ]); + ]), + {"indexScanStartupCost": 1e-9}); assert.eq(nDocs, res.executionStats.nReturned); assertValueOnPlanPath("IndexScan", res, "child.child.nodeType"); } diff --git a/jstests/cqf_parallel/groupby.js b/jstests/cqf_parallel/groupby.js index 88d33207a07..b4abfe0f7de 100644 --- a/jstests/cqf_parallel/groupby.js +++ b/jstests/cqf_parallel/groupby.js @@ -17,7 +17,10 @@ assert.commandWorked(t.insert({a: 4})); assert.commandWorked(t.insert({a: 5})); // Demonstrate local-global optimization. -const res = t.explain("executionStats").aggregate([{$group: {_id: "$a", cnt: {$sum: 1}}}]); +// TODO SERVER-71552 The tests freezes with new cost model. +const res = runCommandWithCostModel( + () => t.explain("executionStats").aggregate([{$group: {_id: "$a", cnt: {$sum: 1}}}]), + {"groupByStartupCost": 1e-6}); assert.eq(5, res.executionStats.nReturned); assertValueOnPlanPath("Exchange", res, "child.nodeType"); @@ -33,4 +36,4 @@ assertValueOnPlanPath( "UnknownPartitioning", res, "child.child.child.child.child.properties.physicalProperties.distribution.type"); -}()); \ No newline at end of file +}()); diff --git a/jstests/libs/optimizer_utils.js b/jstests/libs/optimizer_utils.js index 9b7a2bbd8a4..63d009006d4 100644 --- a/jstests/libs/optimizer_utils.js +++ b/jstests/libs/optimizer_utils.js @@ -271,3 +271,19 @@ function assertValueOnPath(value, doc, path) { function assertValueOnPlanPath(value, doc, path) { assertValueOnPathFn(value, doc, path, navigateToPlanPath); } + +function runCommandWithCostModel(func, costModel) { + const oldCostModelOverrides = assert.commandWorked(db.adminCommand( + {'getParameter': 1, 'internalCostModelCoefficients': 1}))['internalCostModelCoefficients']; + + const costModelJson = JSON.stringify(costModel); + assert.commandWorked( + db.adminCommand({'setParameter': 1, 'internalCostModelCoefficients': costModelJson})); + + try { + return assert.commandWorked(func()); + } finally { + assert.commandWorked(db.adminCommand( + {'setParameter': 1, 'internalCostModelCoefficients': oldCostModelOverrides})); + } +} diff --git a/src/mongo/db/exec/sbe/abt/sbe_abt_test.cpp b/src/mongo/db/exec/sbe/abt/sbe_abt_test.cpp index e18113f3a5c..0f98fe3c388 100644 --- a/src/mongo/db/exec/sbe/abt/sbe_abt_test.cpp +++ b/src/mongo/db/exec/sbe/abt/sbe_abt_test.cpp @@ -381,6 +381,7 @@ TEST_F(NodeSBE, Lower1) { auto phaseManager = makePhaseManager(OptPhaseManager::getAllRewritesSet(), prefixId, {{{"test", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); phaseManager.optimize(tree); diff --git a/src/mongo/db/exec/sbe/abt/sbe_abt_test_util.cpp b/src/mongo/db/exec/sbe/abt/sbe_abt_test_util.cpp index 3f9c8517878..89da7cbb03e 100644 --- a/src/mongo/db/exec/sbe/abt/sbe_abt_test_util.cpp +++ b/src/mongo/db/exec/sbe/abt/sbe_abt_test_util.cpp @@ -115,8 +115,11 @@ std::vector runSBEAST(OperationContext* opCtx, OPTIMIZER_DEBUG_LOG( 6264807, 5, "SBE translated ABT", "explain"_attr = ExplainGenerator::explainV2(tree)); - auto phaseManager = makePhaseManager( - OptPhaseManager::getAllRewritesSet(), prefixId, {{}}, DebugInfo::kDefaultForTests); + auto phaseManager = makePhaseManager(OptPhaseManager::getAllRewritesSet(), + prefixId, + {{}}, + boost::none /*costModel*/, + DebugInfo::kDefaultForTests); phaseManager.optimize(tree); diff --git a/src/mongo/db/query/cost_model/cost_model.idl b/src/mongo/db/query/cost_model/cost_model.idl index 75fdf04c8e9..0eb45f4ccbd 100644 --- a/src/mongo/db/query/cost_model/cost_model.idl +++ b/src/mongo/db/query/cost_model/cost_model.idl @@ -34,7 +34,7 @@ imports: structs: CostModelCoefficients: - description: "A type representing Cost Model coefficients." + description: "A type representing Cost Model coefficients. By convention it takes execution time measured in milliseconds." strict: true fields: defaultStartupCost: @@ -169,4 +169,3 @@ structs: optional: false type: double stability: stable - diff --git a/src/mongo/db/query/cost_model/cost_model_manager.cpp b/src/mongo/db/query/cost_model/cost_model_manager.cpp index 917f84bc8b8..b013791f64f 100644 --- a/src/mongo/db/query/cost_model/cost_model_manager.cpp +++ b/src/mongo/db/query/cost_model/cost_model_manager.cpp @@ -38,76 +38,58 @@ namespace { */ void initializeCoefficients(CostModelCoefficients& coefficients) { // These cost should reflect estimated aggregated execution time in milliseconds. - constexpr double ms = 1.0e-3; + // The coeffeicient ns converts values from nanoseconds to milliseconds. + constexpr double nsToMs = 1.0e-6; - // Startup cost of an operator. This is the minimal cost of an operator since it is - // present even if it doesn't process any input. - // TODO: calibrate the cost individually for each operator - coefficients.setDefaultStartupCost(0.000001); + coefficients.setDefaultStartupCost(1.0 * nsToMs); - // TODO: collection scan should depend on the width of the doc. - // TODO: the actual measured cost is (0.4 * ms), however we increase it here because currently - // it is not possible to estimate the cost of a collection scan vs a full index scan. - coefficients.setScanIncrementalCost(0.6 * ms); - coefficients.setScanStartupCost(0.000001); + coefficients.setScanIncrementalCost(422.31145989 * nsToMs); + coefficients.setScanStartupCost(6175.527218993269 * nsToMs); - // TODO: cost(N fields) ~ (0.55 + 0.025 * N) - coefficients.setIndexScanIncrementalCost(0.5 * ms); - coefficients.setIndexScanStartupCost(0.000001); + coefficients.setIndexScanIncrementalCost(403.68075869 * nsToMs); + coefficients.setIndexScanStartupCost(14054.983953111061 * nsToMs); - // TODO: cost(N fields) ~ 0.7 + 0.19 * N - coefficients.setSeekCost(2.0 * ms); - coefficients.setSeekStartupCost(0.000001); + coefficients.setSeekCost(1174.84136356 * nsToMs); + coefficients.setSeekStartupCost(7488.662376624863 * nsToMs); - // TODO: take the expression into account. - // cost(N conditions) = 0.2 + N * ??? - coefficients.setFilterIncrementalCost(0.2 * ms); - coefficients.setFilterStartupCost(0.000001); + coefficients.setFilterIncrementalCost(83.7274685 * nsToMs); + coefficients.setFilterStartupCost(1461.3148783443378 * nsToMs); - // TODO: the cost of projection depends on number of fields: cost(N fields) ~ 0.1 + 0.2 * N - coefficients.setEvalIncrementalCost(2.0 * ms); - coefficients.setEvalStartupCost(0.000001); + coefficients.setEvalIncrementalCost(430.6176946 * nsToMs); + coefficients.setEvalStartupCost(1103.4048573163343 * nsToMs); - // TODO: cost(N fields) ~ 0.04 + 0.03*(N^2) - coefficients.setGroupByIncrementalCost(0.07 * ms); - coefficients.setGroupByStartupCost(0.000001); + coefficients.setGroupByIncrementalCost(413.07932374 * nsToMs); + coefficients.setGroupByStartupCost(1199.8878012735659 * nsToMs); - coefficients.setUnwindIncrementalCost(0.03 * ms); // TODO: not yet calibrated - coefficients.setUnwindStartupCost(0.000001); + coefficients.setUnwindIncrementalCost(586.57200195 * nsToMs); + coefficients.setUnwindStartupCost(1.0 * nsToMs); - // TODO: not yet calibrated, should be at least as expensive as a filter - coefficients.setBinaryJoinIncrementalCost(0.2 * ms); - coefficients.setBinaryJoinStartupCost(0.000001); + coefficients.setBinaryJoinIncrementalCost(161.62301944 * nsToMs); + coefficients.setBinaryJoinStartupCost(402.8455479458652 * nsToMs); - coefficients.setHashJoinIncrementalCost(0.05 * ms); // TODO: not yet calibrated - coefficients.setHashJoinStartupCost(0.000001); + coefficients.setHashJoinIncrementalCost(250.61365634 * nsToMs); + coefficients.setHashJoinStartupCost(1.0 * nsToMs); // Already calibrated. - coefficients.setMergeJoinIncrementalCost(0.02 * ms); // TODO: not yet calibrated - coefficients.setMergeJoinStartupCost(0.000001); + coefficients.setMergeJoinIncrementalCost(111.23423304 * nsToMs); + coefficients.setMergeJoinStartupCost(1517.7970800404169 * nsToMs); - coefficients.setUniqueIncrementalCost(0.7 * ms); - coefficients.setUniqueStartupCost(0.000001); + coefficients.setUniqueIncrementalCost(269.71368614 * nsToMs); + coefficients.setUniqueStartupCost(1.0 * nsToMs); // Already calibrated. - // TODO: implement collation cost that depends on number and size of sorted fields - // Based on a mix of int and str(64) fields: - // 1 sort field: sort_cost(N) = 1.0/10 * N * log(N) - // 5 sort fields: sort_cost(N) = 2.5/10 * N * log(N) - // 10 sort fields: sort_cost(N) = 3.0/10 * N * log(N) - // field_cost_coeff(F) ~ 0.75 + 0.2 * F - coefficients.setCollationIncrementalCost(2.5 * ms); // 5 fields avg - coefficients.setCollationStartupCost(0.000001); + coefficients.setCollationIncrementalCost(2500 * nsToMs); // TODO: not yet calibrated + coefficients.setCollationStartupCost(1.0 * nsToMs); // TODO: not yet calibrated - coefficients.setCollationWithLimitIncrementalCost(1.0 * ms); // TODO: not yet calibrated - coefficients.setCollationWithLimitStartupCost(0.000001); + coefficients.setCollationWithLimitIncrementalCost(1000 * nsToMs); // TODO: not yet calibrated + coefficients.setCollationWithLimitStartupCost(1.0 * nsToMs); // TODO: not yet calibrated - coefficients.setUnionIncrementalCost(0.02 * ms); - coefficients.setUnionStartupCost(0.000001); + coefficients.setUnionIncrementalCost(111.94945268 * nsToMs); + coefficients.setUnionStartupCost(69.88096657391543 * nsToMs); - coefficients.setExchangeIncrementalCost(0.1 * ms); // TODO: not yet calibrated - coefficients.setExchangeStartupCost(0.000001); + coefficients.setExchangeIncrementalCost(100 * nsToMs); // TODO: not yet calibrated + coefficients.setExchangeStartupCost(1.0 * nsToMs); // TODO: not yet calibrated - coefficients.setLimitSkipIncrementalCost(0.0 * ms); // TODO: not yet calibrated - coefficients.setLimitSkipStartupCost(0.000001); + coefficients.setLimitSkipIncrementalCost(62.42111111 * nsToMs); + coefficients.setLimitSkipStartupCost(655.1342592592522 * nsToMs); } } // namespace diff --git a/src/mongo/db/query/cost_model/cost_model_test.cpp b/src/mongo/db/query/cost_model/cost_model_test.cpp index c25c470055d..bb0feba8004 100644 --- a/src/mongo/db/query/cost_model/cost_model_test.cpp +++ b/src/mongo/db/query/cost_model/cost_model_test.cpp @@ -66,8 +66,8 @@ TEST(CostModel, IncreaseIndexScanCost) { prefixId, {{{"c1", createScanDef({}, {{"index1", makeIndexDefinition("a", CollationOp::Ascending)}})}}}, - {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}, - costCoefs); + costCoefs, + {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; phaseManager.optimize(optimized); @@ -112,8 +112,8 @@ TEST(CostModel, IncreaseIndexScanCost) { prefixId, {{{"c1", createScanDef({}, {{"index1", makeIndexDefinition("a", CollationOp::Ascending)}})}}}, - {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}, - costCoefs); + costCoefs, + {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; phaseManager.optimize(optimized); @@ -180,8 +180,8 @@ TEST(CostModel, IncreaseJoinsCost) { makeIndexDefinition("a", CollationOp::Ascending, false /*isMultiKey*/)}, {"index2", makeIndexDefinition("b", CollationOp::Ascending, false /*isMultiKey*/)}})}}}, - {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}, - costCoefs); + costCoefs, + {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; phaseManager.optimize(optimized); @@ -238,8 +238,8 @@ TEST(CostModel, IncreaseJoinsCost) { makeIndexDefinition("a", CollationOp::Ascending, false /*isMultiKey*/)}, {"index2", makeIndexDefinition("b", CollationOp::Ascending, false /*isMultiKey*/)}})}}}, - {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}, - costCoefs); + costCoefs, + {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; phaseManager.optimize(optimized); @@ -272,8 +272,8 @@ TEST(CostModel, IncreaseJoinsCost) { makeIndexDefinition("a", CollationOp::Ascending, false /*isMultiKey*/)}, {"index2", makeIndexDefinition("b", CollationOp::Ascending, false /*isMultiKey*/)}})}}}, - {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}, - costCoefs); + costCoefs, + {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; phaseManager.optimize(optimized); diff --git a/src/mongo/db/query/optimizer/interval_intersection_test.cpp b/src/mongo/db/query/optimizer/interval_intersection_test.cpp index f4db69d9c36..255f9add383 100644 --- a/src/mongo/db/query/optimizer/interval_intersection_test.cpp +++ b/src/mongo/db/query/optimizer/interval_intersection_test.cpp @@ -57,6 +57,7 @@ ABT optimizedQueryPlan(const std::string& query, OptPhase::MemoImplementationPhase}, prefixId, metadata, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT optimized = translated; diff --git a/src/mongo/db/query/optimizer/logical_rewriter_optimizer_test.cpp b/src/mongo/db/query/optimizer/logical_rewriter_optimizer_test.cpp index 15c8920b3be..4b5491a844e 100644 --- a/src/mongo/db/query/optimizer/logical_rewriter_optimizer_test.cpp +++ b/src/mongo/db/query/optimizer/logical_rewriter_optimizer_test.cpp @@ -77,6 +77,7 @@ TEST(LogicalRewriter, RootNodeMerge) { auto phaseManager = makePhaseManager({OptPhase::MemoSubstitutionPhase}, prefixId, {{{"test", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT rewritten = std::move(rootNode); phaseManager.optimize(rewritten); @@ -291,6 +292,7 @@ TEST(LogicalRewriter, FilterProjectRewrite) { auto phaseManager = makePhaseManager({OptPhase::MemoSubstitutionPhase}, prefixId, {{{"test", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT latest = std::move(rootNode); phaseManager.optimize(latest); @@ -402,6 +404,7 @@ TEST(LogicalRewriter, FilterProjectComplexRewrite) { auto phaseManager = makePhaseManager({OptPhase::MemoSubstitutionPhase}, prefixId, {{{"test", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT latest = std::move(rootNode); phaseManager.optimize(latest); @@ -480,6 +483,7 @@ TEST(LogicalRewriter, FilterProjectGroupRewrite) { auto phaseManager = makePhaseManager({OptPhase::MemoSubstitutionPhase}, prefixId, {{{"test", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT latest = std::move(rootNode); phaseManager.optimize(latest); @@ -550,6 +554,7 @@ TEST(LogicalRewriter, FilterProjectUnwindRewrite) { auto phaseManager = makePhaseManager({OptPhase::MemoSubstitutionPhase}, prefixId, {{{"test", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT latest = std::move(rootNode); phaseManager.optimize(latest); @@ -621,6 +626,7 @@ TEST(LogicalRewriter, FilterProjectExchangeRewrite) { auto phaseManager = makePhaseManager({OptPhase::MemoSubstitutionPhase}, prefixId, {{{"test", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT latest = std::move(rootNode); phaseManager.optimize(latest); @@ -693,6 +699,7 @@ TEST(LogicalRewriter, UnwindCollationRewrite) { auto phaseManager = makePhaseManager({OptPhase::MemoSubstitutionPhase}, prefixId, {{{"test", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT latest = std::move(rootNode); phaseManager.optimize(latest); @@ -806,6 +813,7 @@ TEST(LogicalRewriter, FilterUnionReorderSingleProjection) { makePhaseManager({OptPhase::MemoSubstitutionPhase, OptPhase::MemoExplorationPhase}, prefixId, {{{"test1", createScanDef({}, {})}, {"test2", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); phaseManager.optimize(latest); @@ -970,6 +978,7 @@ TEST(LogicalRewriter, MultipleFilterUnionReorder) { makePhaseManager({OptPhase::MemoSubstitutionPhase, OptPhase::MemoExplorationPhase}, prefixId, {{{"test1", createScanDef({}, {})}, {"test2", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); phaseManager.optimize(latest); @@ -1075,6 +1084,7 @@ TEST(LogicalRewriter, FilterUnionUnionPushdown) { {{{"test1", createScanDef({}, {})}, {"test2", createScanDef({}, {})}, {"test3", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT latest = std::move(rootNode); @@ -1220,6 +1230,7 @@ TEST(LogicalRewriter, UnionPreservesCommonLogicalProps) { makePhaseManager({OptPhase::MemoSubstitutionPhase, OptPhase::MemoExplorationPhase}, prefixId, metadata, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT optimized = rootNode; @@ -1437,6 +1448,7 @@ TEST(LogicalRewriter, SargableCE) { makePhaseManager({OptPhase::MemoSubstitutionPhase, OptPhase::MemoExplorationPhase}, prefixId, {{{"test", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT latest = std::move(rootNode); phaseManager.optimize(latest); @@ -1547,6 +1559,7 @@ TEST(LogicalRewriter, RemoveNoopFilter) { auto phaseManager = makePhaseManager({OptPhase::MemoSubstitutionPhase}, prefixId, {{{"test", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT latest = std::move(rootNode); phaseManager.optimize(latest); @@ -1599,6 +1612,7 @@ TEST(LogicalRewriter, NotPushdownToplevelSuccess) { {DistributionType::Centralized}, {} /*partialReqMap*/}}})}, }}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT latest = std::move(rootNode); phaseManager.optimize(latest); @@ -1659,6 +1673,7 @@ TEST(LogicalRewriter, NotPushdownToplevelFailureMultikey) { {DistributionType::Centralized}, {} /*partialReqMap*/}}})}, }}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT latest = std::move(rootNode); phaseManager.optimize(latest); @@ -1710,6 +1725,7 @@ TEST(LogicalRewriter, NotPushdownComposeM) { auto phaseManager = makePhaseManager({OptPhase::MemoSubstitutionPhase}, prefixId, Metadata{{{"coll", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT latest = std::move(rootNode); phaseManager.optimize(latest); @@ -1796,6 +1812,7 @@ TEST(LogicalRewriter, NotPushdownUnderLambdaSuccess) { {DistributionType::Centralized}, {} /*partialReqMap*/}}})}, }}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT latest = std::move(rootNode); phaseManager.optimize(latest); @@ -1881,6 +1898,7 @@ TEST(LogicalRewriter, NotPushdownUnderLambdaKeepOuterTraverse) { {DistributionType::Centralized}, {} /*partialReqMap*/}}})}, }}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT latest = std::move(rootNode); phaseManager.optimize(latest); @@ -1955,6 +1973,7 @@ TEST(LogicalRewriter, NotPushdownUnderLambdaFailsWithFreeVar) { Metadata{{ {"coll", createScanDef({}, {})}, }}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT latest = std::move(rootNode); phaseManager.optimize(latest); @@ -2023,6 +2042,7 @@ TEST(LogicalRewriter, RemoveTraverseSplitComposeM) { {DistributionType::Centralized}, {} /*partialReqMap*/}}})}, }}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT latest = std::move(rootNode); phaseManager.optimize(latest); @@ -2100,6 +2120,7 @@ TEST(LogicalRewriter, TraverseComposeMTraverse) { {DistributionType::Centralized}, {} /*partialReqMap*/}}})}, }}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests); ABT latest = std::move(rootNode); phaseManager.optimize(latest); @@ -2191,6 +2212,7 @@ TEST(LogicalRewriter, RelaxComposeM) { auto phaseManager = makePhaseManager({OptPhase::MemoSubstitutionPhase}, prefixId, {{{"c1", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo::kDefaultForTests, QueryHints{}); @@ -2267,6 +2289,7 @@ TEST(PhysRewriter, FilterIndexingRIN) { false /*isMultiKey*/, {DistributionType::Centralized}, {}}}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); diff --git a/src/mongo/db/query/optimizer/optimizer_failure_test.cpp b/src/mongo/db/query/optimizer/optimizer_failure_test.cpp index 9d30fc12b6f..ce3f56a730f 100644 --- a/src/mongo/db/query/optimizer/optimizer_failure_test.cpp +++ b/src/mongo/db/query/optimizer/optimizer_failure_test.cpp @@ -55,6 +55,7 @@ DEATH_TEST_REGEX(Optimizer, HitIterationLimitInrunStructuralPhases, "Tripwire as makePhaseManager({OptPhase::PathFuse, OptPhase::ConstEvalPre}, prefixId, {{{"test1", createScanDef({}, {})}, {"test2", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo(true, DebugInfo::kDefaultDebugLevelForTests, 0)); ASSERT_THROWS_CODE(phaseManager.optimize(evalNode), DBException, 6808700); @@ -82,6 +83,7 @@ DEATH_TEST_REGEX(Optimizer, auto phaseManager = makePhaseManager({OptPhase::MemoSubstitutionPhase}, prefixId, {{{"test", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo(true, DebugInfo::kDefaultDebugLevelForTests, 0)); ASSERT_THROWS_CODE(phaseManager.optimize(rootNode), DBException, 6808702); @@ -109,6 +111,7 @@ DEATH_TEST_REGEX(Optimizer, auto phaseManager = makePhaseManager({OptPhase::MemoExplorationPhase}, prefixId, {{{"test", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo(true, DebugInfo::kDefaultDebugLevelForTests, 0)); ASSERT_THROWS_CODE(phaseManager.optimize(rootNode), DBException, 6808702); @@ -134,6 +137,7 @@ DEATH_TEST_REGEX(Optimizer, BadGroupID, "Tripwire assertion.*6808704") { auto phaseManager = makePhaseManager({OptPhase::MemoImplementationPhase}, prefixId, {{{"test", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo(true, DebugInfo::kDefaultDebugLevelForTests, 0)); ASSERT_THROWS_CODE(phaseManager.optimize(rootNode), DBException, 6808704); @@ -165,6 +169,7 @@ DEATH_TEST_REGEX(Optimizer, EnvHasFreeVariables, "Tripwire assertion.*6808711") OptPhase::MemoImplementationPhase}, prefixId, {{{"test", createScanDef({}, {})}}}, + boost::none /*costModel*/, DebugInfo(true, DebugInfo::kDefaultDebugLevelForTests, DebugInfo::kIterationLimitForTests)); ASSERT_THROWS_CODE(phaseManager.optimize(rootNode), DBException, 6808711); diff --git a/src/mongo/db/query/optimizer/physical_rewriter_optimizer_test.cpp b/src/mongo/db/query/optimizer/physical_rewriter_optimizer_test.cpp index 3cd8611893f..1f298967be5 100644 --- a/src/mongo/db/query/optimizer/physical_rewriter_optimizer_test.cpp +++ b/src/mongo/db/query/optimizer/physical_rewriter_optimizer_test.cpp @@ -42,7 +42,6 @@ namespace mongo::optimizer { using PartialSchemaSelHints = ce::PartialSchemaSelHints; namespace { - // Default selectivity of predicates used by HintedCE to force certain plans. constexpr double kDefaultSelectivity = 0.1; @@ -72,6 +71,7 @@ TEST(PhysRewriter, PhysicalRewriterBasic) { OptPhase::MemoImplementationPhase}, prefixId, {{{"test", createScanDef({}, {})}}}, + /*costModel*/ boost::none, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -113,7 +113,7 @@ TEST(PhysRewriter, PhysicalRewriterBasic) { // Plan output with properties. ASSERT_EXPLAIN_PROPS_V2( - "Properties [cost: 0.620002, localCost: 0, adjustedCE: 10]\n" + "Properties [cost: 0.438321, localCost: 0, adjustedCE: 10]\n" "| | Logical:\n" "| | cardinalityEstimate: \n" "| | ce: 10\n" @@ -137,7 +137,7 @@ TEST(PhysRewriter, PhysicalRewriterBasic) { "| | p2\n" "| RefBlock: \n" "| Variable [p2]\n" - "Properties [cost: 0.620002, localCost: 0.020001, adjustedCE: 10]\n" + "Properties [cost: 0.438321, localCost: 0.00983406, adjustedCE: 10]\n" "| | Logical:\n" "| | cardinalityEstimate: \n" "| | ce: 10\n" @@ -166,7 +166,7 @@ TEST(PhysRewriter, PhysicalRewriterBasic) { "| PathGet [a]\n" "| PathCompare [Eq]\n" "| Const [1]\n" - "Properties [cost: 0.600001, localCost: 0, adjustedCE: 100]\n" + "Properties [cost: 0.428487, localCost: 0, adjustedCE: 100]\n" "| | Logical:\n" "| | cardinalityEstimate: \n" "| | ce: 100\n" @@ -193,7 +193,7 @@ TEST(PhysRewriter, PhysicalRewriterBasic) { "| EvalPath []\n" "| | Variable [p1]\n" "| PathIdentity []\n" - "Properties [cost: 0.600001, localCost: 0, adjustedCE: 100]\n" + "Properties [cost: 0.428487, localCost: 0, adjustedCE: 100]\n" "| | Logical:\n" "| | cardinalityEstimate: \n" "| | ce: 100\n" @@ -217,7 +217,7 @@ TEST(PhysRewriter, PhysicalRewriterBasic) { "| EvalFilter []\n" "| | Variable [p1]\n" "| PathIdentity []\n" - "Properties [cost: 0.600001, localCost: 0.600001, adjustedCE: 1000]\n" + "Properties [cost: 0.428487, localCost: 0.428487, adjustedCE: 1000]\n" "| | Logical:\n" "| | cardinalityEstimate: \n" "| | ce: 1000\n" @@ -277,6 +277,7 @@ TEST(PhysRewriter, GroupBy) { OptPhase::MemoImplementationPhase}, prefixId, {{{"test", createScanDef({}, {})}}}, + /*costModel*/ boost::none, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -347,6 +348,7 @@ TEST(PhysRewriter, GroupBy1) { OptPhase::MemoImplementationPhase}, prefixId, {{{"test", createScanDef({}, {})}}}, + /*costModel*/ boost::none, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -419,6 +421,7 @@ TEST(PhysRewriter, Unwind) { OptPhase::MemoImplementationPhase}, prefixId, {{{"test", createScanDef({}, {})}}}, + /*costModel*/ boost::none, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -497,6 +500,7 @@ TEST(PhysRewriter, DuplicateFilter) { OptPhase::MemoImplementationPhase}, prefixId, {{{"c1", createScanDef({}, {})}}}, + /*costModel*/ boost::none, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -558,6 +562,7 @@ TEST(PhysRewriter, FilterCollation) { OptPhase::MemoImplementationPhase}, prefixId, {{{"c1", createScanDef({}, {})}}}, + /*costModel*/ boost::none, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -614,6 +619,7 @@ TEST(PhysRewriter, EvalCollation) { OptPhase::MemoImplementationPhase}, prefixId, {{{"c1", createScanDef({}, {})}}}, + /*costModel*/ boost::none, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -669,6 +675,7 @@ TEST(PhysRewriter, FilterEvalCollation) { OptPhase::MemoImplementationPhase}, prefixId, {{{"c1", createScanDef({}, {})}}}, + /*costModel*/ boost::none, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -724,6 +731,7 @@ TEST(PhysRewriter, FilterIndexing) { prefixId, {{{"c1", createScanDef({}, {{"index1", makeIndexDefinition("a", CollationOp::Ascending)}})}}}, + /*costModel*/ boost::none, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); // Demonstrate sargable node is rewritten from filter node. @@ -769,6 +777,7 @@ TEST(PhysRewriter, FilterIndexing) { prefixId, {{{"c1", createScanDef({}, {{"index1", makeIndexDefinition("a", CollationOp::Ascending)}})}}}, + /*costModel*/ boost::none, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -810,6 +819,7 @@ TEST(PhysRewriter, FilterIndexing) { OptPhase::MemoImplementationPhase}, prefixId, {{{"c1", createScanDef({}, {})}}}, + /*costModel*/ boost::none, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -875,6 +885,7 @@ TEST(PhysRewriter, FilterIndexing1) { prefixId, {{{"c1", createScanDef({}, {{"index1", makeIndexDefinition("a", CollationOp::Ascending)}})}}}, + /*costModel*/ boost::none, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -944,6 +955,7 @@ TEST(PhysRewriter, FilterIndexing2) { {{{make("a", make("b", make())), CollationOp::Ascending}}, false /*isMultiKey*/}}})}}}, + /*costModel*/ boost::none, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -1023,6 +1035,7 @@ TEST(PhysRewriter, FilterIndexing2NonSarg) { {}, {{"index1", makeIndexDefinition("a", CollationOp::Ascending, false /*isMultiKey*/)}})}}}, + /*costModel*/ boost::none, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -1145,6 +1158,7 @@ TEST(PhysRewriter, FilterIndexing3) { false /*isMultiKey*/, {DistributionType::Centralized}, {}}}})}}}, + /*costModel*/ boost::none, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -1199,6 +1213,7 @@ TEST(PhysRewriter, FilterIndexing3MultiKey) { true /*isMultiKey*/, {DistributionType::Centralized}, {}}}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -1296,6 +1311,7 @@ TEST(PhysRewriter, FilterIndexing4) { false /*isMultiKey*/, {DistributionType::Centralized}, {}}}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -1303,7 +1319,7 @@ TEST(PhysRewriter, FilterIndexing4) { // For now leave only GroupBy+Union RIDIntersect. phaseManager.getHints()._disableHashJoinRIDIntersect = true; phaseManager.optimize(optimized); - ASSERT_BETWEEN(60, 75, phaseManager.getMemo().getStats()._physPlanExplorationCount); + ASSERT_BETWEEN(20, 35, phaseManager.getMemo().getStats()._physPlanExplorationCount); // Assert the correct CEs for each node in group 1. Group 1 contains residual predicates. std::vector> pathAndCEs = { @@ -1401,11 +1417,12 @@ TEST(PhysRewriter, FilterIndexing5) { false /*isMultiKey*/, {DistributionType::Centralized}, {}}}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); phaseManager.optimize(optimized); - ASSERT_BETWEEN(20, 30, phaseManager.getMemo().getStats()._physPlanExplorationCount); + ASSERT_BETWEEN(10, 25, phaseManager.getMemo().getStats()._physPlanExplorationCount); // We can cover both fields with the index, and need separate sort on "b". ASSERT_EXPLAIN_V2( @@ -1490,11 +1507,12 @@ TEST(PhysRewriter, FilterIndexing6) { false /*isMultiKey*/, {DistributionType::Centralized}, {}}}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); phaseManager.optimize(optimized); - ASSERT_BETWEEN(5, 10, phaseManager.getMemo().getStats()._physPlanExplorationCount); + ASSERT_BETWEEN(5, 15, phaseManager.getMemo().getStats()._physPlanExplorationCount); // We can cover both fields with the index, and do not need a separate sort on "b". ASSERT_EXPLAIN_V2( @@ -1564,6 +1582,7 @@ TEST(PhysRewriter, FilterIndexingStress) { false /*isMultiKey*/, {DistributionType::Centralized}, {}}}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -1649,6 +1668,7 @@ TEST(PhysRewriter, FilterIndexingVariable) { {}, {{"index1", makeIndexDefinition("a", CollationOp::Ascending, false /*isMultiKey*/)}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -1740,6 +1760,7 @@ TEST(PhysRewriter, FilterIndexingMaxKey) { OptPhase::MemoImplementationPhase}, prefixId, {{{"c1", createScanDef({}, {})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -1806,6 +1827,7 @@ TEST(PhysRewriter, SargableProjectionRenames) { {OptPhase::MemoSubstitutionPhase}, prefixId, {{{"c1", createScanDef({}, {})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -1868,6 +1890,7 @@ TEST(PhysRewriter, SargableAcquireProjection) { {OptPhase::MemoSubstitutionPhase}, prefixId, {{{"c1", createScanDef({}, {})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -1937,6 +1960,7 @@ TEST(PhysRewriter, FilterReorder) { prefixId, {{{"c1", createScanDef({}, {})}}}, makeHintedCE(std::move(hints)), + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -2032,6 +2056,7 @@ TEST(PhysRewriter, CoveredScan) { {{"index1", makeIndexDefinition("a", CollationOp::Ascending, false /*isMultiKey*/)}})}}}, makeHintedCE(std::move(hints)), + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -2099,6 +2124,7 @@ TEST(PhysRewriter, EvalIndexing) { {}, {{"index1", makeIndexDefinition("a", CollationOp::Ascending, false /*isMultiKey*/)}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -2132,6 +2158,7 @@ TEST(PhysRewriter, EvalIndexing) { {}, {{"index1", makeIndexDefinition("a", CollationOp::Clustered, false /*isMultiKey*/)}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -2191,6 +2218,7 @@ TEST(PhysRewriter, EvalIndexing1) { {}, {{"index1", makeIndexDefinition("a", CollationOp::Ascending, false /*isMultiKey*/)}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -2263,12 +2291,13 @@ TEST(PhysRewriter, EvalIndexing2) { {}, {{"index1", makeIndexDefinition("a", CollationOp::Ascending, false /*isMultiKey*/)}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; phaseManager.getHints()._fastIndexNullHandling = true; phaseManager.optimize(optimized); - ASSERT_BETWEEN(10, 20, phaseManager.getMemo().getStats()._physPlanExplorationCount); + ASSERT_BETWEEN(8, 18, phaseManager.getMemo().getStats()._physPlanExplorationCount); // Verify collation is subsumed into the index scan. ASSERT_EXPLAIN_V2( @@ -2336,6 +2365,11 @@ TEST(PhysRewriter, MultiKeyIndex) { ABT rootNode = make(ProjectionRequirement{ProjectionNameVector{"root"}}, std::move(collationNode)); + // TODO SERVER-71551 Follow up unit tests with overriden Cost Model. + auto costModel = getTestCostModel(); + costModel.setEvalStartupCost(1e-6); + costModel.setGroupByIncrementalCost(1e-4); + auto phaseManager = makePhaseManager( {OptPhase::MemoSubstitutionPhase, OptPhase::MemoExplorationPhase, @@ -2348,6 +2382,7 @@ TEST(PhysRewriter, MultiKeyIndex) { {"index2", makeIndexDefinition("b", CollationOp::Descending, false /*isMultiKey*/)}})}}}, makeHintedCE(std::move(hints)), + costModel, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); { @@ -2556,6 +2591,10 @@ TEST(PhysRewriter, CompoundIndex1) { ABT rootNode = make(ProjectionRequirement{ProjectionNameVector{"root"}}, std::move(filterDNode)); + // TODO SERVER-71551 Follow up unit tests with overriden Cost Model. + auto costModel = getTestCostModel(); + costModel.setIndexScanStartupCost(1e-6); + auto phaseManager = makePhaseManager( {OptPhase::MemoSubstitutionPhase, OptPhase::MemoExplorationPhase, @@ -2572,6 +2611,7 @@ TEST(PhysRewriter, CompoundIndex1) { IndexDefinition{{{makeNonMultikeyIndexPath("b"), CollationOp::Ascending}, {makeNonMultikeyIndexPath("d"), CollationOp::Ascending}}, false /*isMultiKey*/}}})}}}, + std::move(costModel), {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -2641,6 +2681,10 @@ TEST(PhysRewriter, CompoundIndex2) { ABT rootNode = make(ProjectionRequirement{ProjectionNameVector{"root"}}, std::move(collationNode)); + // TODO SERVER-71551 Follow up unit tests with overriden Cost Model. + auto costModel = getTestCostModel(); + costModel.setIndexScanStartupCost(1e-6); + auto phaseManager = makePhaseManager( {OptPhase::MemoSubstitutionPhase, OptPhase::MemoExplorationPhase, @@ -2659,11 +2703,12 @@ TEST(PhysRewriter, CompoundIndex2) { {makeNonMultikeyIndexPath("d"), CollationOp::Ascending}}, false /*isMultiKey*/}}, })}}}, + std::move(costModel), {true /*debugMode*/, 3 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; phaseManager.optimize(optimized); - ASSERT_BETWEEN(60, 80, phaseManager.getMemo().getStats()._physPlanExplorationCount); + ASSERT_BETWEEN(50, 70, phaseManager.getMemo().getStats()._physPlanExplorationCount); const BSONObj& explainRoot = ExplainGenerator::explainBSONObj(optimized); ASSERT_BSON_PATH("\"BinaryJoin\"", explainRoot, "child.nodeType"); @@ -2728,6 +2773,10 @@ TEST(PhysRewriter, CompoundIndex3) { ABT rootNode = make(ProjectionRequirement{ProjectionNameVector{"root"}}, std::move(collationNode)); + // TODO SERVER-71551 Follow up unit tests with overriden Cost Model. + auto costModel = getTestCostModel(); + costModel.setIndexScanStartupCost(1e-6); + auto phaseManager = makePhaseManager( {OptPhase::MemoSubstitutionPhase, OptPhase::MemoExplorationPhase, @@ -2743,6 +2792,7 @@ TEST(PhysRewriter, CompoundIndex3) { IndexDefinition{{{makeIndexPath("b"), CollationOp::Ascending}, {makeIndexPath("d"), CollationOp::Ascending}}, true /*isMultiKey*/}}})}}}, + std::move(costModel), {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -2832,6 +2882,7 @@ TEST(PhysRewriter, CompoundIndex4Negative) { {makeNonMultikeyIndexPath("d"), CollationOp::Ascending}}, false /*isMultiKey*/}}})}}}, makeHintedCE(std::move(hints)), + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -2886,6 +2937,7 @@ TEST(PhysRewriter, CompoundIndex5) { IndexDefinition{{{makeNonMultikeyIndexPath("a"), CollationOp::Ascending}, {makeNonMultikeyIndexPath("b"), CollationOp::Ascending}}, false /*isMultiKey*/}}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -2987,11 +3039,12 @@ TEST(PhysRewriter, IndexBoundsIntersect) { IndexDefinition{{{makeIndexPath("b"), CollationOp::Ascending}, {makeIndexPath("a"), CollationOp::Ascending}}, true /*isMultiKey*/}}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; phaseManager.optimize(optimized); - ASSERT_BETWEEN(20, 30, phaseManager.getMemo().getStats()._physPlanExplorationCount); + ASSERT_BETWEEN(10, 20, phaseManager.getMemo().getStats()._physPlanExplorationCount); // Demonstrate that the predicates >70 and <90 are NOT combined into the same interval (70, 90) // since the paths are multiKey. With the heuristic estimate we may get either interval in the @@ -3062,6 +3115,7 @@ TEST(PhysRewriter, IndexBoundsIntersect1) { {{"index1", IndexDefinition{{{makeNonMultikeyIndexPath("a"), CollationOp::Ascending}}, false /*isMultiKey*/}}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -3119,6 +3173,11 @@ TEST(PhysRewriter, IndexBoundsIntersect2) { ABT rootNode = make(ProjectionRequirement{ProjectionNameVector{"root"}}, std::move(filterNode)); + // TODO SERVER-71551 Follow up unit tests with overriden Cost Model. + auto costModel = getTestCostModel(); + costModel.setSeekStartupCost(1e-6); + costModel.setIndexScanStartupCost(1e-6); + auto phaseManager = makePhaseManager( {OptPhase::MemoSubstitutionPhase, OptPhase::MemoExplorationPhase, @@ -3129,6 +3188,7 @@ TEST(PhysRewriter, IndexBoundsIntersect2) { {{"index1", IndexDefinition{{{makeIndexPath("a"), CollationOp::Ascending}}, true /*isMultiKey*/}}})}}}, + std::move(costModel), {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -3209,6 +3269,7 @@ TEST(PhysRewriter, IndexBoundsIntersect3) { IndexDefinition{{{makeIndexPath(FieldPathType{"a", "c"}, true /*isMultiKey*/), CollationOp::Ascending}}, true /*isMultiKey*/}}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -3284,15 +3345,16 @@ TEST(PhysRewriter, IndexResidualReq) { IndexDefinition{{{makeNonMultikeyIndexPath("a"), CollationOp::Ascending}, {makeNonMultikeyIndexPath("b"), CollationOp::Ascending}}, false /*isMultiKey*/}}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; phaseManager.optimize(optimized); - ASSERT_BETWEEN(10, 20, phaseManager.getMemo().getStats()._physPlanExplorationCount); + ASSERT_BETWEEN(5, 15, phaseManager.getMemo().getStats()._physPlanExplorationCount); // Make sure we can use the index to cover "b" while testing "b.c" with a separate filter. ASSERT_EXPLAIN_PROPS_V2( - "Properties [cost: 0.231002, localCost: 0, adjustedCE: 189.571]\n" + "Properties [cost: 0.176361, localCost: 0, adjustedCE: 189.571]\n" "| | Logical:\n" "| | cardinalityEstimate: \n" "| | ce: 189.571\n" @@ -3316,7 +3378,7 @@ TEST(PhysRewriter, IndexResidualReq) { "| | pa\n" "| RefBlock: \n" "| Variable [pa]\n" - "Properties [cost: 0.231002, localCost: 0.231002, adjustedCE: 330]\n" + "Properties [cost: 0.176361, localCost: 0.176361, adjustedCE: 330]\n" "| | Logical:\n" "| | cardinalityEstimate: \n" "| | ce: 189.571\n" @@ -3414,12 +3476,13 @@ TEST(PhysRewriter, IndexResidualReq1) { makeCompositeIndexDefinition({{"a", CollationOp::Ascending, false /*isMultiKey*/}, {"d", CollationOp::Ascending, false /*isMultiKey*/}}, false /*isMultiKey*/)}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; phaseManager.getHints()._fastIndexNullHandling = true; phaseManager.optimize(optimized); - ASSERT_BETWEEN(50, 75, phaseManager.getMemo().getStats()._physPlanExplorationCount); + ASSERT_BETWEEN(65, 90, phaseManager.getMemo().getStats()._physPlanExplorationCount); // Prefer index1 over index2 and index3 in order to cover all fields. ASSERT_EXPLAIN_V2( @@ -3485,6 +3548,7 @@ TEST(PhysRewriter, IndexResidualReq2) { {{"a", CollationOp::Ascending, true /*isMultiKey*/}, {"c", CollationOp::Ascending, true /*isMultiKey*/}, {"b", CollationOp::Ascending, true /*isMultiKey*/}})}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -3558,6 +3622,7 @@ TEST(PhysRewriter, ElemMatchIndex) { prefixId, {{{"c1", createScanDef({}, {{"index1", makeIndexDefinition("a", CollationOp::Ascending)}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -3641,6 +3706,7 @@ TEST(PhysRewriter, ElemMatchIndex1) { makeCompositeIndexDefinition( {{"b", CollationOp::Ascending, true /*isMultiKey*/}, {"a", CollationOp::Ascending, true /*isMultiKey*/}})}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -3717,6 +3783,7 @@ TEST(PhysRewriter, ElemMatchIndexNoArrays) { {}, {{"index1", makeIndexDefinition("a", CollationOp::Ascending, false /*multiKey*/)}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -3783,11 +3850,12 @@ TEST(PhysRewriter, ObjectElemMatchResidual) { makeCompositeIndexDefinition( {{"b", CollationOp::Ascending, true /*isMultiKey*/}, {"a", CollationOp::Ascending, true /*isMultiKey*/}})}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; phaseManager.optimize(optimized); - ASSERT_BETWEEN(40, 50, phaseManager.getMemo().getStats()._physPlanExplorationCount); + ASSERT_BETWEEN(20, 30, phaseManager.getMemo().getStats()._physPlanExplorationCount); // We should pick the index, and do at least some filtering before the fetch. // We don't have index bounds, both because 'a' is not the first field of the index, @@ -3898,9 +3966,8 @@ TEST(PhysRewriter, ObjectElemMatchBounds) { {{"index1", IndexDefinition{{{makeIndexPath(FieldPathType{"a", "b"}, true /*isMultiKey*/), CollationOp::Ascending}}, - true /*isMultiKey*/}}} - - )}}}, + true /*isMultiKey*/}}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -3970,6 +4037,12 @@ TEST(PhysRewriter, NestedElemMatch) { ABT rootNode = make(ProjectionRequirement{ProjectionNameVector{"root"}}, std::move(filterNode)); + // TODO SERVER-71551 Follow up unit tests with overriden Cost Model. + auto costModel = getTestCostModel(); + costModel.setGroupByStartupCost(1e-6); + costModel.setGroupByIncrementalCost(1e-4); + costModel.setEvalStartupCost(1e-6); + auto phaseManager = makePhaseManager( {OptPhase::MemoSubstitutionPhase, OptPhase::MemoExplorationPhase, @@ -3980,6 +4053,7 @@ TEST(PhysRewriter, NestedElemMatch) { {}, {{"index1", makeIndexDefinition("a", CollationOp::Ascending, true /*isMultiKey*/)}})}}}, + std::move(costModel), {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -4098,6 +4172,7 @@ TEST(PhysRewriter, PathObj) { {{"a", CollationOp::Ascending, false /*isMultiKey*/}, {"b", CollationOp::Ascending, true /*isMultiKey*/}})}})}}}, makeHintedCE(std::move(hints)), + boost::none /*costModel*/, DebugInfo{true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}, {} /*hints*/); @@ -4181,6 +4256,7 @@ TEST(PhysRewriter, ArrayConstantIndex) { makeCompositeIndexDefinition( {{"b", CollationOp::Ascending, true /*isMultiKey*/}, {"a", CollationOp::Ascending, true /*isMultiKey*/}})}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -4281,6 +4357,7 @@ TEST(PhysRewriter, ArrayConstantNoIndex) { OptPhase::MemoImplementationPhase}, prefixId, {{{"c1", createScanDef({}, {})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -4345,6 +4422,7 @@ TEST(PhysRewriter, ParallelScan) { {{{"c1", createScanDef({}, {}, ConstEval::constFold, {DistributionType::UnknownPartitioning})}}, 5 /*numberOfPartitions*/}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -4411,6 +4489,7 @@ TEST(PhysRewriter, HashPartitioning) { {DistributionType::HashPartitioning, makeSeq(make("a", make()))})}}, 5 /*numberOfPartitions*/}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -4501,11 +4580,12 @@ TEST(PhysRewriter, IndexPartitioning) { {DistributionType::HashPartitioning, makeSeq(makeNonMultikeyIndexPath("b"))})}}, 5 /*numberOfPartitions*/}, makeHintedCE(std::move(hints)), + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; phaseManager.optimize(optimized); - ASSERT_BETWEEN(75, 125, phaseManager.getMemo().getStats()._physPlanExplorationCount); + ASSERT_BETWEEN(60, 100, phaseManager.getMemo().getStats()._physPlanExplorationCount); ASSERT_EXPLAIN_V2( "Root []\n" @@ -4600,6 +4680,11 @@ TEST(PhysRewriter, IndexPartitioning1) { ABT rootNode = make(ProjectionRequirement{ProjectionNameVector{"pc"}}, std::move(groupByNode)); + // TODO SERVER-71551 Follow up unit tests with overriden Cost Model. + auto costModel = getTestCostModel(); + costModel.setBinaryJoinIncrementalCost(0.002); + costModel.setHashJoinIncrementalCost(5e-5); + auto phaseManager = makePhaseManager( {OptPhase::MemoSubstitutionPhase, OptPhase::MemoExplorationPhase, @@ -4624,11 +4709,12 @@ TEST(PhysRewriter, IndexPartitioning1) { {DistributionType::HashPartitioning, makeSeq(makeNonMultikeyIndexPath("c"))})}}, 5 /*numberOfPartitions*/}, makeHintedCE(std::move(hints)), + std::move(costModel), {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; phaseManager.optimize(optimized); - ASSERT_BETWEEN(125, 175, phaseManager.getMemo().getStats()._physPlanExplorationCount); + ASSERT_BETWEEN(110, 160, phaseManager.getMemo().getStats()._physPlanExplorationCount); const BSONObj& result = ExplainGenerator::explainBSONObj(optimized); @@ -4684,6 +4770,7 @@ TEST(PhysRewriter, LocalGlobalAgg) { {{{"c1", createScanDef({}, {}, ConstEval::constFold, {DistributionType::UnknownPartitioning})}}, 5 /*numberOfPartitions*/}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -4762,6 +4849,7 @@ TEST(PhysRewriter, LocalGlobalAgg1) { {{{"c1", createScanDef({}, {}, ConstEval::constFold, {DistributionType::UnknownPartitioning})}}, 5 /*numberOfPartitions*/}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -4817,6 +4905,7 @@ TEST(PhysRewriter, LocalLimitSkip) { {{{"c1", createScanDef({}, {}, ConstEval::constFold, {DistributionType::UnknownPartitioning})}}, 5 /*numberOfPartitions*/}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -4824,7 +4913,7 @@ TEST(PhysRewriter, LocalLimitSkip) { ASSERT_BETWEEN(5, 15, phaseManager.getMemo().getStats()._physPlanExplorationCount); ASSERT_EXPLAIN_PROPS_V2( - "Properties [cost: 0.0066022, localCost: 0, adjustedCE: 20]\n" + "Properties [cost: 0.00929774, localCost: 0, adjustedCE: 20]\n" "| | Logical:\n" "| | cardinalityEstimate: \n" "| | ce: 20\n" @@ -4845,7 +4934,7 @@ TEST(PhysRewriter, LocalLimitSkip) { "| | root\n" "| RefBlock: \n" "| Variable [root]\n" - "Properties [cost: 0.0066022, localCost: 1e-06, adjustedCE: 30]\n" + "Properties [cost: 0.00929774, localCost: 0.00252777, adjustedCE: 30]\n" "| | Logical:\n" "| | cardinalityEstimate: \n" "| | ce: 1000\n" @@ -4872,7 +4961,7 @@ TEST(PhysRewriter, LocalLimitSkip) { "| limitSkip:\n" "| limit: 20\n" "| skip: 10\n" - "Properties [cost: 0.0066012, localCost: 0.003001, adjustedCE: 30]\n" + "Properties [cost: 0.00676997, localCost: 0.003001, adjustedCE: 30]\n" "| | Logical:\n" "| | cardinalityEstimate: \n" "| | ce: 1000\n" @@ -4897,7 +4986,7 @@ TEST(PhysRewriter, LocalLimitSkip) { "| | distribution: \n" "| | type: Centralized\n" "| RefBlock: \n" - "Properties [cost: 0.0036002, localCost: 0.0036002, adjustedCE: 30]\n" + "Properties [cost: 0.00376897, localCost: 0.00376897, adjustedCE: 30]\n" "| | Logical:\n" "| | cardinalityEstimate: \n" "| | ce: 1000\n" @@ -4949,6 +5038,7 @@ TEST(PhysRewriter, CollationLimit) { OptPhase::MemoImplementationPhase}, prefixId, {{{"c1", createScanDef({}, {})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -4958,7 +5048,7 @@ TEST(PhysRewriter, CollationLimit) { // We have a collation node with limit-skip physical properties. It will be lowered to a // sort node with limit. ASSERT_EXPLAIN_PROPS_V2( - "Properties [cost: 4.92193, localCost: 0, adjustedCE: 20]\n" + "Properties [cost: 4.75042, localCost: 0, adjustedCE: 20]\n" "| | Logical:\n" "| | cardinalityEstimate: \n" "| | ce: 20\n" @@ -4978,7 +5068,7 @@ TEST(PhysRewriter, CollationLimit) { "| | root\n" "| RefBlock: \n" "| Variable [root]\n" - "Properties [cost: 4.92193, localCost: 4.32193, adjustedCE: 20]\n" + "Properties [cost: 4.75042, localCost: 4.32193, adjustedCE: 20]\n" "| | Logical:\n" "| | cardinalityEstimate: \n" "| | ce: 1000\n" @@ -5012,7 +5102,7 @@ TEST(PhysRewriter, CollationLimit) { "| | pa: Ascending\n" "| RefBlock: \n" "| Variable [pa]\n" - "Properties [cost: 0.600001, localCost: 0.600001, adjustedCE: 1000]\n" + "Properties [cost: 0.428487, localCost: 0.428487, adjustedCE: 1000]\n" "| | Logical:\n" "| | cardinalityEstimate: \n" "| | ce: 1000\n" @@ -5096,6 +5186,7 @@ TEST(PhysRewriter, PartialIndex1) { true /*isMultiKey*/, {DistributionType::Centralized}, std::move(conversionResult->_reqMap)}}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -5177,6 +5268,7 @@ TEST(PhysRewriter, PartialIndex2) { true /*isMultiKey*/, {DistributionType::Centralized}, std::move(conversionResult->_reqMap)}}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -5257,6 +5349,7 @@ TEST(PhysRewriter, PartialIndexReject) { true /*isMultiKey*/, {DistributionType::Centralized}, std::move(conversionResult->_reqMap)}}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -5429,6 +5522,7 @@ TEST(PhysRewriter, UnionRewrite) { OptPhase::MemoImplementationPhase}, prefixId, {{{"test1", createScanDef({}, {})}, {"test2", createScanDef({}, {})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -5498,6 +5592,7 @@ TEST(PhysRewriter, JoinRewrite) { OptPhase::MemoImplementationPhase}, prefixId, {{{"test1", createScanDef({}, {})}, {"test2", createScanDef({}, {})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -5578,6 +5673,7 @@ TEST(PhysRewriter, JoinRewrite1) { {{"index1", {{{makeNonMultikeyIndexPath("b"), CollationOp::Ascending}}, false /*isMultiKey*/}}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = std::move(rootNode); @@ -5628,6 +5724,7 @@ TEST(PhysRewriter, RootInterval) { OptPhase::MemoImplementationPhase}, prefixId, {{{"c1", createScanDef({}, {})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -5684,6 +5781,7 @@ TEST(PhysRewriter, EqMemberSargable) { {}, {{"index1", makeIndexDefinition("a", CollationOp::Ascending, false /*isMultiKey*/)}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -5726,6 +5824,7 @@ TEST(PhysRewriter, EqMemberSargable) { prefixId, {{{"c1", createScanDef({}, {{"index1", makeIndexDefinition("a", CollationOp::Ascending)}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; @@ -5829,11 +5928,12 @@ TEST(PhysRewriter, IndexSubfieldCovered) { {}, {{"index1", makeIndexDefinition("a", CollationOp::Ascending, false /*isMultiKey*/)}})}}}, + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; phaseManager.optimize(optimized); - ASSERT_BETWEEN(35, 50, phaseManager.getMemo().getStats()._physPlanExplorationCount); + ASSERT_BETWEEN(20, 35, phaseManager.getMemo().getStats()._physPlanExplorationCount); // Observe we have a covered plan. The filters for subfields "b" and "c" are expressed as // residual predicates. Also observe the traverse for "a.c" is removed due to "a" being @@ -5916,12 +6016,13 @@ TEST(PhysRewriter, PerfOnlyPreds1) { {"a", CollationOp::Ascending, false /*isMultiKey*/}}, false /*isMultiKey*/)}})}}}, makeHintedCE(std::move(hints)), + boost::none /*costModel*/, {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; phaseManager.getHints()._disableYieldingTolerantPlans = false; phaseManager.optimize(optimized); - ASSERT_BETWEEN(10, 15, phaseManager.getMemo().getStats()._physPlanExplorationCount); + ASSERT_BETWEEN(15, 20, phaseManager.getMemo().getStats()._physPlanExplorationCount); // Demonstrate predicates are repeated on the Seek side. Also demonstrate null handling, and the // fact that we apply the predicates on the Seek side in increasing selectivity order. @@ -5991,6 +6092,12 @@ TEST(PhysRewriter, PerfOnlyPreds2) { ABT rootNode = make(ProjectionRequirement{ProjectionNameVector{"pa"}}, std::move(filterNode2)); + // TODO SERVER-71551 Follow up unit tests with overriden Cost Model. + auto costModel = getTestCostModel(); + costModel.setSeekStartupCost(1e-6); + costModel.setIndexScanStartupCost(1e-6); + costModel.setMergeJoinStartupCost(1e-6); + PrefixId prefixId; auto phaseManager = makePhaseManager( {OptPhase::MemoSubstitutionPhase, @@ -6004,6 +6111,7 @@ TEST(PhysRewriter, PerfOnlyPreds2) { {"index2", makeIndexDefinition("b", CollationOp::Ascending, false /*isMultiKey*/)}})}}}, makeHintedCE(std::move(hints)), + std::move(costModel), {true /*debugMode*/, 2 /*debugLevel*/, DebugInfo::kIterationLimitForTests}); ABT optimized = rootNode; diff --git a/src/mongo/db/query/optimizer/utils/unit_test_pipeline_utils.cpp b/src/mongo/db/query/optimizer/utils/unit_test_pipeline_utils.cpp index 0bac198c20c..eeadc6d7d40 100644 --- a/src/mongo/db/query/optimizer/utils/unit_test_pipeline_utils.cpp +++ b/src/mongo/db/query/optimizer/utils/unit_test_pipeline_utils.cpp @@ -204,14 +204,67 @@ ABT translatetoABT(const std::string& pipelineStr, metadata, pipelineStr, prefixId.getNextId("scan"), scanDefName, prefixId, involvedNss); } +cost_model::CostModelCoefficients getPipelineTestDefaultCoefficients() { + // These cost should reflect estimated aggregated execution time in milliseconds. + // The coeffeicient us converts values from microseconds to milliseconds. + cost_model::CostModelCoefficients coefficients{}; + constexpr double usToMs = 1.0e-3; + coefficients.setDefaultStartupCost(0.000001); + coefficients.setScanIncrementalCost(0.6 * usToMs); + coefficients.setScanStartupCost(0.000001); + coefficients.setIndexScanIncrementalCost(0.5 * usToMs); + coefficients.setIndexScanStartupCost(0.000001); + coefficients.setSeekCost(2.0 * usToMs); + coefficients.setSeekStartupCost(0.000001); + coefficients.setFilterIncrementalCost(0.2 * usToMs); + coefficients.setFilterStartupCost(0.000001); + coefficients.setEvalIncrementalCost(2.0 * usToMs); + coefficients.setEvalStartupCost(0.000001); + coefficients.setGroupByIncrementalCost(0.07 * usToMs); + coefficients.setGroupByStartupCost(0.000001); + + coefficients.setUnwindIncrementalCost(0.03 * usToMs); + coefficients.setUnwindStartupCost(0.000001); + + coefficients.setBinaryJoinIncrementalCost(0.2 * usToMs); + coefficients.setBinaryJoinStartupCost(0.000001); + + coefficients.setHashJoinIncrementalCost(0.05 * usToMs); + coefficients.setHashJoinStartupCost(0.000001); + + coefficients.setMergeJoinIncrementalCost(0.02 * usToMs); + coefficients.setMergeJoinStartupCost(0.000001); + + coefficients.setUniqueIncrementalCost(0.7 * usToMs); + coefficients.setUniqueStartupCost(0.000001); + + coefficients.setCollationIncrementalCost(2.5 * usToMs); + coefficients.setCollationStartupCost(0.000001); + + coefficients.setCollationWithLimitIncrementalCost(1.0 * usToMs); + coefficients.setCollationWithLimitStartupCost(0.000001); + + coefficients.setUnionIncrementalCost(0.02 * usToMs); + coefficients.setUnionStartupCost(0.000001); + + coefficients.setExchangeIncrementalCost(0.1 * usToMs); + coefficients.setExchangeStartupCost(0.000001); + + coefficients.setLimitSkipIncrementalCost(0.0 * usToMs); + coefficients.setLimitSkipStartupCost(0.000001); + return coefficients; +} + ABT optimizeABT(ABT abt, opt::unordered_set phaseSet, Metadata metadata, + cost_model::CostModelCoefficients&& costModel, PathToIntervalFn pathToInterval, bool phaseManagerDisableScan) { PrefixId prefixId; - auto phaseManager = makePhaseManager(phaseSet, prefixId, metadata, DebugInfo::kDefaultForTests); + auto phaseManager = makePhaseManager( + phaseSet, prefixId, metadata, std::move(costModel), DebugInfo::kDefaultForTests); if (phaseManagerDisableScan) { phaseManager.getHints()._disableScan = true; } @@ -249,8 +302,13 @@ std::string testABTTranslationAndOptimization( std::string explained; if (optimizePipeline) { - ABT optimized = - optimizeABT(translated, phaseSet, metadata, pathToInterval, phaseManagerDisableScan); + ABT optimized = optimizeABT(translated, + phaseSet, + metadata, + // TODO SERVER-71554 + getPipelineTestDefaultCoefficients(), + pathToInterval, + phaseManagerDisableScan); explained = ExplainGenerator::explainV2(optimized); } else { explained = ExplainGenerator::explainV2(translated); @@ -259,5 +317,4 @@ std::string testABTTranslationAndOptimization( stream << explained << std::endl << std::endl; return explained; } - } // namespace mongo::optimizer diff --git a/src/mongo/db/query/optimizer/utils/unit_test_utils.cpp b/src/mongo/db/query/optimizer/utils/unit_test_utils.cpp index e261c5ac537..3286896f3f6 100644 --- a/src/mongo/db/query/optimizer/utils/unit_test_utils.cpp +++ b/src/mongo/db/query/optimizer/utils/unit_test_utils.cpp @@ -254,66 +254,61 @@ std::unique_ptr makeHintedCE(ce::PartialSchemaSelHints hin return std::make_unique(std::move(hints)); } +cost_model::CostModelCoefficients getTestCostModel() { + return cost_model::CostModelManager::getDefaultCoefficients(); +} + std::unique_ptr makeCostEstimator() { - return std::make_unique( - cost_model::CostModelManager::getDefaultCoefficients()); + return makeCostEstimator(getTestCostModel()); } -OptPhaseManager makePhaseManager(OptPhaseManager::PhaseSet phaseSet, - PrefixId& prefixId, - Metadata metadata, - DebugInfo debugInfo, - QueryHints queryHints) { +std::unique_ptr makeCostEstimator( + const cost_model::CostModelCoefficients& costModel) { + return std::make_unique(costModel); +} + + +OptPhaseManager makePhaseManager( + OptPhaseManager::PhaseSet phaseSet, + PrefixId& prefixId, + Metadata metadata, + const boost::optional& costModel, + DebugInfo debugInfo, + QueryHints queryHints) { return OptPhaseManager{std::move(phaseSet), prefixId, false /*requireRID*/, std::move(metadata), makeHeuristicCE(), // primary CE makeHeuristicCE(), // substitution phase CE, same as primary - makeCostEstimator(), + makeCostEstimator(costModel ? *costModel : getTestCostModel()), defaultConvertPathToInterval, ConstEval::constFold, std::move(debugInfo), std::move(queryHints)}; } -OptPhaseManager makePhaseManager(OptPhaseManager::PhaseSet phaseSet, - PrefixId& prefixId, - Metadata metadata, - std::unique_ptr ce, - DebugInfo debugInfo, - QueryHints queryHints) { +OptPhaseManager makePhaseManager( + OptPhaseManager::PhaseSet phaseSet, + PrefixId& prefixId, + Metadata metadata, + std::unique_ptr ce, + const boost::optional& costModel, + DebugInfo debugInfo, + QueryHints queryHints) { return OptPhaseManager{std::move(phaseSet), prefixId, false /*requireRID*/, std::move(metadata), std::move(ce), // primary CE makeHeuristicCE(), // substitution phase CE - makeCostEstimator(), + makeCostEstimator(costModel ? *costModel : getTestCostModel()), defaultConvertPathToInterval, ConstEval::constFold, std::move(debugInfo), std::move(queryHints)}; } -OptPhaseManager makePhaseManager(OptPhaseManager::PhaseSet phaseSet, - PrefixId& prefixId, - Metadata metadata, - DebugInfo debugInfo, - mongo::cost_model::CostModelCoefficients coefs, - QueryHints queryHints) { - return OptPhaseManager{std::move(phaseSet), - prefixId, - false /*requireRID*/, - std::move(metadata), - makeHeuristicCE(), // primary CE - makeHeuristicCE(), // substitution phase CE, same as primary - std::make_unique(coefs), - defaultConvertPathToInterval, - ConstEval::constFold, - std::move(debugInfo), - std::move(queryHints)}; -} OptPhaseManager makePhaseManagerRequireRID(OptPhaseManager::PhaseSet phaseSet, PrefixId& prefixId, diff --git a/src/mongo/db/query/optimizer/utils/unit_test_utils.h b/src/mongo/db/query/optimizer/utils/unit_test_utils.h index a1e6549e52f..d3e2559c3ee 100644 --- a/src/mongo/db/query/optimizer/utils/unit_test_utils.h +++ b/src/mongo/db/query/optimizer/utils/unit_test_utils.h @@ -176,38 +176,44 @@ std::unique_ptr makeHeuristicCE(); std::unique_ptr makeHintedCE(ce::PartialSchemaSelHints hints); /** - * A convenience factory function to create costing. + * Return default CostModel used in unit tests. + */ +cost_model::CostModelCoefficients getTestCostModel(); + +/* + * A convenience factory function to create costing with default CostModel. */ std::unique_ptr makeCostEstimator(); /** - * A convenience factory function to create OptPhaseManager for unit tests. + * A convenience factory function to create costing with overriden CostModel. */ -OptPhaseManager makePhaseManager(OptPhaseManager::PhaseSet phaseSet, - PrefixId& prefixId, - Metadata metadata, - DebugInfo debugInfo, - QueryHints queryHints = {}); +std::unique_ptr makeCostEstimator( + const cost_model::CostModelCoefficients& costModel); /** - * A convenience factory function to create OptPhaseManager for unit tests with CE hints. + * A convenience factory function to create OptPhaseManager for unit tests with cost model. */ -OptPhaseManager makePhaseManager(OptPhaseManager::PhaseSet phaseSet, - PrefixId& prefixId, - Metadata metadata, - std::unique_ptr ce, - DebugInfo debugInfo, - QueryHints queryHints = {}); +OptPhaseManager makePhaseManager( + OptPhaseManager::PhaseSet phaseSet, + PrefixId& prefixId, + Metadata metadata, + const boost::optional& costModel, + DebugInfo debugInfo, + QueryHints queryHints = {}); /** - * A convenience factory function to create OptPhaseManager for unit tests with cost models. + * A convenience factory function to create OptPhaseManager for unit tests with CE hints and cost + * model. */ -OptPhaseManager makePhaseManager(OptPhaseManager::PhaseSet phaseSet, - PrefixId& prefixId, - Metadata metadata, - DebugInfo debugInfo, - mongo::cost_model::CostModelCoefficients coefs, - QueryHints queryHints = {}); +OptPhaseManager makePhaseManager( + OptPhaseManager::PhaseSet phaseSet, + PrefixId& prefixId, + Metadata metadata, + std::unique_ptr ce, + const boost::optional& costModel, + DebugInfo debugInfo, + QueryHints queryHints = {}); /** * A convenience factory function to create OptPhaseManager for unit tests which requires RID. -- cgit v1.2.1