diff options
42 files changed, 1223 insertions, 381 deletions
diff --git a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml index 98da98b5108..3fff159880e 100644 --- a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml +++ b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml @@ -31,12 +31,18 @@ selector: # Enable when 4.2 becomes last-stable. - jstests/sharding/aggregation_internal_parameters.js - jstests/sharding/agg_error_reports_shard_host_and_port.js + # mongos in 4.0 doesn't like an aggregation explain without stages for optimized away pipelines, + # so blacklisting the test until 4.2 becomes last-stable. + - jstests/sharding/agg_explain_fmt.js - jstests/sharding/change_stream_metadata_notifications.js - jstests/sharding/change_stream_transaction_sharded.js - jstests/sharding/change_streams.js - jstests/sharding/collation_lookup.js - jstests/sharding/collation_targeting.js - jstests/sharding/collation_targeting_inherited.js + # mongos in 4.0 doesn't like an aggregation explain without stages for optimized away pipelines, + # so blacklisting the test until 4.2 becomes last-stable. + - jstests/sharding/explain_agg_read_pref.js - jstests/sharding/explain_cmd.js - jstests/sharding/failcommand_failpoint_not_parallel.js - jstests/sharding/failcommand_ignores_internal.js @@ -68,6 +74,9 @@ selector: - jstests/sharding/update_immutable_fields.js - jstests/sharding/update_shard_key_doc_on_same_shard.js - jstests/sharding/update_shard_key_doc_moves_shards.js + # mongos in 4.0 doesn't like an aggregation explain without stages for optimized away pipelines, + # so blacklisting the test until 4.2 becomes last-stable. + - jstests/sharding/views.js # TODO: SERVER-38541 remove from blacklist - jstests/sharding/shard_collection_existing_zones.js - jstests/sharding/single_shard_transaction_with_arbiter.js diff --git a/jstests/aggregation/optimize_away_pipeline.js b/jstests/aggregation/optimize_away_pipeline.js new file mode 100644 index 00000000000..191a98023d4 --- /dev/null +++ b/jstests/aggregation/optimize_away_pipeline.js @@ -0,0 +1,338 @@ +// Tests that an aggregation pipeline can be optimized away and the query can be answered using +// just the query layer if the pipeline has only one $cursor source, or if the pipeline can be +// collapsed into a single $cursor source pipeline. The resulting cursor in this case will look +// like what the client would have gotten from find command. +// +// Relies on the pipeline stages to be collapsed into a single $cursor stage, so pipelines cannot +// be wrapped into a facet stage to not prevent this optimization. +// TODO SERVER-40323: Plan analyzer helper functions cannot correctly handle explain output for +// sharded collections. +// @tags: [do_not_wrap_aggregations_in_facets, assumes_unsharded_collection] +(function() { + "use strict"; + + load("jstests/aggregation/extras/utils.js"); // For 'orderedArrayEq' and 'arrayEq'. + load("jstests/concurrency/fsm_workload_helpers/server_types.js"); // For isWiredTiger. + load("jstests/libs/analyze_plan.js"); // For 'aggPlanHasStage' and other explain helpers. + load("jstests/libs/fixture_helpers.js"); // For 'isMongos' and 'isSharded'. + + const coll = db.optimize_away_pipeline; + coll.drop(); + assert.writeOK(coll.insert({_id: 1, x: 10})); + assert.writeOK(coll.insert({_id: 2, x: 20})); + assert.writeOK(coll.insert({_id: 3, x: 30})); + + // Asserts that the give pipeline has *not* been optimized away and the request is answered + // using the aggregation module. There should be pipeline stages present in the explain output. + // The functions also asserts that a query stage passed in the 'stage' argument is present in + // the explain output. If 'expectedResult' is provided, the pipeline is executed and the + // returned result as validated agains the expected result without respecting the order of the + // documents. If 'preserveResultOrder' is 'true' - the order is respected. + function assertPipelineUsesAggregation({ + pipeline = [], + pipelineOptions = {}, + expectedStage = null, + expectedResult = null, + preserveResultOrder = false + } = {}) { + const explainOutput = coll.explain().aggregate(pipeline, pipelineOptions); + + assert(isAggregationPlan(explainOutput), + "Expected pipeline " + tojsononeline(pipeline) + + " to use an aggregation framework in the explain output: " + + tojson(explainOutput)); + assert(!isQueryPlan(explainOutput), + "Expected pipeline " + tojsononeline(pipeline) + + " *not* to use a query layer at the root level in the explain output: " + + tojson(explainOutput)); + + let cursor = getAggPlanStage(explainOutput, "$cursor"); + if (cursor) { + cursor = cursor.$cursor; + } else { + cursor = getAggPlanStage(explainOutput, "$geoNearCursor").$geoNearCursor; + } + + assert(cursor, + "Expected pipeline " + tojsononeline(pipeline) + " to include a $cursor " + + " stage in the explain output: " + tojson(explainOutput)); + assert(cursor.queryPlanner.optimizedPipeline === undefined, + "Expected pipeline " + tojsononeline(pipeline) + " to *not* include an " + + "'optimizedPipeline' field in the explain output: " + tojson(explainOutput)); + assert(aggPlanHasStage(explainOutput, expectedStage), + "Expected pipeline " + tojsononeline(pipeline) + " to include a " + expectedStage + + " stage in the explain output: " + tojson(explainOutput)); + + if (expectedResult) { + const actualResult = coll.aggregate(pipeline, pipelineOptions).toArray(); + assert(preserveResultOrder ? orderedArrayEq(actualResult, expectedResult) + : arrayEq(actualResult, expectedResult)); + } + + return explainOutput; + } + + // Asserts that the give pipeline has been optimized away and the request is answered using + // just the query module. There should be no pipeline stages present in the explain output. + // The functions also asserts that a query stage passed in the 'stage' argument is present in + // the explain output. If 'expectedResult' is provided, the pipeline is executed and the + // returned result as validated agains the expected result without respecting the order of the + // documents. If 'preserveResultOrder' is 'true' - the order is respected. + function assertPipelineDoesNotUseAggregation({ + pipeline = [], + pipelineOptions = {}, + expectedStage = null, + expectedResult = null, + preserveResultOrder = false + } = {}) { + const explainOutput = coll.explain().aggregate(pipeline, pipelineOptions); + + assert(!isAggregationPlan(explainOutput), + "Expected pipeline " + tojsononeline(pipeline) + + " *not* to use an aggregation framework in the explain output: " + + tojson(explainOutput)); + assert(isQueryPlan(explainOutput), + "Expected pipeline " + tojsononeline(pipeline) + + " to use a query layer at the root level in the explain output: " + + tojson(explainOutput)); + if (explainOutput.hasOwnProperty("shards")) { + Object.keys(explainOutput.shards) + .forEach((shard) => assert( + explainOutput.shards[shard].queryPlanner.optimizedPipeline === true, + "Expected pipeline " + tojsononeline(pipeline) + " to include an " + + "'optimizedPipeline' field in the explain output: " + + tojson(explainOutput))); + } else { + assert(explainOutput.queryPlanner.optimizedPipeline === true, + "Expected pipeline " + tojsononeline(pipeline) + " to include an " + + "'optimizedPipeline' field in the explain output: " + tojson(explainOutput)); + } + assert(planHasStage(db, explainOutput, expectedStage), + "Expected pipeline " + tojsononeline(pipeline) + " to include a " + expectedStage + + " stage in the explain output: " + tojson(explainOutput)); + + if (expectedResult) { + const actualResult = coll.aggregate(pipeline, pipelineOptions).toArray(); + assert(preserveResultOrder ? orderedArrayEq(actualResult, expectedResult) + : arrayEq(actualResult, expectedResult)); + } + + return explainOutput; + } + + // Test that getMore works with the optimized query. + function testGetMore({command = null, expectedResult = null} = {}) { + const documents = + new DBCommandCursor(db, assert.commandWorked(db.runCommand(command)), 1 /* batchsize */) + .toArray(); + assert(arrayEq(documents, expectedResult)); + } + + let explainOutput; + + // Basic pipelines. + + // Test basic scenarios when a pipeline has a single $cursor stage or can be collapsed into a + // single cursor stage. + assertPipelineDoesNotUseAggregation({ + pipeline: [], + expectedStage: "COLLSCAN", + expectedResult: [{_id: 1, x: 10}, {_id: 2, x: 20}, {_id: 3, x: 30}] + }); + assertPipelineDoesNotUseAggregation({ + pipeline: [{$match: {x: 20}}], + expectedStage: "COLLSCAN", + expectedResult: [{_id: 2, x: 20}] + }); + + // Pipelines with a collation. + + // Test a simple pipeline with a case-insensitive collation. + assert.writeOK(coll.insert({_id: 4, x: 40, b: "abc"})); + assertPipelineDoesNotUseAggregation({ + pipeline: [{$match: {b: "ABC"}}], + pipelineOptions: {collation: {locale: "en_US", strength: 2}}, + expectedStage: "COLLSCAN", + expectedResult: [{_id: 4, x: 40, b: "abc"}] + }); + assert.commandWorked(coll.deleteOne({_id: 4})); + + // Pipelines with covered queries. + + // We can collapse a covered query into a single $cursor when $project and $sort are present and + // the latter is near the front of the pipeline. Skip this test in sharded modes as we cannot + // correctly handle explain output in plan analyzer helper functions. + assert.commandWorked(coll.createIndex({x: 1})); + assertPipelineDoesNotUseAggregation({ + pipeline: [{$sort: {x: 1}}, {$project: {x: 1, _id: 0}}], + expectedStage: "IXSCAN", + expectedResult: [{x: 10}, {x: 20}, {x: 30}], + preserveResultOrder: true + }); + assertPipelineDoesNotUseAggregation({ + pipeline: [{$match: {x: {$gte: 20}}}, {$sort: {x: 1}}, {$project: {x: 1, _id: 0}}], + expectedStage: "IXSCAN", + expectedResult: [{x: 20}, {x: 30}], + preserveResultOrder: true + }); + // TODO: SERVER-36723 We cannot collapse if there is a $limit stage though. + assertPipelineUsesAggregation({ + pipeline: + [{$match: {x: {$gte: 20}}}, {$sort: {x: 1}}, {$limit: 1}, {$project: {x: 1, _id: 0}}], + expectedStage: "IXSCAN", + expectedResult: [{x: 20}] + }); + assert.commandWorked(coll.dropIndexes()); + + // Pipelines which cannot be optimized away. + + // TODO SERVER-40254: Uncovered queries. + assert.writeOK(coll.insert({_id: 4, x: 40, a: {b: "ab1"}})); + assertPipelineUsesAggregation({ + pipeline: [{$project: {x: 1, _id: 0}}], + expectedStage: "COLLSCAN", + expectedResult: [{x: 10}, {x: 20}, {x: 30}, {x: 40}] + }); + assertPipelineUsesAggregation({ + pipeline: [{$match: {x: 20}}, {$project: {x: 1, _id: 0}}], + expectedStage: "COLLSCAN", + expectedResult: [{x: 20}] + }); + assertPipelineUsesAggregation({ + pipeline: [{$project: {x: 1, "a.b": 1, _id: 0}}], + expectedStage: "COLLSCAN", + expectedResult: [{x: 10}, {x: 20}, {x: 30}, {x: 40, a: {b: "ab1"}}] + }); + assertPipelineUsesAggregation({ + pipeline: [{$match: {x: 40}}, {$project: {"a.b": 1, _id: 0}}], + expectedStage: "COLLSCAN", + expectedResult: [{a: {b: "ab1"}}] + }); + assert.commandWorked(coll.deleteOne({_id: 4})); + + // TODO SERVER-36723: $limit stage is not supported yet. + assertPipelineUsesAggregation({ + pipeline: [{$match: {x: 20}}, {$limit: 1}], + expectedStage: "COLLSCAN", + expectedResult: [{_id: 2, x: 20}] + }); + // TODO SERVER-36723: $skip stage is not supported yet. + assertPipelineUsesAggregation({ + pipeline: [{$match: {x: {$gte: 20}}}, {$skip: 1}], + expectedStage: "COLLSCAN", + expectedResult: [{_id: 3, x: 30}] + }); + // We cannot collapse a $project stage if it has a complex pipeline expression. + assertPipelineUsesAggregation( + {pipeline: [{$project: {x: {$substr: ["$y", 0, 1]}, _id: 0}}], expectedStage: "COLLSCAN"}); + assertPipelineUsesAggregation({ + pipeline: [{$match: {x: 20}}, {$project: {x: {$substr: ["$y", 0, 1]}, _id: 0}}], + expectedStage: "COLLSCAN" + }); + // We cannot optimize away a pipeline if there are stages which have no equivalent in the + // find command. + assertPipelineUsesAggregation({ + pipeline: [{$match: {x: {$gte: 20}}}, {$count: "count"}], + expectedStage: "COLLSCAN", + expectedResult: [{count: 2}] + }); + assertPipelineUsesAggregation({ + pipeline: [{$match: {x: {$gte: 20}}}, {$group: {_id: "null", s: {$sum: "$x"}}}], + expectedStage: "COLLSCAN", + expectedResult: [{_id: "null", s: 50}] + }); + // TODO SERVER-40253: We cannot optimize away text search queries. + assert.commandWorked(coll.createIndex({y: "text"})); + assertPipelineUsesAggregation( + {pipeline: [{$match: {$text: {$search: "abc"}}}], expectedStage: "IXSCAN"}); + assert.commandWorked(coll.dropIndexes()); + // We cannot optimize away geo near queries. + assert.commandWorked(coll.createIndex({"y": "2d"})); + assertPipelineUsesAggregation({ + pipeline: [{$geoNear: {near: [0, 0], distanceField: "y", spherical: true}}], + expectedStage: "GEO_NEAR_2D" + }); + assert.commandWorked(coll.dropIndexes()); + + // getMore cases. + + // Test getMore on a collection with an optimized away pipeline. + testGetMore({ + command: {aggregate: coll.getName(), pipeline: [], cursor: {batchSize: 1}}, + expectedResult: [{_id: 1, x: 10}, {_id: 2, x: 20}, {_id: 3, x: 30}] + }); + testGetMore({ + command: { + aggregate: coll.getName(), + pipeline: [{$match: {x: {$gte: 20}}}], + cursor: {batchSize: 1} + }, + expectedResult: [{_id: 2, x: 20}, {_id: 3, x: 30}] + }); + testGetMore({ + command: { + aggregate: coll.getName(), + pipeline: [{$match: {x: {$gte: 20}}}, {$project: {x: 1, _id: 0}}], + cursor: {batchSize: 1} + }, + expectedResult: [{x: 20}, {x: 30}] + }); + // Test getMore on a view with an optimized away pipeline. Since views cannot be created when + // imlicit sharded collection mode is on, this test will be run only on a non-sharded + // collection. + let view; + if (!FixtureHelpers.isSharded(coll)) { + view = db.optimize_away_pipeline_view; + view.drop(); + assert.commandWorked(db.createView(view.getName(), coll.getName(), [])); + testGetMore({ + command: {find: view.getName(), filter: {}, batchSize: 1}, + expectedResult: [{_id: 1, x: 10}, {_id: 2, x: 20}, {_id: 3, x: 30}] + }); + } + // Test getMore puts a correct namespace into profile data for a colletion with optimized away + // pipeline. Cannot be run on mongos as profiling can be enabled only on mongod. Also profiling + // is supported on WiredTiger only. + if (!FixtureHelpers.isMongos(db) && isWiredTiger(db)) { + db.system.profile.drop(); + db.setProfilingLevel(2); + testGetMore({ + command: { + aggregate: coll.getName(), + pipeline: [{$match: {x: 10}}], + cursor: {batchSize: 1}, + comment: 'optimize_away_pipeline' + }, + expectedResult: [{_id: 1, x: 10}] + }); + db.setProfilingLevel(0); + let profile = db.system.profile.find({}, {op: 1, ns: 1, comment: 'optimize_away_pipeline'}) + .sort({ts: 1}) + .toArray(); + assert(arrayEq( + profile, + [{op: "command", ns: coll.getFullName()}, {op: "getmore", ns: coll.getFullName()}])); + // Test getMore puts a correct namespace into profile data for a view with an optimized away + // pipeline. + if (!FixtureHelpers.isSharded(coll)) { + db.system.profile.drop(); + db.setProfilingLevel(2); + testGetMore({ + command: { + find: view.getName(), + filter: {x: 10}, + batchSize: 1, + comment: 'optimize_away_pipeline' + }, + expectedResult: [{_id: 1, x: 10}] + }); + db.setProfilingLevel(0); + profile = db.system.profile.find({}, {op: 1, ns: 1, comment: 'optimize_away_pipeline'}) + .sort({ts: 1}) + .toArray(); + assert(arrayEq( + profile, + [{op: "query", ns: view.getFullName()}, {op: "getmore", ns: view.getFullName()}])); + } + } +}()); diff --git a/jstests/aggregation/sources/explain_out.js b/jstests/aggregation/sources/explain_out.js index c7b9dbb8767..bec7a83765e 100644 --- a/jstests/aggregation/sources/explain_out.js +++ b/jstests/aggregation/sources/explain_out.js @@ -12,9 +12,6 @@ load("jstests/libs/analyze_plan.js"); // For getAggPlanStage(). load("jstests/aggregation/extras/out_helpers.js"); // For withEachOutMode(). - // Mongos currently uses its own error code if any shard's explain fails. - const kErrorCode = FixtureHelpers.isMongos(db) ? 17403 : 51029; - let sourceColl = db.explain_out_source; let targetColl = db.explain_out_target; sourceColl.drop(); @@ -47,7 +44,7 @@ explain: {aggregate: sourceColl.getName(), pipeline: [outStage], cursor: {}}, verbosity: verbosity }), - kErrorCode); + 51029); assert.eq(targetColl.find().itcount(), 0); } diff --git a/jstests/aggregation/sources/project/remove_redundant_projects.js b/jstests/aggregation/sources/project/remove_redundant_projects.js index 9b86d3fa6b3..e3c7af08573 100644 --- a/jstests/aggregation/sources/project/remove_redundant_projects.js +++ b/jstests/aggregation/sources/project/remove_redundant_projects.js @@ -20,8 +20,11 @@ * ('expectProjectToCoalesce') and the corresponding project stage ('removedProjectStage') does * not exist in the explain output. */ - function assertResultsMatch( - pipeline, expectProjectToCoalesce, removedProjectStage = null, index = indexSpec) { + function assertResultsMatch({pipeline = [], + expectProjectToCoalesce = false, + removedProjectStage = null, + index = indexSpec, + pipelineOptimizedAway = false} = {}) { // Add a match stage to ensure index scans are considered for planning (workaround for // SERVER-20066). pipeline = [{$match: {a: {$gte: 0}}}].concat(pipeline); @@ -33,23 +36,29 @@ // Projection does not get pushed down when sharding filter is used. if (!explain.hasOwnProperty("shards")) { + let result; + + if (pipelineOptimizedAway) { + assert(isQueryPlan(explain)); + result = explain.queryPlanner.winningPlan; + } else { + assert(isAggregationPlan(explain)); + result = explain.stages[0].$cursor.queryPlanner.winningPlan; + } + // Check that $project uses the query system. - assert.eq( - expectProjectToCoalesce, - planHasStage( - db, explain.stages[0].$cursor.queryPlanner.winningPlan, "PROJECTION_DEFAULT") || - planHasStage(db, - explain.stages[0].$cursor.queryPlanner.winningPlan, - "PROJECTION_COVERED") || - planHasStage(db, - explain.stages[0].$cursor.queryPlanner.winningPlan, - "PROJECTION_SIMPLE")); - - // Check that $project was removed from pipeline and pushed to the query system. - explain.stages.forEach(function(stage) { - if (stage.hasOwnProperty("$project")) - assert.neq(removedProjectStage, stage["$project"]); - }); + assert.eq(expectProjectToCoalesce, + planHasStage(db, result, "PROJECTION_DEFAULT") || + planHasStage(db, result, "PROJECTION_COVERED") || + planHasStage(db, result, "PROJECTION_SIMPLE")); + + if (!pipelineOptimizedAway) { + // Check that $project was removed from pipeline and pushed to the query system. + explain.stages.forEach(function(stage) { + if (stage.hasOwnProperty("$project")) + assert.neq(removedProjectStage, stage["$project"]); + }); + } } // Again without an index. @@ -61,54 +70,82 @@ // Test that covered projections correctly use the query system for projection and the $project // stage is removed from the pipeline. - assertResultsMatch([{$project: {_id: 0, a: 1}}], true, {_id: 0, a: 1}); - assertResultsMatch( - [{$project: {_id: 0, a: 1}}, {$group: {_id: null, a: {$sum: "$a"}}}], true, {_id: 0, a: 1}); - assertResultsMatch([{$sort: {a: -1}}, {$project: {_id: 0, a: 1}}], true, {_id: 0, a: 1}); - assertResultsMatch( - [ - {$sort: {a: 1, 'c.d': 1}}, - {$project: {_id: 0, a: 1}}, - {$group: {_id: "$a", arr: {$push: "$a"}}} + assertResultsMatch({ + pipeline: [{$project: {_id: 0, a: 1}}], + expectProjectToCoalesce: true, + removedProjectStage: {_id: 0, a: 1}, + pipelineOptimizedAway: true + }); + assertResultsMatch({ + pipeline: [{$project: {_id: 0, a: 1}}, {$group: {_id: null, a: {$sum: "$a"}}}], + expectProjectToCoalesce: true, + removedProjectStage: {_id: 0, a: 1} + }); + assertResultsMatch({ + pipeline: [{$sort: {a: -1}}, {$project: {_id: 0, a: 1}}], + expectProjectToCoalesce: true, + removedProjectStage: {_id: 0, a: 1}, + pipelineOptimizedAway: true + }); + assertResultsMatch({ + pipeline: [ + {$sort: {a: 1, 'c.d': 1}}, + {$project: {_id: 0, a: 1}}, + {$group: {_id: "$a", arr: {$push: "$a"}}} ], - true, - {_id: 0, a: 1}); - assertResultsMatch([{$project: {_id: 0, c: {d: 1}}}], true, {_id: 0, c: {d: 1}}); + expectProjectToCoalesce: true, + removedProjectStage: {_id: 0, a: 1} + }); + assertResultsMatch({ + pipeline: [{$project: {_id: 0, c: {d: 1}}}], + expectProjectToCoalesce: true, + removedProjectStage: {_id: 0, c: {d: 1}}, + pipelineOptimizedAway: true + }); // Test that projections with renamed fields are not removed from the pipeline, however an // inclusion projection is still pushed to the query system. - assertResultsMatch([{$project: {_id: 0, f: "$a"}}], true); - assertResultsMatch([{$project: {_id: 0, a: 1, f: "$a"}}], true); + assertResultsMatch({pipeline: [{$project: {_id: 0, f: "$a"}}], expectProjectToCoalesce: true}); + assertResultsMatch( + {pipeline: [{$project: {_id: 0, a: 1, f: "$a"}}], expectProjectToCoalesce: true}); // Test that uncovered projections include the $project stage in the pipeline. - assertResultsMatch([{$project: {_id: 1, a: 1}}], false); - assertResultsMatch([{$project: {_id: 0, a: 1, b: 1}}], false); - assertResultsMatch([{$sort: {a: 1}}, {$project: {_id: 1, b: 1}}], false); assertResultsMatch( - [{$sort: {a: 1}}, {$group: {_id: "$_id", arr: {$push: "$a"}}}, {$project: {arr: 1}}], - false); + {pipeline: [{$sort: {a: 1}}, {$project: {_id: 1, b: 1}}], expectProjectToCoalesce: false}); + assertResultsMatch({ + pipeline: + [{$sort: {a: 1}}, {$group: {_id: "$_id", arr: {$push: "$a"}}}, {$project: {arr: 1}}], + expectProjectToCoalesce: false + }); // Test that projections with computed fields are kept in the pipeline. - assertResultsMatch([{$project: {computedField: {$sum: "$a"}}}], false); - assertResultsMatch([{$project: {a: ["$a", "$b"]}}], false); assertResultsMatch( - [{$project: {e: {$filter: {input: "$e", as: "item", cond: {"$eq": ["$$item", "elem0"]}}}}}], - false); + {pipeline: [{$project: {computedField: {$sum: "$a"}}}], expectProjectToCoalesce: false}); + assertResultsMatch({pipeline: [{$project: {a: ["$a", "$b"]}}], expectProjectToCoalesce: false}); + assertResultsMatch({ + pipeline: [{ + $project: + {e: {$filter: {input: "$e", as: "item", cond: {"$eq": ["$$item", "elem0"]}}}} + }], + expectProjectToCoalesce: false + }); // Test that only the first projection is removed from the pipeline. - assertResultsMatch( - [ - {$project: {_id: 0, a: 1}}, - {$group: {_id: "$a", arr: {$push: "$a"}, a: {$sum: "$a"}}}, - {$project: {_id: 0}} + assertResultsMatch({ + pipeline: [ + {$project: {_id: 0, a: 1}}, + {$group: {_id: "$a", arr: {$push: "$a"}, a: {$sum: "$a"}}}, + {$project: {_id: 0}} ], - true, - {_id: 0, a: 1}); + expectProjectToCoalesce: true, + removedProjectStage: {_id: 0, a: 1} + }); // Test that projections on _id with nested fields are not removed from pipeline. Due to // SERVER-7502, the dependency analysis does not generate a covered projection for nested // fields in _id and thus we cannot remove the stage. indexSpec = {'_id.a': 1, a: 1}; - assertResultsMatch([{$project: {'_id.a': 1}}], false, null, indexSpec); + assertResultsMatch( + {pipeline: [{$project: {'_id.a': 1}}], expectProjectToCoalesce: false, index: indexSpec}); }()); diff --git a/jstests/aggregation/use_query_project_and_sort.js b/jstests/aggregation/use_query_project_and_sort.js index b5afc09a7cc..4d3c4a7a45a 100644 --- a/jstests/aggregation/use_query_project_and_sort.js +++ b/jstests/aggregation/use_query_project_and_sort.js @@ -21,19 +21,16 @@ function assertQueryCoversProjectionAndSort(pipeline) { const explainOutput = coll.explain().aggregate(pipeline); - assert(!aggPlanHasStage(explainOutput, "FETCH"), + assert(isQueryPlan(explainOutput)); + assert(!planHasStage(db, explainOutput, "FETCH"), "Expected pipeline " + tojsononeline(pipeline) + " *not* to include a FETCH stage in the explain output: " + tojson(explainOutput)); - assert(!aggPlanHasStage(explainOutput, "SORT"), + assert(!planHasStage(db, explainOutput, "SORT"), "Expected pipeline " + tojsononeline(pipeline) + " *not* to include a SORT stage in the explain output: " + tojson(explainOutput)); - assert(!aggPlanHasStage(explainOutput, "$sort"), - "Expected pipeline " + tojsononeline(pipeline) + - " *not* to include a SORT stage in the explain output: " + - tojson(explainOutput)); - assert(aggPlanHasStage(explainOutput, "IXSCAN"), + assert(planHasStage(db, explainOutput, "IXSCAN"), "Expected pipeline " + tojsononeline(pipeline) + " to include an index scan in the explain output: " + tojson(explainOutput)); assert(!hasRejectedPlans(explainOutput), diff --git a/jstests/aggregation/use_query_projection.js b/jstests/aggregation/use_query_projection.js index cf239a7e5d7..e86a357b782 100644 --- a/jstests/aggregation/use_query_projection.js +++ b/jstests/aggregation/use_query_projection.js @@ -19,15 +19,28 @@ } assert.writeOK(bulk.execute()); - function assertQueryCoversProjection(pipeline) { + function assertQueryCoversProjection({pipeline = [], pipelineOptimizedAway = true} = {}) { const explainOutput = coll.explain().aggregate(pipeline); - assert(!aggPlanHasStage(explainOutput, "FETCH"), - "Expected pipeline " + tojsononeline(pipeline) + - " *not* to include a FETCH stage in the explain output: " + - tojson(explainOutput)); - assert(aggPlanHasStage(explainOutput, "IXSCAN"), - "Expected pipeline " + tojsononeline(pipeline) + - " to include an index scan in the explain output: " + tojson(explainOutput)); + + if (pipelineOptimizedAway) { + assert(isQueryPlan(explainOutput)); + assert(!planHasStage(db, explainOutput, "FETCH"), + "Expected pipeline " + tojsononeline(pipeline) + + " *not* to include a FETCH stage in the explain output: " + + tojson(explainOutput)); + assert(planHasStage(db, explainOutput, "IXSCAN"), + "Expected pipeline " + tojsononeline(pipeline) + + " to include an index scan in the explain output: " + tojson(explainOutput)); + } else { + assert(isAggregationPlan(explainOutput)); + assert(!aggPlanHasStage(explainOutput, "FETCH"), + "Expected pipeline " + tojsononeline(pipeline) + + " *not* to include a FETCH stage in the explain output: " + + tojson(explainOutput)); + assert(aggPlanHasStage(explainOutput, "IXSCAN"), + "Expected pipeline " + tojsononeline(pipeline) + + " to include an index scan in the explain output: " + tojson(explainOutput)); + } assert(!hasRejectedPlans(explainOutput), "Expected pipeline " + tojsononeline(pipeline) + " not to have any rejected plans in the explain output: " + @@ -35,16 +48,31 @@ return explainOutput; } - function assertQueryDoesNotCoverProjection(pipeline) { + function assertQueryDoesNotCoverProjection({pipeline = [], pipelineOptimizedAway = true} = {}) { const explainOutput = coll.explain().aggregate(pipeline); - assert(aggPlanHasStage(explainOutput, "FETCH") || aggPlanHasStage("COLLSCAN"), - "Expected pipeline " + tojsononeline(pipeline) + - " to include a FETCH or COLLSCAN stage in the explain output: " + - tojson(explainOutput)); - assert(!hasRejectedPlans(explainOutput), - "Expected pipeline " + tojsononeline(pipeline) + - " not to have any rejected plans in the explain output: " + - tojson(explainOutput)); + + if (pipelineOptimizedAway) { + assert(isQueryPlan(explainOutput)); + assert(planHasStage(db, explainOutput, "FETCH") || aggPlanHasStage("COLLSCAN"), + "Expected pipeline " + tojsononeline(pipeline) + + " to include a FETCH or COLLSCAN stage in the explain output: " + + tojson(explainOutput)); + assert(!hasRejectedPlans(explainOutput), + "Expected pipeline " + tojsononeline(pipeline) + + " not to have any rejected plans in the explain output: " + + tojson(explainOutput)); + } else { + assert(isAggregationPlan(explainOutput)); + assert(aggPlanHasStage(explainOutput, "FETCH") || aggPlanHasStage("COLLSCAN"), + "Expected pipeline " + tojsononeline(pipeline) + + " to include a FETCH or COLLSCAN stage in the explain output: " + + tojson(explainOutput)); + assert(!hasRejectedPlans(explainOutput), + "Expected pipeline " + tojsononeline(pipeline) + + " not to have any rejected plans in the explain output: " + + tojson(explainOutput)); + } + return explainOutput; } @@ -52,29 +80,48 @@ // Test that a pipeline requiring a subset of the fields in a compound index can use that index // to cover the query. - assertQueryCoversProjection([{$match: {x: "string"}}, {$project: {_id: 1, x: 1}}]); - assertQueryCoversProjection([{$match: {x: "string"}}, {$project: {_id: 0, x: 1}}]); - assertQueryCoversProjection([{$match: {x: "string"}}, {$project: {_id: 0, x: 1, a: 1}}]); - assertQueryCoversProjection([{$match: {x: "string"}}, {$project: {_id: 1, x: 1, a: 1}}]); + assertQueryCoversProjection({pipeline: [{$match: {x: "string"}}, {$project: {_id: 1, x: 1}}]}); + assertQueryCoversProjection({pipeline: [{$match: {x: "string"}}, {$project: {_id: 0, x: 1}}]}); + assertQueryCoversProjection( + {pipeline: [{$match: {x: "string"}}, {$project: {_id: 0, x: 1, a: 1}}]}); + assertQueryCoversProjection( + {pipeline: [{$match: {x: "string"}}, {$project: {_id: 1, x: 1, a: 1}}]}); assertQueryCoversProjection( - [{$match: {_id: 0, x: "string"}}, {$project: {_id: 1, x: 1, a: 1}}]); + {pipeline: [{$match: {_id: 0, x: "string"}}, {$project: {_id: 1, x: 1, a: 1}}]}); // Test that a pipeline requiring a field that is not in the index cannot use a covered plan. - assertQueryDoesNotCoverProjection([{$match: {x: "string"}}, {$project: {notThere: 1}}]); + assertQueryDoesNotCoverProjection({ + pipeline: [{$match: {x: "string"}}, {$project: {notThere: 1}}], + pipelineOptimizedAway: false + }); // Test that a covered plan is the only plan considered, even if another plan would be equally // selective. Add an equally selective index, then rely on assertQueryCoversProjection() to // assert that there is only one considered plan, and it is a covered plan. assert.commandWorked(coll.createIndex({x: 1})); - assertQueryCoversProjection([ - {$match: {_id: 0, x: "string"}}, - {$sort: {x: 1, a: 1}}, // Note: not indexable, but doesn't add any additional dependencies. - {$project: {_id: 1, x: 1, a: 1}}, - ]); + assertQueryCoversProjection({ + pipeline: [ + {$match: {_id: 0, x: "string"}}, + { + $sort: { + x: 1, + a: 1 + } + }, // Note: not indexable, but doesn't add any additional dependencies. + {$project: {_id: 1, x: 1, a: 1}}, + ], + pipelineOptimizedAway: false + }); // Test that a multikey index will prevent a covered plan. assert.commandWorked(coll.dropIndex({x: 1})); // Make sure there is only one plan considered. assert.writeOK(coll.insert({x: ["an", "array!"]})); - assertQueryDoesNotCoverProjection([{$match: {x: "string"}}, {$project: {_id: 1, x: 1}}]); - assertQueryDoesNotCoverProjection([{$match: {x: "string"}}, {$project: {_id: 1, x: 1, a: 1}}]); + assertQueryDoesNotCoverProjection({ + pipeline: [{$match: {x: "string"}}, {$project: {_id: 1, x: 1}}], + pipelineOptimizedAway: false + }); + assertQueryDoesNotCoverProjection({ + pipeline: [{$match: {x: "string"}}, {$project: {_id: 1, x: 1, a: 1}}], + pipelineOptimizedAway: false + }); }()); diff --git a/jstests/aggregation/use_query_sort.js b/jstests/aggregation/use_query_sort.js index 7d0e1b74f36..26542252ff4 100644 --- a/jstests/aggregation/use_query_sort.js +++ b/jstests/aggregation/use_query_sort.js @@ -20,15 +20,12 @@ function assertHasNonBlockingQuerySort(pipeline) { const explainOutput = coll.explain().aggregate(pipeline); - assert(!aggPlanHasStage(explainOutput, "$sort"), - "Expected pipeline " + tojsononeline(pipeline) + - " *not* to include a $sort stage in the explain output: " + - tojson(explainOutput)); - assert(!aggPlanHasStage(explainOutput, "SORT"), + assert(isQueryPlan(explainOutput)); + assert(!planHasStage(db, explainOutput, "SORT"), "Expected pipeline " + tojsononeline(pipeline) + " *not* to include a SORT stage in the explain output: " + tojson(explainOutput)); - assert(aggPlanHasStage(explainOutput, "IXSCAN"), + assert(planHasStage(db, explainOutput, "IXSCAN"), "Expected pipeline " + tojsononeline(pipeline) + " to include an index scan in the explain output: " + tojson(explainOutput)); assert(!hasRejectedPlans(explainOutput), @@ -40,6 +37,7 @@ function assertDoesNotHaveQuerySort(pipeline) { const explainOutput = coll.explain().aggregate(pipeline); + assert(isAggregationPlan(explainOutput)); assert(aggPlanHasStage(explainOutput, "$sort"), "Expected pipeline " + tojsononeline(pipeline) + " to include a $sort stage in the explain output: " + tojson(explainOutput)); diff --git a/jstests/core/agg_hint.js b/jstests/core/agg_hint.js index 6a376a6c385..2d088daaf7b 100644 --- a/jstests/core/agg_hint.js +++ b/jstests/core/agg_hint.js @@ -15,10 +15,11 @@ assert.commandWorked(testDB.dropDatabase()); const coll = testDB.getCollection("test"); const view = testDB.getCollection("view"); - const NO_HINT = null; - function confirmWinningPlanUsesExpectedIndex(explainResult, expectedKeyPattern, stageName) { - const planStage = getAggPlanStage(explainResult, stageName); + function confirmWinningPlanUsesExpectedIndex( + explainResult, expectedKeyPattern, stageName, pipelineOptimizedAway) { + const planStage = pipelineOptimizedAway ? getPlanStage(explainResult, stageName) + : getAggPlanStage(explainResult, stageName); assert.neq(null, planStage); assert.eq(planStage.keyPattern, expectedKeyPattern, tojson(planStage)); @@ -26,30 +27,44 @@ // Runs explain on 'command', with the hint specified by 'hintKeyPattern' when not null. // Confirms that the winning query plan uses the index specified by 'expectedKeyPattern'. - function confirmCommandUsesIndex( - command, hintKeyPattern, expectedKeyPattern, stageName = "IXSCAN") { + // If 'pipelineOptimizedAway' is set to true, then we expect the pipeline to be entirely + // optimized away from the plan and replaced with a query tier. + function confirmCommandUsesIndex({command = null, + hintKeyPattern = null, + expectedKeyPattern = null, + stageName = "IXSCAN", + pipelineOptimizedAway = false} = {}) { if (hintKeyPattern) { command["hint"] = hintKeyPattern; } const res = assert.commandWorked(testDB.runCommand({explain: command, verbosity: "queryPlanner"})); - confirmWinningPlanUsesExpectedIndex(res, expectedKeyPattern, stageName); + confirmWinningPlanUsesExpectedIndex( + res, expectedKeyPattern, stageName, pipelineOptimizedAway); } // Runs explain on an aggregation with a pipeline specified by 'aggPipeline' and a hint // specified by 'hintKeyPattern' if not null. Confirms that the winning query plan uses the - // index specified by 'expectedKeyPattern'. + // index specified by 'expectedKeyPattern'. If 'pipelineOptimizedAway' is set to true, then + // we expect the pipeline to be entirely optimized away from the plan and replaced with a + // query tier. // // This method exists because the explain command does not support the aggregation command. - function confirmAggUsesIndex( - collName, aggPipeline, hintKeyPattern, expectedKeyPattern, stageName = "IXSCAN") { + function confirmAggUsesIndex({collName = null, + aggPipeline = [], + hintKeyPattern = null, + expectedKeyPattern = null, + stageName = "IXSCAN", + pipelineOptimizedAway = false} = {}) { let options = {}; + if (hintKeyPattern) { options = {hint: hintKeyPattern}; } const res = assert.commandWorked( testDB.getCollection(collName).explain().aggregate(aggPipeline, options)); - confirmWinningPlanUsesExpectedIndex(res, expectedKeyPattern, stageName); + confirmWinningPlanUsesExpectedIndex( + res, expectedKeyPattern, stageName, pipelineOptimizedAway); } // Specify hint as a string, representing index name. @@ -58,7 +73,13 @@ assert.writeOK(coll.insert({x: i})); } - confirmAggUsesIndex("test", [{$match: {x: 3}}], "x_1", {x: 1}); + confirmAggUsesIndex({ + collName: "test", + aggPipeline: [{$match: {x: 3}}], + hintKeyPattern: "x_1", + expectedKeyPattern: {x: 1}, + pipelineOptimizedAway: true + }); // // For each of the following tests we confirm: @@ -74,9 +95,26 @@ assert.writeOK(coll.insert({x: i})); } - confirmAggUsesIndex("test", [{$match: {x: 3}}], NO_HINT, {x: 1}); - confirmAggUsesIndex("test", [{$match: {x: 3}}], {x: 1}, {x: 1}); - confirmAggUsesIndex("test", [{$match: {x: 3}}], {_id: 1}, {_id: 1}); + confirmAggUsesIndex({ + collName: "test", + aggPipeline: [{$match: {x: 3}}], + expectedKeyPattern: {x: 1}, + pipelineOptimizedAway: true + }); + confirmAggUsesIndex({ + collName: "test", + aggPipeline: [{$match: {x: 3}}], + hintKeyPattern: {x: 1}, + expectedKeyPattern: {x: 1}, + pipelineOptimizedAway: true + }); + confirmAggUsesIndex({ + collName: "test", + aggPipeline: [{$match: {x: 3}}], + hintKeyPattern: {_id: 1}, + expectedKeyPattern: {_id: 1}, + pipelineOptimizedAway: true + }); // With no hint specified, aggregation will always prefer an index that provides sort order over // one that requires a blocking sort. A hinted aggregation should allow for choice of an index @@ -88,9 +126,25 @@ assert.writeOK(coll.insert({x: i, y: i})); } - confirmAggUsesIndex("test", [{$match: {x: {$gte: 0}}}, {$sort: {y: 1}}], NO_HINT, {y: 1}); - confirmAggUsesIndex("test", [{$match: {x: {$gte: 0}}}, {$sort: {y: 1}}], {y: 1}, {y: 1}); - confirmAggUsesIndex("test", [{$match: {x: {$gte: 0}}}, {$sort: {y: 1}}], {x: 1}, {x: 1}); + confirmAggUsesIndex({ + collName: "test", + aggPipeline: [{$match: {x: {$gte: 0}}}, {$sort: {y: 1}}], + expectedKeyPattern: {y: 1}, + pipelineOptimizedAway: true + }); + confirmAggUsesIndex({ + collName: "test", + aggPipeline: [{$match: {x: {$gte: 0}}}, {$sort: {y: 1}}], + hintKeyPattern: {y: 1}, + expectedKeyPattern: {y: 1}, + pipelineOptimizedAway: true + }); + confirmAggUsesIndex({ + collName: "test", + aggPipeline: [{$match: {x: {$gte: 0}}}, {$sort: {y: 1}}], + hintKeyPattern: {x: 1}, + expectedKeyPattern: {x: 1} + }); // With no hint specified, aggregation will always prefer an index that provides a covered // projection over one that does not. A hinted aggregation should allow for choice of an index @@ -102,16 +156,25 @@ assert.writeOK(coll.insert({x: i, y: i})); } - confirmAggUsesIndex("test", - [{$match: {x: {$gte: 0}}}, {$project: {x: 1, y: 1, _id: 0}}], - NO_HINT, - {x: 1, y: 1}); - confirmAggUsesIndex("test", - [{$match: {x: {$gte: 0}}}, {$project: {x: 1, y: 1, _id: 0}}], - {x: 1, y: 1}, - {x: 1, y: 1}); - confirmAggUsesIndex( - "test", [{$match: {x: {$gte: 0}}}, {$project: {x: 1, y: 1, _id: 0}}], {x: 1}, {x: 1}); + confirmAggUsesIndex({ + collName: "test", + aggPipeline: [{$match: {x: {$gte: 0}}}, {$project: {x: 1, y: 1, _id: 0}}], + expectedKeyPattern: {x: 1, y: 1}, + pipelineOptimizedAway: true + }); + confirmAggUsesIndex({ + collName: "test", + aggPipeline: [{$match: {x: {$gte: 0}}}, {$project: {x: 1, y: 1, _id: 0}}], + hintKeyPattern: {x: 1, y: 1}, + expectedKeyPattern: {x: 1, y: 1}, + pipelineOptimizedAway: true + }); + confirmAggUsesIndex({ + collName: "test", + aggPipeline: [{$match: {x: {$gte: 0}}}, {$project: {x: 1, y: 1, _id: 0}}], + hintKeyPattern: {x: 1}, + expectedKeyPattern: {x: 1} + }); // Confirm that a hinted agg can be executed against a view. coll.drop(); @@ -120,11 +183,28 @@ for (let i = 0; i < 5; ++i) { assert.writeOK(coll.insert({x: i})); } - assert.commandWorked(testDB.createView("view", "test", [])); + assert.commandWorked(testDB.createView("view", "test", [{$match: {x: {$gte: 0}}}])); - confirmAggUsesIndex("view", [{$match: {x: 3}}], NO_HINT, {x: 1}); - confirmAggUsesIndex("view", [{$match: {x: 3}}], {x: 1}, {x: 1}); - confirmAggUsesIndex("view", [{$match: {x: 3}}], {_id: 1}, {_id: 1}); + confirmAggUsesIndex({ + collName: "view", + aggPipeline: [{$match: {x: 3}}], + expectedKeyPattern: {x: 1}, + pipelineOptimizedAway: true + }); + confirmAggUsesIndex({ + collName: "view", + aggPipeline: [{$match: {x: 3}}], + hintKeyPattern: {x: 1}, + expectedKeyPattern: {x: 1}, + pipelineOptimizedAway: true + }); + confirmAggUsesIndex({ + collName: "view", + aggPipeline: [{$match: {x: 3}}], + hintKeyPattern: {_id: 1}, + expectedKeyPattern: {_id: 1}, + pipelineOptimizedAway: true + }); // Confirm that a hinted find can be executed against a view. coll.drop(); @@ -135,9 +215,23 @@ } assert.commandWorked(testDB.createView("view", "test", [])); - confirmCommandUsesIndex({find: "view", filter: {x: 3}}, NO_HINT, {x: 1}); - confirmCommandUsesIndex({find: "view", filter: {x: 3}}, {x: 1}, {x: 1}); - confirmCommandUsesIndex({find: "view", filter: {x: 3}}, {_id: 1}, {_id: 1}); + confirmCommandUsesIndex({ + command: {find: "view", filter: {x: 3}}, + expectedKeyPattern: {x: 1}, + pipelineOptimizedAway: true + }); + confirmCommandUsesIndex({ + command: {find: "view", filter: {x: 3}}, + hintKeyPattern: {x: 1}, + expectedKeyPattern: {x: 1}, + pipelineOptimizedAway: true + }); + confirmCommandUsesIndex({ + command: {find: "view", filter: {x: 3}}, + hintKeyPattern: {_id: 1}, + expectedKeyPattern: {_id: 1}, + pipelineOptimizedAway: true + }); // Confirm that a hinted count can be executed against a view. coll.drop(); @@ -148,7 +242,20 @@ } assert.commandWorked(testDB.createView("view", "test", [])); - confirmCommandUsesIndex({count: "view", query: {x: 3}}, NO_HINT, {x: 1}, "COUNT_SCAN"); - confirmCommandUsesIndex({count: "view", query: {x: 3}}, {x: 1}, {x: 1}, "COUNT_SCAN"); - confirmCommandUsesIndex({count: "view", query: {x: 3}}, {_id: 1}, {_id: 1}); -})();
\ No newline at end of file + confirmCommandUsesIndex({ + command: {count: "view", query: {x: 3}}, + expectedKeyPattern: {x: 1}, + stageName: "COUNT_SCAN" + }); + confirmCommandUsesIndex({ + command: {count: "view", query: {x: 3}}, + hintKeyPattern: {x: 1}, + expectedKeyPattern: {x: 1}, + stageName: "COUNT_SCAN" + }); + confirmCommandUsesIndex({ + command: {count: "view", query: {x: 3}}, + hintKeyPattern: {_id: 1}, + expectedKeyPattern: {_id: 1} + }); +})(); diff --git a/jstests/core/collation.js b/jstests/core/collation.js index 8f56c123ea3..ba5cab6f282 100644 --- a/jstests/core/collation.js +++ b/jstests/core/collation.js @@ -332,7 +332,7 @@ coll.drop(); assert.commandWorked(db.createCollection(coll.getName(), {collation: {locale: "en_US"}})); assert.commandWorked(coll.ensureIndex({a: 1}, {collation: {locale: "en_US"}})); - var explain = coll.explain("queryPlanner").aggregate([{$match: {a: "foo"}}]).stages[0].$cursor; + var explain = coll.explain("queryPlanner").aggregate([{$match: {a: "foo"}}]); assert(isIxscan(db, explain.queryPlanner.winningPlan)); // Aggregation should not use index when no collation specified and collection default @@ -340,7 +340,7 @@ coll.drop(); assert.commandWorked(db.createCollection(coll.getName(), {collation: {locale: "en_US"}})); assert.commandWorked(coll.ensureIndex({a: 1}, {collation: {locale: "simple"}})); - var explain = coll.explain("queryPlanner").aggregate([{$match: {a: "foo"}}]).stages[0].$cursor; + var explain = coll.explain("queryPlanner").aggregate([{$match: {a: "foo"}}]); assert(isCollscan(db, explain.queryPlanner.winningPlan)); // Explain of aggregation with collation should succeed. diff --git a/jstests/core/explain_shell_helpers.js b/jstests/core/explain_shell_helpers.js index 0947cc30c6b..1aaac48ae90 100644 --- a/jstests/core/explain_shell_helpers.js +++ b/jstests/core/explain_shell_helpers.js @@ -199,26 +199,26 @@ assert.commandWorked(results[0]); // .aggregate() // -explain = t.explain().aggregate([{$match: {a: 3}}]); +explain = t.explain().aggregate([{$match: {a: 3}}, {$group: {_id: null}}]); assert.commandWorked(explain); -assert.eq(1, explain.stages.length); +assert.eq(2, explain.stages.length); assert("queryPlanner" in explain.stages[0].$cursor); // Legacy varargs format. -explain = t.explain().aggregate({$match: {a: 3}}); +explain = t.explain().aggregate({$group: {_id: null}}); assert.commandWorked(explain); -assert.eq(1, explain.stages.length); +assert.eq(2, explain.stages.length); assert("queryPlanner" in explain.stages[0].$cursor); -explain = t.explain().aggregate({$match: {a: 3}}, {$project: {a: 1}}); +explain = t.explain().aggregate({$project: {a: 3}}, {$group: {_id: null}}); assert.commandWorked(explain); -assert.eq(2, explain.stages.length); +assert.eq(3, explain.stages.length); assert("queryPlanner" in explain.stages[0].$cursor); // Options already provided. -explain = t.explain().aggregate([{$match: {a: 3}}], {allowDiskUse: true}); +explain = t.explain().aggregate([{$match: {a: 3}}, {$group: {_id: null}}], {allowDiskUse: true}); assert.commandWorked(explain); -assert.eq(1, explain.stages.length); +assert.eq(2, explain.stages.length); assert("queryPlanner" in explain.stages[0].$cursor); // diff --git a/jstests/core/expr_index_use.js b/jstests/core/expr_index_use.js index d02b16f45cc..79fe6d87b86 100644 --- a/jstests/core/expr_index_use.js +++ b/jstests/core/expr_index_use.js @@ -110,16 +110,14 @@ explain = assert.commandWorked(coll.explain("executionStats").aggregate(pipeline, aggOptions)); - verifyExplainOutput(explain, getAggPlanStage); + verifyExplainOutput(explain, getPlanStage); cursor = coll.explain("executionStats").find({$expr: expr}); if (collation) { cursor = cursor.collation(collation); } explain = assert.commandWorked(cursor.finish()); - verifyExplainOutput( - explain, - (explain, stage) => getPlanStage(explain.executionStats.executionStages, stage)); + verifyExplainOutput(explain, getPlanStage); } // Comparison of field and constant. diff --git a/jstests/core/index_partial_read_ops.js b/jstests/core/index_partial_read_ops.js index f06ee85c621..27fdb430fba 100644 --- a/jstests/core/index_partial_read_ops.js +++ b/jstests/core/index_partial_read_ops.js @@ -57,11 +57,11 @@ load("jstests/libs/analyze_plan.js"); // // Aggregate operation that should use index. - explain = coll.aggregate([{$match: {x: {$gt: 1}, a: 1}}], {explain: true}).stages[0].$cursor; + explain = coll.aggregate([{$match: {x: {$gt: 1}, a: 1}}], {explain: true}); assert(isIxscan(db, explain.queryPlanner.winningPlan)); // Aggregate operation that should not use index. - explain = coll.aggregate([{$match: {x: {$gt: 1}, a: 2}}], {explain: true}).stages[0].$cursor; + explain = coll.aggregate([{$match: {x: {$gt: 1}, a: 2}}], {explain: true}); assert(isCollscan(db, explain.queryPlanner.winningPlan)); // diff --git a/jstests/core/sort_array.js b/jstests/core/sort_array.js index 9cf013569e8..48ccdea93c4 100644 --- a/jstests/core/sort_array.js +++ b/jstests/core/sort_array.js @@ -24,7 +24,7 @@ assert.eq(cursor.toArray(), expected); } let explain = coll.find(filter, project).sort(sort).explain(); - assert(planHasStage(db, explain.queryPlanner.winningPlan, "SORT")); + assert(planHasStage(db, explain, "SORT")); let pipeline = [ {$_internalInhibitOptimization: {}}, @@ -139,17 +139,14 @@ assert.commandWorked(coll.createIndex({a: 1, "b.c": 1})); assert.writeOK(coll.insert({a: [1, 2, 3], b: {c: 9}})); explain = coll.find({a: 2}).sort({"b.c": -1}).explain(); - assert(planHasStage(db, explain.queryPlanner.winningPlan, "IXSCAN")); - assert(!planHasStage(db, explain.queryPlanner.winningPlan, "SORT")); + assert(planHasStage(db, explain, "IXSCAN")); + assert(!planHasStage(db, explain, "SORT")); - const pipeline = [ - {$match: {a: 2}}, - {$sort: {"b.c": -1}}, - ]; + const pipeline = [{$match: {a: 2}}, {$sort: {"b.c": -1}}]; explain = coll.explain().aggregate(pipeline); - assert(aggPlanHasStage(explain, "IXSCAN")); - assert(!aggPlanHasStage(explain, "SORT")); - assert(!aggPlanHasStage(explain, "$sort")); + assert(isQueryPlan(explain)); + assert(planHasStage(db, explain, "IXSCAN")); + assert(!planHasStage(db, explain, "SORT")); // Test that we can correctly sort by an array field in agg when there are additional fields not // involved in the sort pattern. @@ -228,5 +225,4 @@ hint: {"a.x": 1}, expected: [{_id: 1}, {_id: 0}, {_id: 2}] }); - }()); diff --git a/jstests/core/views/views_collation.js b/jstests/core/views/views_collation.js index 428927751c8..32b103ae2fb 100644 --- a/jstests/core/views/views_collation.js +++ b/jstests/core/views/views_collation.js @@ -494,12 +494,13 @@ let explain, cursorStage; // Test that aggregate against a view with a default collation correctly uses the collation. + // We expect the pipeline to be optimized away, so there should be no pipeline stages in + // the explain. assert.eq(1, viewsDB.case_sensitive_coll.aggregate([{$match: {f: "case"}}]).itcount()); assert.eq(3, viewsDB.case_insensitive_view.aggregate([{$match: {f: "case"}}]).itcount()); explain = viewsDB.case_insensitive_view.explain().aggregate([{$match: {f: "case"}}]); - cursorStage = getAggPlanStage(explain, "$cursor"); - assert.neq(null, cursorStage, tojson(explain)); - assert.eq(1, cursorStage.$cursor.queryPlanner.collation.strength, tojson(cursorStage)); + assert.neq(null, explain.queryPlanner, tojson(explain)); + assert.eq(1, explain.queryPlanner.collation.strength, tojson(explain)); // Test that count against a view with a default collation correctly uses the collation. assert.eq(1, viewsDB.case_sensitive_coll.count({f: "case"})); @@ -518,6 +519,8 @@ assert.eq(1, cursorStage.$cursor.queryPlanner.collation.strength, tojson(cursorStage)); // Test that find against a view with a default collation correctly uses the collation. + // We expect the pipeline to be optimized away, so there should be no pipeline stages in + // the explain output. let findRes = viewsDB.runCommand({find: "case_sensitive_coll", filter: {f: "case"}}); assert.commandWorked(findRes); assert.eq(1, findRes.cursor.firstBatch.length); @@ -525,7 +528,6 @@ assert.commandWorked(findRes); assert.eq(3, findRes.cursor.firstBatch.length); explain = viewsDB.runCommand({explain: {find: "case_insensitive_view", filter: {f: "case"}}}); - cursorStage = getAggPlanStage(explain, "$cursor"); - assert.neq(null, cursorStage, tojson(explain)); - assert.eq(1, cursorStage.$cursor.queryPlanner.collation.strength, tojson(cursorStage)); + assert.neq(null, explain.queryPlanner, tojson(explain)); + assert.eq(1, explain.queryPlanner.collation.strength, tojson(explain)); }()); diff --git a/jstests/core/views/views_find.js b/jstests/core/views/views_find.js index df263e3f441..a4d24d1b9b3 100644 --- a/jstests/core/views/views_find.js +++ b/jstests/core/views/views_find.js @@ -78,20 +78,20 @@ // Find with explicit explain modes works on a view. let explainPlan = assert.commandWorked(viewsDB.identityView.find().explain("queryPlanner")); - assert.eq(explainPlan.stages[0].$cursor.queryPlanner.namespace, "views_find.coll"); - assert(!explainPlan.stages[0].$cursor.hasOwnProperty("executionStats")); + assert.eq(explainPlan.queryPlanner.namespace, "views_find.coll"); + assert(!explainPlan.hasOwnProperty("executionStats")); explainPlan = assert.commandWorked(viewsDB.identityView.find().explain("executionStats")); - assert.eq(explainPlan.stages[0].$cursor.queryPlanner.namespace, "views_find.coll"); - assert(explainPlan.stages[0].$cursor.hasOwnProperty("executionStats")); - assert.eq(explainPlan.stages[0].$cursor.executionStats.nReturned, 5); - assert(!explainPlan.stages[0].$cursor.executionStats.hasOwnProperty("allPlansExecution")); + assert.eq(explainPlan.queryPlanner.namespace, "views_find.coll"); + assert(explainPlan.hasOwnProperty("executionStats")); + assert.eq(explainPlan.executionStats.nReturned, 5); + assert(!explainPlan.executionStats.hasOwnProperty("allPlansExecution")); explainPlan = assert.commandWorked(viewsDB.identityView.find().explain("allPlansExecution")); - assert.eq(explainPlan.stages[0].$cursor.queryPlanner.namespace, "views_find.coll"); - assert(explainPlan.stages[0].$cursor.hasOwnProperty("executionStats")); - assert.eq(explainPlan.stages[0].$cursor.executionStats.nReturned, 5); - assert(explainPlan.stages[0].$cursor.executionStats.hasOwnProperty("allPlansExecution")); + assert.eq(explainPlan.queryPlanner.namespace, "views_find.coll"); + assert(explainPlan.hasOwnProperty("executionStats")); + assert.eq(explainPlan.executionStats.nReturned, 5); + assert(explainPlan.executionStats.hasOwnProperty("allPlansExecution")); // Only simple 0 or 1 projections are allowed on views. assert.writeOK(viewsDB.coll.insert({arr: [{x: 1}]})); diff --git a/jstests/libs/analyze_plan.js b/jstests/libs/analyze_plan.js index d18ebe91695..46b31899a41 100644 --- a/jstests/libs/analyze_plan.js +++ b/jstests/libs/analyze_plan.js @@ -26,13 +26,21 @@ function getPlanStages(root, stage) { } } + if ("queryPlanner" in root) { + results = results.concat(getPlanStages(root.queryPlanner.winningPlan, stage)); + } + if ("shards" in root) { - for (var i = 0; i < root.shards.length; i++) { - if ("winningPlan" in root.shards[i]) { - results = results.concat(getPlanStages(root.shards[i].winningPlan, stage)); - } else { - results = results.concat(getPlanStages(root.shards[i].executionStages, stage)); - } + if (Array.isArray(root.shards)) { + results = root.shards.reduce( + (res, shard) => res.concat(getPlanStages( + shard.hasOwnProperty("winningPlan") ? shard.winningPlan : shard.executionStages, + stage)), + results); + } else { + const shards = Object.keys(root.shards); + results = shards.reduce( + (res, shard) => res.concat(getPlanStages(root.shards[shard], stage)), results); } } @@ -92,7 +100,6 @@ function hasRejectedPlans(root) { if (root.hasOwnProperty("shards")) { // This is a sharded agg explain. const cursorStages = getAggPlanStages(root, "$cursor"); - assert(cursorStages.length !== 0, "Did not find any $cursor stages in sharded agg explain"); return cursorStages.find((cursorStage) => cursorStageHasRejectedPlans(cursorStage)) !== undefined; } else if (root.hasOwnProperty("stages")) { @@ -180,11 +187,17 @@ function getAggPlanStages(root, stage) { if (root.stages[0].hasOwnProperty("$cursor")) { results = results.concat(getStagesFromInsideCursorStage(root.stages[0].$cursor)); + } else if (root.stages[0].hasOwnProperty("$geoNearCursor")) { + results = results.concat(getStagesFromInsideCursorStage(root.stages[0].$geoNearCursor)); } } if (root.hasOwnProperty("shards")) { for (let elem in root.shards) { + if (!root.shards[elem].hasOwnProperty("stages")) { + continue; + } + assert(root.shards[elem].stages.constructor === Array); results = results.concat(getDocumentSources(root.shards[elem].stages)); @@ -192,6 +205,8 @@ function getAggPlanStages(root, stage) { const firstStage = root.shards[elem].stages[0]; if (firstStage.hasOwnProperty("$cursor")) { results = results.concat(getStagesFromInsideCursorStage(firstStage.$cursor)); + } else if (firstStage.hasOwnProperty("$geoNearCursor")) { + results = results.concat(getStagesFromInsideCursorStage(firstStage.$geoNearCursor)); } } } @@ -279,6 +294,34 @@ function isCollscan(db, root) { } /** + * Returns true if the BSON representation of a plan rooted at 'root' is using the aggregation + * framework, and false otherwise. + */ +function isAggregationPlan(root) { + if (root.hasOwnProperty("shards")) { + const shards = Object.keys(root.shards); + return shards.reduce( + (res, shard) => res + root.shards[shard].hasOwnProperty("stages") ? 1 : 0, 0) > + 0; + } + return root.hasOwnProperty("stages"); +} + +/** + * Returns true if the BSON representation of a plan rooted at 'root' is using just the query layer, + * and false otherwise. + */ +function isQueryPlan(root) { + if (root.hasOwnProperty("shards")) { + const shards = Object.keys(root.shards); + return shards.reduce( + (res, shard) => res + root.shards[shard].hasOwnProperty("queryPlanner") ? 1 : 0, + 0) > 0; + } + return root.hasOwnProperty("queryPlanner"); +} + +/** * Get the number of chunk skips for the BSON exec stats tree rooted at 'root'. */ function getChunkSkips(root) { diff --git a/jstests/noPassthrough/match_expression_optimization_failpoint.js b/jstests/noPassthrough/match_expression_optimization_failpoint.js index 8cfdf1d51b4..9b30b41a767 100644 --- a/jstests/noPassthrough/match_expression_optimization_failpoint.js +++ b/jstests/noPassthrough/match_expression_optimization_failpoint.js @@ -21,8 +21,7 @@ const enabledPlan = coll.explain().aggregate(pipeline); // Test that a single equality condition $in was optimized to an $eq. - assert.eq(getAggPlanStage(enabledPlan, "$cursor").$cursor.queryPlanner.parsedQuery._id.$eq, - kTestZip); + assert.eq(enabledPlan.queryPlanner.parsedQuery._id.$eq, kTestZip); const enabledResult = coll.aggregate(pipeline).toArray(); @@ -32,8 +31,7 @@ const disabledPlan = coll.explain().aggregate(pipeline); // Test that the $in query still exists and hasn't been optimized to an $eq. - assert.eq(getAggPlanStage(disabledPlan, "$cursor").$cursor.queryPlanner.parsedQuery._id.$in, - [kTestZip]); + assert.eq(disabledPlan.queryPlanner.parsedQuery._id.$in, [kTestZip]); const disabledResult = coll.aggregate(pipeline).toArray(); diff --git a/jstests/noPassthrough/query_knobs_validation.js b/jstests/noPassthrough/query_knobs_validation.js index 259110af23c..536f4d6f995 100644 --- a/jstests/noPassthrough/query_knobs_validation.js +++ b/jstests/noPassthrough/query_knobs_validation.js @@ -168,4 +168,4 @@ MongoRunner.stopMongod(conn); -})();
\ No newline at end of file +})(); diff --git a/jstests/sharding/agg_explain_fmt.js b/jstests/sharding/agg_explain_fmt.js index 31c3c1ccb72..c331b2686b1 100644 --- a/jstests/sharding/agg_explain_fmt.js +++ b/jstests/sharding/agg_explain_fmt.js @@ -29,15 +29,14 @@ for (let shardId in explain.shards) { const shardExplain = explain.shards[shardId]; assert(shardExplain.hasOwnProperty("host"), shardExplain); - assert(shardExplain.hasOwnProperty("stages"), shardExplain); + assert(shardExplain.hasOwnProperty("stages") || shardExplain.hasOwnProperty("queryPlanner"), + shardExplain); } // Do a sharded explain from a mongod, not mongos, to ensure that it does not have a // SHARDING_FILTER stage."); const shardDB = st.shard0.getDB(mongosDB.getName()); explain = shardDB[coll.getName()].explain().aggregate([{$match: {}}]); - assert(!planHasStage( - shardDB, explain.stages[0].$cursor.queryPlanner.winningPlan, "SHARDING_FILTER"), - explain); + assert(!planHasStage(shardDB, explain.queryPlanner.winningPlan, "SHARDING_FILTER"), explain); st.stop(); }()); diff --git a/jstests/sharding/out_with_chunk_migrations.js b/jstests/sharding/out_with_chunk_migrations.js index 8a4b4d658c7..f6ebdee6258 100644 --- a/jstests/sharding/out_with_chunk_migrations.js +++ b/jstests/sharding/out_with_chunk_migrations.js @@ -21,10 +21,13 @@ setAggHang("alwaysOn"); let comment = outMode + "_" + shardedColl.getName() + "_1"; + // The $_internalInhibitOptimization stage is added to the pipeline to prevent the pipeline + // from being optimized away after it's been split. Otherwise, we won't hit the failpoint. let outFn = ` const sourceDB = db.getSiblingDB(jsTestName()); const sourceColl = sourceDB["${sourceColl.getName()}"]; - sourceColl.aggregate([{$out: {to: "${targetColl.getName()}", mode: "${outMode}"}}], + sourceColl.aggregate([{$_internalInhibitOptimization: {}}, + {$out: {to: "${targetColl.getName()}", mode: "${outMode}"}}], {comment: "${comment}"}); `; @@ -53,10 +56,13 @@ assert.commandWorked(targetColl.remove({})); setAggHang("alwaysOn"); comment = outMode + "_" + shardedColl.getName() + "_2"; + // The $_internalInhibitOptimization stage is added to the pipeline to prevent the pipeline + // from being optimized away after it's been split. Otherwise, we won't hit the failpoint. outFn = ` const sourceDB = db.getSiblingDB(jsTestName()); const sourceColl = sourceDB["${sourceColl.getName()}"]; - sourceColl.aggregate([{$out: {to: "${targetColl.getName()}", mode: "${outMode}"}}], + sourceColl.aggregate([{$_internalInhibitOptimization: {}}, + {$out: {to: "${targetColl.getName()}", mode: "${outMode}"}}], {comment: "${comment}"}); `; outShell = startParallelShell(outFn, st.s.port); diff --git a/jstests/sharding/views.js b/jstests/sharding/views.js index 27325338025..c1ea43d8e93 100644 --- a/jstests/sharding/views.js +++ b/jstests/sharding/views.js @@ -10,26 +10,33 @@ // Given sharded explain output in 'shardedExplain', verifies that the explain mode 'verbosity' // affected the output verbosity appropriately, and that the response has the expected format. - function verifyExplainResult(shardedExplain, verbosity) { + // Set 'optimizedAwayPipeline' to true if the pipeline is expected to be optimized away. + function verifyExplainResult( + {shardedExplain = null, verbosity = "", optimizedAwayPipeline = false} = {}) { assert.commandWorked(shardedExplain); assert(shardedExplain.hasOwnProperty("shards"), tojson(shardedExplain)); for (let elem in shardedExplain.shards) { let shard = shardedExplain.shards[elem]; - assert(shard.stages[0].hasOwnProperty("$cursor"), tojson(shardedExplain)); - assert(shard.stages[0].$cursor.hasOwnProperty("queryPlanner"), tojson(shardedExplain)); - if (verbosity === "queryPlanner") { - assert(!shard.stages[0].$cursor.hasOwnProperty("executionStats"), + let root; + if (optimizedAwayPipeline) { + assert(shard.hasOwnProperty("queryPlanner"), tojson(shardedExplain)); + root = shard; + } else { + assert(shard.stages[0].hasOwnProperty("$cursor"), tojson(shardedExplain)); + assert(shard.stages[0].$cursor.hasOwnProperty("queryPlanner"), tojson(shardedExplain)); + root = shard.stages[0].$cursor; + } + if (verbosity === "queryPlanner") { + assert(!root.hasOwnProperty("executionStats"), tojson(shardedExplain)); } else if (verbosity === "executionStats") { - assert(shard.stages[0].$cursor.hasOwnProperty("executionStats"), - tojson(shardedExplain)); - assert(!shard.stages[0].$cursor.executionStats.hasOwnProperty("allPlansExecution"), + assert(root.hasOwnProperty("executionStats"), tojson(shardedExplain)); + assert(!root.executionStats.hasOwnProperty("allPlansExecution"), tojson("shardedExplain")); } else { assert.eq(verbosity, "allPlansExecution", tojson(shardedExplain)); - assert(shard.stages[0].$cursor.hasOwnProperty("executionStats"), - tojson(shardedExplain)); - assert(shard.stages[0].$cursor.executionStats.hasOwnProperty("allPlansExecution"), + assert(root.hasOwnProperty("executionStats"), tojson(shardedExplain)); + assert(root.executionStats.hasOwnProperty("allPlansExecution"), tojson(shardedExplain)); } } @@ -67,11 +74,13 @@ assert.eq(5, view.find({a: {$lte: 8}}).itcount()); let result = db.runCommand({explain: {find: "view", filter: {a: {$lte: 7}}}}); - verifyExplainResult(result, "allPlansExecution"); + verifyExplainResult( + {shardedExplain: result, verbosity: "allPlansExecution", optimizedAwayPipeline: true}); for (let verbosity of explainVerbosities) { result = db.runCommand({explain: {find: "view", filter: {a: {$lte: 7}}}, verbosity: verbosity}); - verifyExplainResult(result, verbosity); + verifyExplainResult( + {shardedExplain: result, verbosity: verbosity, optimizedAwayPipeline: true}); } // @@ -82,17 +91,20 @@ // Test that the explain:true flag for the aggregate command results in queryPlanner verbosity. result = db.runCommand({aggregate: "view", pipeline: [{$match: {a: {$lte: 8}}}], explain: true}); - verifyExplainResult(result, "queryPlanner"); + verifyExplainResult( + {shardedExplain: result, verbosity: "queryPlanner", optimizedAwayPipeline: true}); result = db.runCommand( {explain: {aggregate: "view", pipeline: [{$match: {a: {$lte: 8}}}], cursor: {}}}); - verifyExplainResult(result, "allPlansExecution"); + verifyExplainResult( + {shardedExplain: result, verbosity: "allPlansExecution", optimizedAwayPipeline: true}); for (let verbosity of explainVerbosities) { result = db.runCommand({ explain: {aggregate: "view", pipeline: [{$match: {a: {$lte: 8}}}], cursor: {}}, verbosity: verbosity }); - verifyExplainResult(result, verbosity); + verifyExplainResult( + {shardedExplain: result, verbosity: verbosity, optimizedAwayPipeline: true}); } // @@ -101,11 +113,11 @@ assert.eq(5, view.count({a: {$lte: 8}})); result = db.runCommand({explain: {count: "view", query: {a: {$lte: 8}}}}); - verifyExplainResult(result, "allPlansExecution"); + verifyExplainResult({shardedExplain: result, verbosity: "allPlansExecution"}); for (let verbosity of explainVerbosities) { result = db.runCommand({explain: {count: "view", query: {a: {$lte: 8}}}, verbosity: verbosity}); - verifyExplainResult(result, verbosity); + verifyExplainResult({shardedExplain: result, verbosity: verbosity}); } // @@ -116,11 +128,11 @@ assert.eq([4, 5, 6, 7, 8], result.values.sort()); result = db.runCommand({explain: {distinct: "view", key: "a", query: {a: {$lte: 8}}}}); - verifyExplainResult(result, "allPlansExecution"); + verifyExplainResult({shardedExplain: result, verbosity: "allPlansExecution"}); for (let verbosity of explainVerbosities) { result = db.runCommand( {explain: {distinct: "view", key: "a", query: {a: {$lte: 8}}}, verbosity: verbosity}); - verifyExplainResult(result, verbosity); + verifyExplainResult({shardedExplain: result, verbosity: verbosity}); } // diff --git a/src/mongo/db/commands/count_cmd.cpp b/src/mongo/db/commands/count_cmd.cpp index 360fb2f3128..72ce0734bea 100644 --- a/src/mongo/db/commands/count_cmd.cpp +++ b/src/mongo/db/commands/count_cmd.cpp @@ -174,7 +174,7 @@ public: auto exec = std::move(statusWithPlanExecutor.getValue()); auto bodyBuilder = result->getBodyBuilder(); - Explain::explainStages(exec.get(), collection, verbosity, &bodyBuilder); + Explain::explainStages(exec.get(), collection, verbosity, BSONObj(), &bodyBuilder); return Status::OK(); } diff --git a/src/mongo/db/commands/distinct.cpp b/src/mongo/db/commands/distinct.cpp index 2f51a659469..1679ea78656 100644 --- a/src/mongo/db/commands/distinct.cpp +++ b/src/mongo/db/commands/distinct.cpp @@ -163,7 +163,7 @@ public: getExecutorDistinct(opCtx, collection, QueryPlannerParams::DEFAULT, &parsedDistinct)); auto bodyBuilder = result->getBodyBuilder(); - Explain::explainStages(executor.get(), collection, verbosity, &bodyBuilder); + Explain::explainStages(executor.get(), collection, verbosity, BSONObj(), &bodyBuilder); return Status::OK(); } diff --git a/src/mongo/db/commands/find_and_modify.cpp b/src/mongo/db/commands/find_and_modify.cpp index 13a5b38e941..ae25da086c8 100644 --- a/src/mongo/db/commands/find_and_modify.cpp +++ b/src/mongo/db/commands/find_and_modify.cpp @@ -262,7 +262,7 @@ public: uassertStatusOK(getExecutorDelete(opCtx, opDebug, collection, &parsedDelete)); auto bodyBuilder = result->getBodyBuilder(); - Explain::explainStages(exec.get(), collection, verbosity, &bodyBuilder); + Explain::explainStages(exec.get(), collection, verbosity, BSONObj(), &bodyBuilder); } else { UpdateRequest request(nsString); const bool isExplain = true; @@ -286,7 +286,7 @@ public: uassertStatusOK(getExecutorUpdate(opCtx, opDebug, collection, &parsedUpdate)); auto bodyBuilder = result->getBodyBuilder(); - Explain::explainStages(exec.get(), collection, verbosity, &bodyBuilder); + Explain::explainStages(exec.get(), collection, verbosity, BSONObj(), &bodyBuilder); } return Status::OK(); diff --git a/src/mongo/db/commands/find_cmd.cpp b/src/mongo/db/commands/find_cmd.cpp index 2c17b974c5c..6646384d4a9 100644 --- a/src/mongo/db/commands/find_cmd.cpp +++ b/src/mongo/db/commands/find_cmd.cpp @@ -220,7 +220,7 @@ public: auto bodyBuilder = result->getBodyBuilder(); // Got the execution tree. Explain it. - Explain::explainStages(exec.get(), collection, verbosity, &bodyBuilder); + Explain::explainStages(exec.get(), collection, verbosity, BSONObj(), &bodyBuilder); } /** diff --git a/src/mongo/db/commands/getmore_cmd.cpp b/src/mongo/db/commands/getmore_cmd.cpp index 54478be0dff..4e09cf82a72 100644 --- a/src/mongo/db/commands/getmore_cmd.cpp +++ b/src/mongo/db/commands/getmore_cmd.cpp @@ -347,13 +347,26 @@ public: } else { invariant(cursorPin->lockPolicy() == ClientCursorParams::LockPolicy::kLockExternally); + if (MONGO_FAIL_POINT(GetMoreHangBeforeReadLock)) { log() << "GetMoreHangBeforeReadLock fail point enabled. Blocking until fail " "point is disabled."; MONGO_FAIL_POINT_PAUSE_WHILE_SET_OR_INTERRUPTED(opCtx, GetMoreHangBeforeReadLock); } - readLock.emplace(opCtx, _request.nss); + + // Lock the backing collection by using the executor's namespace. Note that it may + // be different from the cursor's namespace. One such possible scenario is when + // getMore() is executed against a view. Technically, views are pipelines and under + // normal circumstances use 'kLocksInternally' policy, so we shouldn't be getting + // into here in the first place. However, if the pipeline was optimized away and + // replaced with a query plan, its lock policy would have also been changed to + // 'kLockExternally'. So, we'll use the executor's namespace to take the lock (which + // is always the backing collection namespace), but will use the namespace provided + // in the user request for profiling. + // Otherwise, these two namespaces will match. + readLock.emplace(opCtx, cursorPin->getExecutor()->nss()); + const int doNotChangeProfilingLevel = 0; statsTracker.emplace(opCtx, _request.nss, diff --git a/src/mongo/db/commands/run_aggregate.cpp b/src/mongo/db/commands/run_aggregate.cpp index 850e1823525..6aeb2f40659 100644 --- a/src/mongo/db/commands/run_aggregate.cpp +++ b/src/mongo/db/commands/run_aggregate.cpp @@ -48,6 +48,7 @@ #include "mongo/db/pipeline/document.h" #include "mongo/db/pipeline/document_source.h" #include "mongo/db/pipeline/document_source_exchange.h" +#include "mongo/db/pipeline/document_source_geo_near.h" #include "mongo/db/pipeline/expression.h" #include "mongo/db/pipeline/expression_context.h" #include "mongo/db/pipeline/lite_parsed_pipeline.h" @@ -59,6 +60,7 @@ #include "mongo/db/query/find_common.h" #include "mongo/db/query/get_executor.h" #include "mongo/db/query/plan_summary_stats.h" +#include "mongo/db/query/query_planner_common.h" #include "mongo/db/read_concern.h" #include "mongo/db/repl/oplog.h" #include "mongo/db/repl/read_concern_args.h" @@ -86,6 +88,36 @@ using stdx::make_unique; namespace { /** + * Returns true if this PlanExecutor is for a Pipeline. + */ +bool isPipelineExecutor(const PlanExecutor* exec) { + invariant(exec); + auto rootStage = exec->getRootStage(); + return rootStage->stageType() == StageType::STAGE_PIPELINE_PROXY || + rootStage->stageType() == StageType::STAGE_CHANGE_STREAM_PROXY; +} + +/** + * If a pipeline is empty (assuming that a $cursor stage hasn't been created yet), it could mean + * that we were able to absorb all pipeline stages and pull them into a single PlanExecutor. So, + * instead of creating a whole pipeline to do nothing more than forward the results of its cursor + * document source, we can optimize away the entire pipeline and answer the request using the query + * engine only. This function checks if such optimization is possible. + */ +bool canOptimizeAwayPipeline(const Pipeline* pipeline, + const PlanExecutor* exec, + const AggregationRequest& request, + bool hasGeoNearStage, + bool hasChangeStreamStage) { + return pipeline && exec && !hasGeoNearStage && !hasChangeStreamStage && + pipeline->getSources().empty() && + // For exchange we will create a number of pipelines consisting of a single + // DocumentSourceExchange stage, so cannot not optimize it away. + !request.getExchangeSpec() && + !QueryPlannerCommon::hasNode(exec->getCanonicalQuery()->root(), MatchExpression::TEXT); +} + +/** * Returns true if we need to keep a ClientCursor saved for this pipeline (for future getMore * requests). Otherwise, returns false. The passed 'nsForCursor' is only used to determine the * namespace used in the returned cursor, which will be registered with the global cursor manager, @@ -135,8 +167,11 @@ bool handleCursorCommand(OperationContext* opCtx, options.isInitialResponse = true; CursorResponseBuilder responseBuilder(result, options); - ClientCursor* cursor = cursors[0]; + auto curOp = CurOp::get(opCtx); + auto cursor = cursors[0]; invariant(cursor); + auto exec = cursor->getExecutor(); + invariant(exec); BSONObj next; bool stashedResult = false; @@ -146,12 +181,13 @@ bool handleCursorCommand(OperationContext* opCtx, PlanExecutor::ExecState state; try { - state = cursor->getExecutor()->getNext(&next, nullptr); + state = exec->getNext(&next, nullptr); } catch (const ExceptionFor<ErrorCodes::CloseChangeStream>&) { // This exception is thrown when a $changeStream stage encounters an event // that invalidates the cursor. We should close the cursor and return without // error. cursor = nullptr; + exec = nullptr; break; } @@ -159,6 +195,7 @@ bool handleCursorCommand(OperationContext* opCtx, if (!cursor->isTailable()) { // make it an obvious error to use cursor or executor after this point cursor = nullptr; + exec = nullptr; } break; } @@ -171,22 +208,23 @@ bool handleCursorCommand(OperationContext* opCtx, // If adding this object will cause us to exceed the message size limit, then we stash it // for later. if (!FindCommon::haveSpaceForNext(next, objCount, responseBuilder.bytesUsed())) { - cursor->getExecutor()->enqueue(next); + exec->enqueue(next); stashedResult = true; break; } // TODO SERVER-38539: We need to set both the latestOplogTimestamp and the PBRT until the // former is removed in a future release. - responseBuilder.setLatestOplogTimestamp(cursor->getExecutor()->getLatestOplogTimestamp()); - responseBuilder.setPostBatchResumeToken(cursor->getExecutor()->getPostBatchResumeToken()); + responseBuilder.setLatestOplogTimestamp(exec->getLatestOplogTimestamp()); + responseBuilder.setPostBatchResumeToken(exec->getPostBatchResumeToken()); responseBuilder.append(next); } if (cursor) { + invariant(cursor->getExecutor() == exec); + // For empty batches, or in the case where the final result was added to the batch rather // than being stashed, we update the PBRT to ensure that it is the most recent available. - const auto* exec = cursor->getExecutor(); if (!stashedResult) { // TODO SERVER-38539: We need to set both the latestOplogTimestamp and the PBRT until // the former is removed in a future release. @@ -197,14 +235,14 @@ bool handleCursorCommand(OperationContext* opCtx, // cursor (for use by future getmore ops). cursor->setLeftoverMaxTimeMicros(opCtx->getRemainingMaxTimeMicros()); - CurOp::get(opCtx)->debug().cursorid = cursor->cursorid(); + curOp->debug().cursorid = cursor->cursorid(); // Cursor needs to be in a saved state while we yield locks for getmore. State // will be restored in getMore(). - cursor->getExecutor()->saveState(); - cursor->getExecutor()->detachFromOperationContext(); + exec->saveState(); + exec->detachFromOperationContext(); } else { - CurOp::get(opCtx)->debug().cursorExhausted = true; + curOp->debug().cursorExhausted = true; } const CursorId cursorId = cursor ? cursor->cursorid() : 0LL; @@ -387,6 +425,69 @@ void _adjustChangeStreamReadConcern(OperationContext* opCtx) { waitForReadConcern(opCtx, readConcernArgs, true, PrepareConflictBehavior::kIgnore)); } +/** + * If the aggregation 'request' contains an exchange specification, create a new pipeline for each + * consumer and put it into the resulting vector. Otherwise, return the original 'pipeline' as a + * single vector element. + */ +std::vector<std::unique_ptr<Pipeline, PipelineDeleter>> createExchangePipelinesIfNeeded( + OperationContext* opCtx, + boost::intrusive_ptr<ExpressionContext> expCtx, + const AggregationRequest& request, + std::unique_ptr<Pipeline, PipelineDeleter> pipeline, + boost::optional<UUID> uuid) { + std::vector<std::unique_ptr<Pipeline, PipelineDeleter>> pipelines; + + if (request.getExchangeSpec() && !expCtx->explain) { + boost::intrusive_ptr<Exchange> exchange = + new Exchange(request.getExchangeSpec().get(), std::move(pipeline)); + + for (size_t idx = 0; idx < exchange->getConsumers(); ++idx) { + // For every new pipeline we have create a new ExpressionContext as the context + // cannot be shared between threads. There is no synchronization for pieces of + // the execution machinery above the Exchange, so nothing above the Exchange can be + // shared between different exchange-producer cursors. + expCtx = makeExpressionContext(opCtx, + request, + expCtx->getCollator() ? expCtx->getCollator()->clone() + : nullptr, + uuid); + + // Create a new pipeline for the consumer consisting of a single + // DocumentSourceExchange. + boost::intrusive_ptr<DocumentSource> consumer = new DocumentSourceExchange( + expCtx, exchange, idx, expCtx->mongoProcessInterface->getResourceYielder()); + pipelines.emplace_back(uassertStatusOK(Pipeline::create({consumer}, expCtx))); + } + } else { + pipelines.emplace_back(std::move(pipeline)); + } + + return pipelines; +} + +/** + * Create a PlanExecutor to execute the given 'pipeline'. + */ +std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> createOuterPipelineProxyExecutor( + OperationContext* opCtx, + const NamespaceString& nss, + std::unique_ptr<Pipeline, PipelineDeleter> pipeline, + bool hasChangeStream) { + // Transfer ownership of the Pipeline to the PipelineProxyStage. + auto ws = make_unique<WorkingSet>(); + auto proxy = hasChangeStream + ? make_unique<ChangeStreamProxyStage>(opCtx, std::move(pipeline), ws.get()) + : make_unique<PipelineProxyStage>(opCtx, std::move(pipeline), ws.get()); + + // This PlanExecutor will simply forward requests to the Pipeline, so does not need + // to yield or to be registered with any collection's CursorManager to receive + // invalidations. The Pipeline may contain PlanExecutors which *are* yielding + // PlanExecutors and which *are* registered with their respective collection's + // CursorManager + return uassertStatusOK( + PlanExecutor::make(opCtx, std::move(ws), std::move(proxy), nss, PlanExecutor::NO_YIELD)); +} } // namespace Status runAggregate(OperationContext* opCtx, @@ -405,6 +506,11 @@ Status runAggregate(OperationContext* opCtx, // The UUID of the collection for the execution namespace of this aggregation. boost::optional<UUID> uuid; + // If emplaced, AutoGetCollectionForReadCommand will throw if the sharding version for this + // connection is out of date. If the namespace is a view, the lock will be released before + // re-running the expanded aggregation. + boost::optional<AutoGetCollectionForReadCommand> ctx; + std::vector<unique_ptr<PlanExecutor, PlanExecutor::Deleter>> execs; boost::intrusive_ptr<ExpressionContext> expCtx; auto curOp = CurOp::get(opCtx); @@ -428,11 +534,6 @@ Status runAggregate(OperationContext* opCtx, const auto& pipelineInvolvedNamespaces = liteParsedPipeline.getInvolvedNamespaces(); - // If emplaced, AutoGetCollectionForReadCommand will throw if the sharding version for this - // connection is out of date. If the namespace is a view, the lock will be released before - // re-running the expanded aggregation. - boost::optional<AutoGetCollectionForReadCommand> ctx; - // If this is a collectionless aggregation, we won't create 'ctx' but will still need an // AutoStatsTracker to record CurOp and Top entries. boost::optional<AutoStatsTracker> statsTracker; @@ -538,65 +639,63 @@ Status runAggregate(OperationContext* opCtx, pipeline->optimizePipeline(); + // Check if the pipeline has a $geoNear stage, as it will be ripped away during the build + // query executor phase below (to be replaced with a $geoNearCursorStage later during the + // executor attach phase). + auto hasGeoNearStage = !pipeline->getSources().empty() && + dynamic_cast<DocumentSourceGeoNear*>(pipeline->peekFront()); + // Prepare a PlanExecutor to provide input into the pipeline, if needed. + std::pair<PipelineD::AttachExecutorCallback, + std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> + attachExecutorCallback; if (liteParsedPipeline.hasChangeStream()) { // If we are using a change stream, the cursor stage should have a simple collation, // regardless of what the user's collation was. std::unique_ptr<CollatorInterface> collatorForCursor = nullptr; auto collatorStash = expCtx->temporarilyChangeCollator(std::move(collatorForCursor)); - PipelineD::prepareCursorSource(collection, nss, &request, pipeline.get()); + attachExecutorCallback = + PipelineD::buildInnerQueryExecutor(collection, nss, &request, pipeline.get()); } else { - PipelineD::prepareCursorSource(collection, nss, &request, pipeline.get()); + attachExecutorCallback = + PipelineD::buildInnerQueryExecutor(collection, nss, &request, pipeline.get()); } - // Optimize again, since there may be additional optimizations that can be done after adding - // the initial cursor stage. Note this has to be done outside the above blocks to ensure - // this process uses the correct collation if it does any string comparisons. - pipeline->optimizePipeline(); - std::vector<std::unique_ptr<Pipeline, PipelineDeleter>> pipelines; - - if (request.getExchangeSpec() && !expCtx->explain) { - boost::intrusive_ptr<Exchange> exchange = - new Exchange(request.getExchangeSpec().get(), std::move(pipeline)); - - for (size_t idx = 0; idx < exchange->getConsumers(); ++idx) { - // For every new pipeline we have create a new ExpressionContext as the context - // cannot be shared between threads. There is no synchronization for pieces of the - // execution machinery above the Exchange, so nothing above the Exchange can be - // shared between different exchange-producer cursors. - expCtx = makeExpressionContext( - opCtx, - request, - expCtx->getCollator() ? expCtx->getCollator()->clone() : nullptr, - uuid); - - // Create a new pipeline for the consumer consisting of a single - // DocumentSourceExchange. - boost::intrusive_ptr<DocumentSource> consumer = new DocumentSourceExchange( - expCtx, exchange, idx, expCtx->mongoProcessInterface->getResourceYielder()); - pipelines.emplace_back(uassertStatusOK(Pipeline::create({consumer}, expCtx))); - } + if (canOptimizeAwayPipeline(pipeline.get(), + attachExecutorCallback.second.get(), + request, + hasGeoNearStage, + liteParsedPipeline.hasChangeStream())) { + // This pipeline is currently empty, but once completed it will have only one source, + // which is a DocumentSourceCursor. Instead of creating a whole pipeline to do nothing + // more than forward the results of its cursor document source, we can use the + // PlanExecutor by itself. The resulting cursor will look like what the client would + // have gotten from find command. + execs.emplace_back(std::move(attachExecutorCallback.second)); } else { - pipelines.emplace_back(std::move(pipeline)); - } + // Complete creation of the initial $cursor stage, if needed. + PipelineD::attachInnerQueryExecutorToPipeline(collection, + attachExecutorCallback.first, + std::move(attachExecutorCallback.second), + pipeline.get()); + + // Optimize again, since there may be additional optimizations that can be done after + // adding the initial cursor stage. + pipeline->optimizePipeline(); + + auto pipelines = + createExchangePipelinesIfNeeded(opCtx, expCtx, request, std::move(pipeline), uuid); + for (auto&& pipelineIt : pipelines) { + execs.emplace_back(createOuterPipelineProxyExecutor( + opCtx, nss, std::move(pipelineIt), liteParsedPipeline.hasChangeStream())); + } - for (size_t idx = 0; idx < pipelines.size(); ++idx) { - // Transfer ownership of the Pipeline to the PipelineProxyStage. - auto ws = make_unique<WorkingSet>(); - auto proxy = liteParsedPipeline.hasChangeStream() - ? make_unique<ChangeStreamProxyStage>(opCtx, std::move(pipelines[idx]), ws.get()) - : make_unique<PipelineProxyStage>(opCtx, std::move(pipelines[idx]), ws.get()); - - // This PlanExecutor will simply forward requests to the Pipeline, so does not need to - // yield or to be registered with any collection's CursorManager to receive - // invalidations. The Pipeline may contain PlanExecutors which *are* yielding - // PlanExecutors and which *are* registered with their respective collection's - // CursorManager - - auto statusWithPlanExecutor = PlanExecutor::make( - opCtx, std::move(ws), std::move(proxy), nss, PlanExecutor::NO_YIELD); - invariant(statusWithPlanExecutor.isOK()); - execs.emplace_back(std::move(statusWithPlanExecutor.getValue())); + // With the pipelines created, we can relinquish locks as they will manage the locks + // internally further on. We still need to keep the lock for an optimized away pipeline + // though, as we will be changing its lock policy to 'kLockExternally' (see details + // below), and in order to execute the initial getNext() call in 'handleCursorCommand', + // we need to hold the collection lock. + ctx.reset(); } { @@ -620,14 +719,22 @@ Status runAggregate(OperationContext* opCtx, p.deleteUnderlying(); } }); - for (size_t idx = 0; idx < execs.size(); ++idx) { + for (auto&& exec : execs) { + // PlanExecutors for pipelines always have a 'kLocksInternally' policy. If this executor is + // not for a pipeline, though, that means the pipeline was optimized away and the + // PlanExecutor will answer the query using the query engine only. Without the + // DocumentSourceCursor to do its locking, an executor needs a 'kLockExternally' policy. + auto lockPolicy = isPipelineExecutor(exec.get()) + ? ClientCursorParams::LockPolicy::kLocksInternally + : ClientCursorParams::LockPolicy::kLockExternally; + ClientCursorParams cursorParams( - std::move(execs[idx]), + std::move(exec), origNss, AuthorizationSession::get(opCtx->getClient())->getAuthenticatedUserNames(), repl::ReadConcernArgs::get(opCtx), cmdObj, - ClientCursorParams::LockPolicy::kLocksInternally, + lockPolicy, privileges); if (expCtx->tailableMode == TailableModeEnum::kTailable) { cursorParams.setTailable(true); @@ -637,15 +744,32 @@ Status runAggregate(OperationContext* opCtx, } auto pin = CursorManager::get(opCtx)->registerCursor(opCtx, std::move(cursorParams)); + invariant(!exec); + cursors.emplace_back(pin.getCursor()); pins.emplace_back(std::move(pin)); } // If both explain and cursor are specified, explain wins. if (expCtx->explain) { + auto explainExecutor = pins[0].getCursor()->getExecutor(); auto bodyBuilder = result->getBodyBuilder(); - Explain::explainPipelineExecutor( - pins[0].getCursor()->getExecutor(), *(expCtx->explain), &bodyBuilder); + if (isPipelineExecutor(explainExecutor)) { + Explain::explainPipelineExecutor(explainExecutor, *(expCtx->explain), &bodyBuilder); + } else { + invariant(pins[0].getCursor()->lockPolicy() == + ClientCursorParams::LockPolicy::kLockExternally); + invariant(!explainExecutor->isDetached()); + invariant(explainExecutor->getOpCtx() == opCtx); + // The explainStages() function for a non-pipeline executor expects to be called with + // the appropriate collection lock already held. Make sure it has not been released yet. + invariant(ctx); + Explain::explainStages(explainExecutor, + ctx->getCollection(), + *(expCtx->explain), + BSON("optimizedPipeline" << true), + &bodyBuilder); + } } else { // Cursor must be specified, if explain is not. const bool keepCursor = @@ -653,13 +777,16 @@ Status runAggregate(OperationContext* opCtx, if (keepCursor) { cursorFreer.dismiss(); } - } - if (!expCtx->explain) { PlanSummaryStats stats; Explain::getSummaryStats(*(pins[0].getCursor()->getExecutor()), &stats); curOp->debug().setPlanSummaryMetrics(stats); curOp->debug().nreturned = stats.nReturned; + // For an optimized away pipeline, signal the cache that a query operation has completed. + // For normal pipelines this is done in DocumentSourceCursor. + if (ctx && ctx->getCollection()) { + ctx->getCollection()->infoCache()->notifyOfQuery(opCtx, stats.indexesUsed); + } } // Any code that needs the cursor pinned must be inside the try block, above. diff --git a/src/mongo/db/commands/write_commands/write_commands.cpp b/src/mongo/db/commands/write_commands/write_commands.cpp index 75879251925..f480dda2886 100644 --- a/src/mongo/db/commands/write_commands/write_commands.cpp +++ b/src/mongo/db/commands/write_commands/write_commands.cpp @@ -379,7 +379,8 @@ private: auto exec = uassertStatusOK(getExecutorUpdate( opCtx, &CurOp::get(opCtx)->debug(), collection.getCollection(), &parsedUpdate)); auto bodyBuilder = result->getBodyBuilder(); - Explain::explainStages(exec.get(), collection.getCollection(), verbosity, &bodyBuilder); + Explain::explainStages( + exec.get(), collection.getCollection(), verbosity, BSONObj(), &bodyBuilder); } write_ops::Update _batch; @@ -452,7 +453,8 @@ private: auto exec = uassertStatusOK(getExecutorDelete( opCtx, &CurOp::get(opCtx)->debug(), collection.getCollection(), &parsedDelete)); auto bodyBuilder = result->getBodyBuilder(); - Explain::explainStages(exec.get(), collection.getCollection(), verbosity, &bodyBuilder); + Explain::explainStages( + exec.get(), collection.getCollection(), verbosity, BSONObj(), &bodyBuilder); } write_ops::Delete _batch; diff --git a/src/mongo/db/pipeline/document_source_cursor.cpp b/src/mongo/db/pipeline/document_source_cursor.cpp index 99e45abbfe9..d215450b169 100644 --- a/src/mongo/db/pipeline/document_source_cursor.cpp +++ b/src/mongo/db/pipeline/document_source_cursor.cpp @@ -240,6 +240,7 @@ Value DocumentSourceCursor::serialize(boost::optional<ExplainOptions::Verbosity> verbosity.get(), _execStatus, _winningPlanTrialStats.get(), + BSONObj(), &explainStatsBuilder); } diff --git a/src/mongo/db/pipeline/document_source_project.cpp b/src/mongo/db/pipeline/document_source_project.cpp index 16b59832e3c..15d8eb66af3 100644 --- a/src/mongo/db/pipeline/document_source_project.cpp +++ b/src/mongo/db/pipeline/document_source_project.cpp @@ -57,7 +57,7 @@ intrusive_ptr<DocumentSource> DocumentSourceProject::create( projectSpec, {ProjectionPolicies::DefaultIdPolicy::kIncludeId, ProjectionPolicies::ArrayRecursionPolicy::kRecurseNestedArrays}), - "$project", + DocumentSourceProject::kStageName.rawData(), isIndependentOfAnyCollection)); return project; } diff --git a/src/mongo/db/pipeline/document_source_project.h b/src/mongo/db/pipeline/document_source_project.h index aa6701120b1..b8d211b602e 100644 --- a/src/mongo/db/pipeline/document_source_project.h +++ b/src/mongo/db/pipeline/document_source_project.h @@ -41,6 +41,8 @@ namespace mongo { */ class DocumentSourceProject final { public: + static constexpr StringData kStageName = "$project"_sd; + /** * Convenience method to create a $project stage from 'projectSpec'. */ diff --git a/src/mongo/db/pipeline/pipeline_d.cpp b/src/mongo/db/pipeline/pipeline_d.cpp index 28f10d4e401..fa47f804077 100644 --- a/src/mongo/db/pipeline/pipeline_d.cpp +++ b/src/mongo/db/pipeline/pipeline_d.cpp @@ -291,17 +291,18 @@ StringData extractGeoNearFieldFromIndexes(OperationContext* opCtx, Collection* c } } // namespace -void PipelineD::prepareCursorSource(Collection* collection, - const NamespaceString& nss, - const AggregationRequest* aggRequest, - Pipeline* pipeline) { +std::pair<PipelineD::AttachExecutorCallback, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> +PipelineD::buildInnerQueryExecutor(Collection* collection, + const NamespaceString& nss, + const AggregationRequest* aggRequest, + Pipeline* pipeline) { auto expCtx = pipeline->getContext(); // We will be modifying the source vector as we go. Pipeline::SourceContainer& sources = pipeline->_sources; if (!sources.empty() && !sources.front()->constraints().requiresInputDocSource) { - return; + return {}; } // We are going to generate an input cursor, so we need to be holding the collection lock. @@ -339,10 +340,15 @@ void PipelineD::prepareCursorSource(Collection* collection, // TODO SERVER-37453 this should no longer be necessary when we no don't need locks // to destroy a PlanExecutor. auto deps = pipeline->getDependencies(DepsTracker::MetadataAvailable::kNoMetadata); - addCursorSource(pipeline, - DocumentSourceCursor::create(collection, std::move(exec), expCtx), - std::move(deps)); - return; + auto attachExecutorCallback = + [deps](Collection* collection, + std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec, + Pipeline* pipeline) { + auto cursor = DocumentSourceCursor::create( + collection, std::move(exec), pipeline->getContext()); + addCursorSource(pipeline, std::move(cursor), std::move(deps)); + }; + return std::make_pair(std::move(attachExecutorCallback), std::move(exec)); } } } @@ -352,12 +358,35 @@ void PipelineD::prepareCursorSource(Collection* collection, const auto geoNearStage = sources.empty() ? nullptr : dynamic_cast<DocumentSourceGeoNear*>(sources.front().get()); if (geoNearStage) { - prepareGeoNearCursorSource(collection, nss, aggRequest, pipeline); + return buildInnerQueryExecutorGeoNear(collection, nss, aggRequest, pipeline); } else { - prepareGenericCursorSource(collection, nss, aggRequest, pipeline); + return buildInnerQueryExecutorGeneric(collection, nss, aggRequest, pipeline); } } +void PipelineD::attachInnerQueryExecutorToPipeline( + Collection* collection, + PipelineD::AttachExecutorCallback attachExecutorCallback, + std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec, + Pipeline* pipeline) { + // If the pipeline doesn't need a $cursor stage, there will be no callback function and + // PlanExecutor provided in the 'attachExecutorCallback' object, so we don't need to do + // anything. + if (attachExecutorCallback && exec) { + attachExecutorCallback(collection, std::move(exec), pipeline); + } +} + +void PipelineD::buildAndAttachInnerQueryExecutorToPipeline(Collection* collection, + const NamespaceString& nss, + const AggregationRequest* aggRequest, + Pipeline* pipeline) { + + auto callback = PipelineD::buildInnerQueryExecutor(collection, nss, aggRequest, pipeline); + PipelineD::attachInnerQueryExecutorToPipeline( + collection, callback.first, std::move(callback.second), pipeline); +} + namespace { /** @@ -397,10 +426,11 @@ getSortAndGroupStagesFromPipeline(const Pipeline::SourceContainer& sources) { } // namespace -void PipelineD::prepareGenericCursorSource(Collection* collection, - const NamespaceString& nss, - const AggregationRequest* aggRequest, - Pipeline* pipeline) { +std::pair<PipelineD::AttachExecutorCallback, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> +PipelineD::buildInnerQueryExecutorGeneric(Collection* collection, + const NamespaceString& nss, + const AggregationRequest* aggRequest, + Pipeline* pipeline) { Pipeline::SourceContainer& sources = pipeline->_sources; auto expCtx = pipeline->getContext(); @@ -477,18 +507,23 @@ void PipelineD::prepareGenericCursorSource(Collection* collection, const bool trackOplogTS = (pipeline->peekFront() && pipeline->peekFront()->constraints().isChangeStreamStage()); - addCursorSource(pipeline, - DocumentSourceCursor::create(collection, std::move(exec), expCtx, trackOplogTS), - deps, - queryObj, - sortObj, - projForQuery); + auto attachExecutorCallback = [deps, queryObj, sortObj, projForQuery, trackOplogTS]( + Collection* collection, + std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec, + Pipeline* pipeline) { + auto cursor = DocumentSourceCursor::create( + collection, std::move(exec), pipeline->getContext(), trackOplogTS); + addCursorSource( + pipeline, std::move(cursor), std::move(deps), queryObj, sortObj, projForQuery); + }; + return std::make_pair(std::move(attachExecutorCallback), std::move(exec)); } -void PipelineD::prepareGeoNearCursorSource(Collection* collection, - const NamespaceString& nss, - const AggregationRequest* aggRequest, - Pipeline* pipeline) { +std::pair<PipelineD::AttachExecutorCallback, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> +PipelineD::buildInnerQueryExecutorGeoNear(Collection* collection, + const NamespaceString& nss, + const AggregationRequest* aggRequest, + Pipeline* pipeline) { uassert(ErrorCodes::NamespaceNotFound, str::stream() << "$geoNear requires a geo index to run, but " << nss.ns() << " does not exist", @@ -532,17 +567,26 @@ void PipelineD::prepareGeoNearCursorSource(Collection* collection, str::stream() << "Unexpectedly got the following sort from the query system: " << sortFromQuerySystem.jsonString()); - auto geoNearCursor = - DocumentSourceGeoNearCursor::create(collection, - std::move(exec), - expCtx, - geoNearStage->getDistanceField(), - geoNearStage->getLocationField(), - geoNearStage->getDistanceMultiplier().value_or(1.0)); - + auto attachExecutorCallback = + [ + deps, + distanceField = geoNearStage->getDistanceField(), + locationField = geoNearStage->getLocationField(), + distanceMultiplier = geoNearStage->getDistanceMultiplier().value_or(1.0) + ](Collection * collection, + std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec, + Pipeline * pipeline) { + auto cursor = DocumentSourceGeoNearCursor::create(collection, + std::move(exec), + pipeline->getContext(), + distanceField, + locationField, + distanceMultiplier); + addCursorSource(pipeline, std::move(cursor), std::move(deps)); + }; // Remove the initial $geoNear; it will be replaced by $geoNearCursor. sources.pop_front(); - addCursorSource(pipeline, std::move(geoNearCursor), std::move(deps)); + return std::make_pair(std::move(attachExecutorCallback), std::move(exec)); } StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> PipelineD::prepareExecutor( @@ -755,7 +799,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> PipelineD::prep << swExecutorProj.getStatus().toString()}; } - // The query system couldn't provide a covered projection. + // The query system couldn't provide a covered or simple uncovered projection. *projectionObj = BSONObj(); // If this doesn't work, nothing will. return attemptToGetExecutor(opCtx, diff --git a/src/mongo/db/pipeline/pipeline_d.h b/src/mongo/db/pipeline/pipeline_d.h index a3e6d2e79d4..a4db94f74b4 100644 --- a/src/mongo/db/pipeline/pipeline_d.h +++ b/src/mongo/db/pipeline/pipeline_d.h @@ -61,10 +61,14 @@ struct PlanSummaryStats; class PipelineD { public: /** - * If the first stage in the pipeline does not generate its own output documents, attaches a - * cursor document source to the front of the pipeline which will output documents from the - * collection to feed into the pipeline. - * + * This callback function is called to attach a query PlanExecutor to the given Pipeline by + * creating a specific DocumentSourceCursor stage using the provided PlanExecutor, and adding + * the new stage to the pipeline. + */ + using AttachExecutorCallback = std::function<void( + Collection*, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>, Pipeline*)>; + + /** * This method looks for early pipeline stages that can be folded into the underlying * PlanExecutor, and removes those stages from the pipeline when they can be absorbed by the * PlanExecutor. For example, an early $match can be removed and replaced with a @@ -73,29 +77,45 @@ public: * Callers must take care to ensure that 'nss' is locked in at least IS-mode. * * When not null, 'aggRequest' provides access to pipeline command options such as hint. + * + * The 'collection' parameter is optional and can be passed as 'nullptr'. + * + * This method will not add a $cursor stage to the pipeline, but will create a PlanExecutor and + * a callback function. The executor and the callback can later be used to create the $cursor + * stage and add it to the pipeline by calling 'attachInnerQueryExecutorToPipeline()' method. + * If the pipeline doesn't require a $cursor stage, the plan executor will be returned as + * 'nullptr'. */ - static void prepareCursorSource(Collection* collection, - const NamespaceString& nss, - const AggregationRequest* aggRequest, - Pipeline* pipeline); + static std::pair<AttachExecutorCallback, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> + buildInnerQueryExecutor(Collection* collection, + const NamespaceString& nss, + const AggregationRequest* aggRequest, + Pipeline* pipeline); /** - * Prepare a generic DocumentSourceCursor for 'pipeline'. + * Completes creation of the $cursor stage using the given callback pair obtained by calling + * 'buildInnerQueryExecutor()' method. If the callback doesn't hold a valid PlanExecutor, the + * method does nothing. Otherwise, a new $cursor stage is created using the given PlanExecutor, + * and added to the pipeline. The 'collection' parameter is optional and can be passed as + * 'nullptr'. */ - static void prepareGenericCursorSource(Collection* collection, - const NamespaceString& nss, - const AggregationRequest* aggRequest, - Pipeline* pipeline); + static void attachInnerQueryExecutorToPipeline( + Collection* collection, + AttachExecutorCallback attachExecutorCallback, + std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec, + Pipeline* pipeline); /** - * Prepare a special DocumentSourceGeoNearCursor for 'pipeline'. Unlike - * 'prepareGenericCursorSource()', throws if 'collection' does not exist, as the $geoNearCursor - * requires a 2d or 2dsphere index. + * This method combines 'buildInnerQueryExecutor()' and 'attachInnerQueryExecutorToPipeline()' + * into a single call to support auto completion of the cursor stage creation process. Can be + * used when the executor attachment phase doesn't need to be deferred and the $cursor stage + * can be created right after buiding the executor. */ - static void prepareGeoNearCursorSource(Collection* collection, - const NamespaceString& nss, - const AggregationRequest* aggRequest, - Pipeline* pipeline); + static void buildAndAttachInnerQueryExecutorToPipeline(Collection* collection, + const NamespaceString& nss, + const AggregationRequest* aggRequest, + Pipeline* pipeline); + static std::string getPlanSummaryStr(const Pipeline* pipeline); @@ -107,6 +127,27 @@ private: PipelineD(); // does not exist: prevent instantiation /** + * Build a PlanExecutor and prepare callback to create a generic DocumentSourceCursor for + * the 'pipeline'. + */ + static std::pair<AttachExecutorCallback, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> + buildInnerQueryExecutorGeneric(Collection* collection, + const NamespaceString& nss, + const AggregationRequest* aggRequest, + Pipeline* pipeline); + + /** + * Build a PlanExecutor and prepare a callback to create a special DocumentSourceGeoNearCursor + * for the 'pipeline'. Unlike 'buildInnerQueryExecutorGeneric()', throws if 'collection' does + * not exist, as the $geoNearCursor requires a 2d or 2dsphere index. + */ + static std::pair<AttachExecutorCallback, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> + buildInnerQueryExecutorGeoNear(Collection* collection, + const NamespaceString& nss, + const AggregationRequest* aggRequest, + Pipeline* pipeline); + + /** * Creates a PlanExecutor to be used in the initial cursor source. If the query system can use * an index to provide a more efficient sort or projection, the sort and/or projection will be * incorporated into the PlanExecutor. diff --git a/src/mongo/db/pipeline/process_interface_standalone.cpp b/src/mongo/db/pipeline/process_interface_standalone.cpp index db3d4c9f08e..58237a83689 100644 --- a/src/mongo/db/pipeline/process_interface_standalone.cpp +++ b/src/mongo/db/pipeline/process_interface_standalone.cpp @@ -384,7 +384,8 @@ unique_ptr<Pipeline, PipelineDeleter> MongoInterfaceStandalone::attachCursorSour Date_t::max(), AutoStatsTracker::LogMode::kUpdateTop); - PipelineD::prepareCursorSource(autoColl->getCollection(), expCtx->ns, nullptr, pipeline.get()); + PipelineD::buildAndAttachInnerQueryExecutorToPipeline( + autoColl->getCollection(), expCtx->ns, nullptr, pipeline.get()); // Optimize again, since there may be additional optimizations that can be done after adding // the initial cursor stage. diff --git a/src/mongo/db/query/canonical_query.cpp b/src/mongo/db/query/canonical_query.cpp index fa2f522d6de..cf6fb23c661 100644 --- a/src/mongo/db/query/canonical_query.cpp +++ b/src/mongo/db/query/canonical_query.cpp @@ -156,6 +156,7 @@ StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::canonicalize( newExpCtx = expCtx; invariant(CollatorInterface::collatorsMatch(collator.get(), expCtx->getCollator())); } + StatusWithMatchExpression statusWithMatcher = MatchExpressionParser::parse( qr->getFilter(), newExpCtx, extensionsCallback, allowedFeatures); if (!statusWithMatcher.isOK()) { @@ -168,6 +169,7 @@ StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::canonicalize( Status initStatus = cq->init(opCtx, + std::move(newExpCtx), std::move(qr), parsingCanProduceNoopMatchNodes(extensionsCallback, allowedFeatures), std::move(me), @@ -203,6 +205,7 @@ StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::canonicalize( // Make the CQ we'll hopefully return. std::unique_ptr<CanonicalQuery> cq(new CanonicalQuery()); Status initStatus = cq->init(opCtx, + nullptr, // no expression context std::move(qr), baseQuery.canHaveNoopMatchNodes(), root->shallowClone(), @@ -215,10 +218,12 @@ StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::canonicalize( } Status CanonicalQuery::init(OperationContext* opCtx, + boost::intrusive_ptr<ExpressionContext> expCtx, std::unique_ptr<QueryRequest> qr, bool canHaveNoopMatchNodes, std::unique_ptr<MatchExpression> root, std::unique_ptr<CollatorInterface> collator) { + _expCtx = expCtx; _qr = std::move(qr); _collator = std::move(collator); diff --git a/src/mongo/db/query/canonical_query.h b/src/mongo/db/query/canonical_query.h index 121d5eed306..366265add3c 100644 --- a/src/mongo/db/query/canonical_query.h +++ b/src/mongo/db/query/canonical_query.h @@ -186,11 +186,14 @@ private: CanonicalQuery() {} Status init(OperationContext* opCtx, + boost::intrusive_ptr<ExpressionContext> expCtx, std::unique_ptr<QueryRequest> qr, bool canHaveNoopMatchNodes, std::unique_ptr<MatchExpression> root, std::unique_ptr<CollatorInterface> collator); + boost::intrusive_ptr<ExpressionContext> _expCtx; + std::unique_ptr<QueryRequest> _qr; // _root points into _qr->getFilter() diff --git a/src/mongo/db/query/explain.cpp b/src/mongo/db/query/explain.cpp index fc6c4fc8581..0b6ac374326 100644 --- a/src/mongo/db/query/explain.cpp +++ b/src/mongo/db/query/explain.cpp @@ -637,6 +637,7 @@ void Explain::getWinningPlanStats(const PlanExecutor* exec, BSONObjBuilder* bob) // static void Explain::generatePlannerInfo(PlanExecutor* exec, const Collection* collection, + BSONObj extraInfo, BSONObjBuilder* out) { CanonicalQuery* query = exec->getCanonicalQuery(); @@ -687,6 +688,10 @@ void Explain::generatePlannerInfo(PlanExecutor* exec, plannerBob.append("planCacheKey", unsignedIntToFixedLengthHex(*planCacheKeyHash)); } + if (!extraInfo.isEmpty()) { + plannerBob.appendElements(extraInfo); + } + BSONObjBuilder winningPlanBob(plannerBob.subobjStart("winningPlan")); const auto winnerStats = getWinningPlanStatsTree(exec); statsToBSON(*winnerStats.get(), &winningPlanBob, ExplainOptions::Verbosity::kQueryPlanner); @@ -830,13 +835,14 @@ void Explain::explainStages(PlanExecutor* exec, ExplainOptions::Verbosity verbosity, Status executePlanStatus, PlanStageStats* winningPlanTrialStats, + BSONObj extraInfo, BSONObjBuilder* out) { // // Use the stats trees to produce explain BSON. // if (verbosity >= ExplainOptions::Verbosity::kQueryPlanner) { - generatePlannerInfo(exec, collection, out); + generatePlannerInfo(exec, collection, extraInfo, out); } if (verbosity >= ExplainOptions::Verbosity::kExecStats) { @@ -867,6 +873,7 @@ void Explain::explainPipelineExecutor(PlanExecutor* exec, void Explain::explainStages(PlanExecutor* exec, const Collection* collection, ExplainOptions::Verbosity verbosity, + BSONObj extraInfo, BSONObjBuilder* out) { auto winningPlanTrialStats = Explain::getWinningPlanTrialStats(exec); @@ -883,7 +890,13 @@ void Explain::explainStages(PlanExecutor* exec, } } - explainStages(exec, collection, verbosity, executePlanStatus, winningPlanTrialStats.get(), out); + explainStages(exec, + collection, + verbosity, + executePlanStatus, + winningPlanTrialStats.get(), + extraInfo, + out); generateServerInfo(out); } diff --git a/src/mongo/db/query/explain.h b/src/mongo/db/query/explain.h index 678b3166345..8317fc50cfc 100644 --- a/src/mongo/db/query/explain.h +++ b/src/mongo/db/query/explain.h @@ -58,6 +58,8 @@ public: * * The explain information is generated with the level of detail specified by 'verbosity'. * + * The 'extraInfo' parameter specifies additional information to include into the output. + * * Does not take ownership of its arguments. * * The caller should hold at least an IS lock on the collection the that the query runs on, @@ -69,6 +71,7 @@ public: static void explainStages(PlanExecutor* exec, const Collection* collection, ExplainOptions::Verbosity verbosity, + BSONObj extraInfo, BSONObjBuilder* out); /** * Adds "queryPlanner" and "executionStats" (if requested in verbosity) fields to 'out'. Unlike @@ -78,6 +81,7 @@ public: * - 'collection' is the relevant collection. The caller should hold at least an IS lock on the * collection which the query ran on, even 'collection' is nullptr. * - 'verbosity' is the verbosity level of the explain. + * - 'extraInfo' specifies additional information to include into the output. * - 'executePlanStatus' is the status returned after executing the query (Status::OK if the * query wasn't executed). * - 'winningPlanTrialStats' is the stats of the winning plan during the trial period. May be @@ -89,6 +93,7 @@ public: ExplainOptions::Verbosity verbosity, Status executePlanStatus, PlanStageStats* winningPlanTrialStats, + BSONObj extraInfo, BSONObjBuilder* out); /** @@ -198,10 +203,12 @@ private: * - 'exec' is the stage tree for the operation being explained. * - 'collection' is the collection used in the operation. The caller should hold an IS lock on * the collection which the query is for, even if 'collection' is nullptr. + * - 'extraInfo' specifies additional information to include into the output. * - 'out' is a builder for the explain output. */ static void generatePlannerInfo(PlanExecutor* exec, const Collection* collection, + BSONObj extraInfo, BSONObjBuilder* out); /** diff --git a/src/mongo/db/query/find.cpp b/src/mongo/db/query/find.cpp index 5072509ffc3..7133be046d5 100644 --- a/src/mongo/db/query/find.cpp +++ b/src/mongo/db/query/find.cpp @@ -619,8 +619,11 @@ std::string runQuery(OperationContext* opCtx, bb.skip(sizeof(QueryResult::Value)); BSONObjBuilder explainBob; - Explain::explainStages( - exec.get(), collection, ExplainOptions::Verbosity::kExecAllPlans, &explainBob); + Explain::explainStages(exec.get(), + collection, + ExplainOptions::Verbosity::kExecAllPlans, + BSONObj(), + &explainBob); // Add the resulting object to the return buffer. BSONObj explainObj = explainBob.obj(); diff --git a/src/mongo/dbtests/query_stage_multiplan.cpp b/src/mongo/dbtests/query_stage_multiplan.cpp index 0bc35c95bad..6e664bb0b3b 100644 --- a/src/mongo/dbtests/query_stage_multiplan.cpp +++ b/src/mongo/dbtests/query_stage_multiplan.cpp @@ -505,7 +505,7 @@ TEST_F(QueryStageMultiPlanTest, MPSExplainAllPlans) { BSONObjBuilder bob; Explain::explainStages( - exec.get(), ctx.getCollection(), ExplainOptions::Verbosity::kExecAllPlans, &bob); + exec.get(), ctx.getCollection(), ExplainOptions::Verbosity::kExecAllPlans, BSONObj(), &bob); BSONObj explained = bob.done(); ASSERT_EQ(explained["executionStats"]["nReturned"].Int(), nDocs); diff --git a/src/mongo/s/query/cluster_aggregate.cpp b/src/mongo/s/query/cluster_aggregate.cpp index 9149ae33530..2cbb292adc8 100644 --- a/src/mongo/s/query/cluster_aggregate.cpp +++ b/src/mongo/s/query/cluster_aggregate.cpp @@ -270,9 +270,28 @@ Status appendExplainResults(sharded_agg_helpers::DispatchShardPipelineResults&& BSONObjBuilder shardExplains(result->subobjStart("shards")); for (const auto& shardResult : dispatchResults.remoteExplainOutput) { invariant(shardResult.shardHostAndPort); - shardExplains.append(shardResult.shardId.toString(), - BSON("host" << shardResult.shardHostAndPort->toString() << "stages" - << shardResult.swResponse.getValue().data["stages"])); + + uassertStatusOK(shardResult.swResponse.getStatus()); + uassertStatusOK(getStatusFromCommandResult(shardResult.swResponse.getValue().data)); + + auto shardId = shardResult.shardId.toString(); + const auto& data = shardResult.swResponse.getValue().data; + BSONObjBuilder explain(shardExplains.subobjStart(shardId)); + explain << "host" << shardResult.shardHostAndPort->toString(); + if (auto stagesElement = data["stages"]) { + explain << "stages" << stagesElement; + } else { + auto queryPlannerElement = data["queryPlanner"]; + uassert(51157, + str::stream() << "Malformed explain response received from shard " << shardId + << ": " + << data.toString(), + queryPlannerElement); + explain << "queryPlanner" << queryPlannerElement; + if (auto executionStatsElement = data["executionStats"]) { + explain << "executionStats" << executionStatsElement; + } + } } return Status::OK(); @@ -812,7 +831,6 @@ Status ClusterAggregate::runAggregate(OperationContext* opCtx, // If the operation is an explain, then we verify that it succeeded on all targeted shards, // write the results to the output builder, and return immediately. if (expCtx->explain) { - uassertAllShardsSupportExplain(shardDispatchResults.remoteExplainOutput); return appendExplainResults(std::move(shardDispatchResults), expCtx, result); } @@ -853,25 +871,6 @@ Status ClusterAggregate::runAggregate(OperationContext* opCtx, privileges); } -void ClusterAggregate::uassertAllShardsSupportExplain( - const std::vector<AsyncRequestsSender::Response>& shardResults) { - for (const auto& result : shardResults) { - auto status = result.swResponse.getStatus(); - if (status.isOK()) { - status = getStatusFromCommandResult(result.swResponse.getValue().data); - } - uassert(17403, - str::stream() << "Shard " << result.shardId.toString() << " failed: " - << causedBy(status), - status.isOK()); - - uassert(17404, - str::stream() << "Shard " << result.shardId.toString() - << " does not support $explain", - result.swResponse.getValue().data.hasField("stages")); - } -} - Status ClusterAggregate::aggPassthrough(OperationContext* opCtx, const Namespaces& namespaces, const ShardId& shardId, diff --git a/src/mongo/s/query/cluster_aggregate.h b/src/mongo/s/query/cluster_aggregate.h index 71ebcd3b3ae..afa774cac37 100644 --- a/src/mongo/s/query/cluster_aggregate.h +++ b/src/mongo/s/query/cluster_aggregate.h @@ -105,9 +105,6 @@ public: unsigned numberRetries = 0); private: - static void uassertAllShardsSupportExplain( - const std::vector<AsyncRequestsSender::Response>& shardResults); - static Status aggPassthrough(OperationContext*, const Namespaces&, const ShardId&, |