diff options
19 files changed, 995 insertions, 308 deletions
diff --git a/jstests/core/timeseries/timeseries_geonear.js b/jstests/core/timeseries/timeseries_geonear.js index 67747b23a11..01323610035 100644 --- a/jstests/core/timeseries/timeseries_geonear.js +++ b/jstests/core/timeseries/timeseries_geonear.js @@ -54,18 +54,21 @@ let agg = tsColl.aggregate([{ }]); assert.eq(agg.itcount(), nMeasurements); -/* TODO (SERVER-58443): enable these tests once they work -assert.commandWorked(tsColl.find({"tags.distance": {$geoNear: [0, 0]}}).itcount()); -assert.commandWorked(tsColl.find({"tags.distance": {$near: [0, 0]}}).itcount()); -assert.commandWorked(tsColl - .find({ - "tags.distance": { - $nearSphere: { - $geometry: {type: "Point", coordinates: [-73.9667, 40.78]}, - $minDistance: 10, - $maxDistance: 20 - } - } - }) - .itcount());*/ +if (TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { + tsColl + .find({ + "tags.loc": { + $nearSphere: { + $geometry: {type: "Point", coordinates: [-73.9667, 40.78]}, + $minDistance: 10, + $maxDistance: 20 + } + } + }) + .itcount(); + + tsColl.createIndex({'tags.loc': '2d'}); + tsColl.find({"tags.loc": {$geoNear: [0, 0]}}).itcount(); + tsColl.find({"tags.loc": {$near: [0, 0]}}).itcount(); +} })(); diff --git a/jstests/core/timeseries/timeseries_text_geonear_disallowed.js b/jstests/core/timeseries/timeseries_text_geonear_disallowed.js index e38d2ca20cb..ad36b318533 100644 --- a/jstests/core/timeseries/timeseries_text_geonear_disallowed.js +++ b/jstests/core/timeseries/timeseries_text_geonear_disallowed.js @@ -23,7 +23,7 @@ if (!TimeseriesTest.timeseriesCollectionsEnabled(db.getMongo())) { const timeFieldName = "time"; const metaFieldName = "tags"; -const testDB = db.getSiblingDB(jsTestName()); +const testDB = db.getSiblingDB('timeseries_text_geonear_disallowed' /*jsTestName()*/); assert.commandWorked(testDB.dropDatabase()); const tsColl = testDB.getCollection("ts_point_data"); @@ -42,29 +42,39 @@ for (let i = 0; i < nMeasurements; i++) { assert.commandWorked(tsColl.insert(docToInsert)); } -// Test that $geoNear fails cleanly because it cannot be issued against a time-series collection. -assert.commandFailedWithCode( - assert.throws(() => tsColl.find({"tags.distance": {$geoNear: [0, 0]}}).itcount()), 5626500); +// Test that geo operators fail cleanly. +if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { + // $geoNear (alias for $near) + assert.commandFailedWithCode( + assert.throws(() => tsColl.find({"tags.distance": {$geoNear: [0, 0]}}).itcount()), + // "$geoNear, $near, and $nearSphere are not allowed in this context" + 5626500); -// Test that unimplemented match exprs on time-series collections fail cleanly. -// $near -assert.commandFailedWithCode( - assert.throws(() => tsColl.find({"tags.distance": {$near: [0, 0]}}).itcount()), 5626500); + // $near + assert.commandFailedWithCode( + assert.throws(() => tsColl.find({"tags.distance": {$near: [0, 0]}}).itcount()), + // "$geoNear, $near, and $nearSphere are not allowed in this context" + 5626500); -// $nearSphere -assert.commandFailedWithCode( - assert.throws(() => tsColl - .find({ - "tags.distance": { - $nearSphere: { - $geometry: {type: "Point", coordinates: [-73.9667, 40.78]}, - $minDistance: 10, - $maxDistance: 20 + // $nearSphere + assert.commandFailedWithCode( + assert.throws(() => tsColl + .find({ + "tags.distance": { + $nearSphere: { + $geometry: + {type: "Point", coordinates: [-73.9667, 40.78]}, + $minDistance: 10, + $maxDistance: 20 + } } - } - }) - .itcount()), - 5626500); + }) + .itcount()), + // "$geoNear, $near, and $nearSphere are not allowed in this context" + 5626500); +} + +// Test that unimplemented match exprs on time-series collections fail cleanly. // $text // Text indices are disallowed on collections clustered by _id. diff --git a/jstests/core/views/geo.js b/jstests/core/views/geo.js new file mode 100644 index 00000000000..4c646bb3629 --- /dev/null +++ b/jstests/core/views/geo.js @@ -0,0 +1,360 @@ +/** + * Tests that $near and $nearSphere work on an identity view. This is a prerequisite for + * supporting $near and $nearSphere on time-series views. + * + * @tags [ + * requires_fcv_51, + * ] + */ +(function() { +"use strict"; + +load('jstests/libs/analyze_plan.js'); +load("jstests/core/timeseries/libs/timeseries.js"); + +if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { + jsTestLog("Skipping test because the time-series collection feature flag is disabled"); + return; +} + +// Sharding passthrough suites override getCollection() implicitly shard the collection. +// This fails when the view already exists (from a previous run of this test), because you can't +// shard a view. To work around this, drop the view before we even call getCollection(). +const collName = 'views_geo_collection'; +const viewName = 'views_geo_view'; +// Sharding passthrough suites also override the drop() method, so use runCommand() to bypass it. +assert.commandWorkedOrFailedWithCode(db.runCommand({'drop': collName}), + [ErrorCodes.NamespaceNotFound]); +assert.commandWorkedOrFailedWithCode(db.runCommand({'drop': viewName}), + [ErrorCodes.NamespaceNotFound]); +// Allow the collection to be implicitly sharded. +const coll = db.getCollection(collName); +// Allow the view to be implicitly sharded. +const view = db.getCollection(viewName); +// Drop the view again so we can create it, as a view. +assert.commandWorkedOrFailedWithCode(db.runCommand({'drop': viewName}), + [ErrorCodes.NamespaceNotFound]); + +// Create some documents with 2D coordinates. +// The coordinates are [0, 0] ... [0, 90]. On a plane, they lie on the positive y axis, +// while on a sphere they lie on the prime meridian, from the equator to the north pole. +const numDocs = 11; +const step = 90 / (numDocs - 1); +coll.drop(); +coll.insert(Array.from({length: numDocs}, (_, i) => ({_id: i, loc: [0, step * i]}))); +assert.eq(numDocs, coll.aggregate([]).itcount()); +// Make sure the last doc lands exactly on the north pole. +assert.eq(coll.find().sort({_id: -1}).limit(1).next(), {_id: numDocs - 1, loc: [0, 90]}); +// Define an identity view. +db.createView(view.getName(), coll.getName(), []); +// Make sure the view contents match the collection. +assert.eq(numDocs, view.aggregate([]).itcount()); +assert.sameMembers(coll.aggregate([]).toArray(), view.aggregate([]).toArray()); + +// There are several choices in how the query is written: +// - $near or $nearSphere +// - [x, y] or GeoJSON +// - minDistance specified or not +// - maxDistance specified or not +// And there are many possible outcomes: +// - Error, or not. +// - Points ordered by spherical distance, or planar. +// - minDistance/maxDistance interpreted as which units: +// - degrees? +// - radians? +// - meters? +// The outcome may depend on which indexes are present. + +// Sets up and tears down indexes around a given function. +function withIndexes(indexKeys, func) { + if (indexKeys.length) { + assert.commandWorked(coll.createIndexes(indexKeys)); + } + try { + return func(); + } finally { + for (const key of indexKeys) { + assert.commandWorked(coll.dropIndex(key)); + } + } +} + +// Runs the cursor and asserts that it fails because a required geospatial index was missing. +function assertNoIndex(cursor) { + const err = assert.throws(() => cursor.next()); + assert.contains(err.code, [ErrorCodes.NoQueryExecutionPlans, ErrorCodes.IndexNotFound], err); + assert(err.message.includes("$geoNear requires a 2d or 2dsphere index, but none were found") || + err.message.includes("unable to find index for $geoNear query"), + err); + return err; +} + +// Runs the cursor and returns an array of just the Y coordinate for each doc. +function getY(cursor) { + return cursor.toArray().map(doc => doc.loc[1]); +} + +// Runs the cursor and asserts that all the points appear, with those closer to [0, 0] first. +function assertOriginFirst(cursor) { + assert.docEq(getY(cursor), + [0, 9, 18, 27, 36, 45, 54, 63, 72, 81, 90], + 'Expected points closer to [0, 0] to be first.'); +} + +// Runs the cursor and asserts that all the points appear, with those closer to [0, 90] first. +function assertOriginLast(cursor) { + assert.docEq(getY(cursor), + [90, 81, 72, 63, 54, 45, 36, 27, 18, 9, 0], + 'Expected points closer to [0, 0] to be last.'); +} + +function degToRad(deg) { + // 180 degrees = pi radians + return deg * (Math.PI / 180); +} +function degToMeters(deg) { + // Earth's circumference is roughly 40,000 km. + // 360 degrees = 1 circumference + const earthCircumference = 40074784; + return deg * (earthCircumference / 360); +} + +// Abbreviation for creating a GeoJSON point. +function geoJSON(coordinates) { + return {type: "Point", coordinates}; +} + +// Test how $near/$nearSphere is interpreted: $nearSphere always means a spherical-distance query, +// but $near can mean either planar or spherical depending on how the query point is written. +// Also test which combinations are allowed with which indexes. +withIndexes([], () => { + // With no geospatial indexes, $near and $nearSphere should always fail. + assertNoIndex(coll.find({loc: {$near: [0, 0]}})); + assertNoIndex(coll.find({loc: {$near: {$geometry: geoJSON([0, 0])}}})); + assertNoIndex(coll.find({loc: {$nearSphere: [0, 0]}})); + assertNoIndex(coll.find({loc: {$nearSphere: {$geometry: geoJSON([0, 0])}}})); + + assertNoIndex(view.find({loc: {$near: [0, 0]}})); + assertNoIndex(view.find({loc: {$near: {$geometry: geoJSON([0, 0])}}})); + assertNoIndex(view.find({loc: {$nearSphere: [0, 0]}})); + assertNoIndex(view.find({loc: {$nearSphere: {$geometry: geoJSON([0, 0])}}})); +}); +withIndexes([{loc: '2d'}], () => { + // Queries written as GeoJSON mean spherical distance, so they fail without a '2dsphere' index. + assertNoIndex(coll.find({loc: {$near: {$geometry: geoJSON([180, 0])}}})); + assertNoIndex(view.find({loc: {$near: {$geometry: geoJSON([180, 0])}}})); + + assertNoIndex(coll.find({loc: {$nearSphere: {$geometry: geoJSON([180, 0])}}})); + assertNoIndex(view.find({loc: {$nearSphere: {$geometry: geoJSON([180, 0])}}})); + + // $near [x, y] means planar distance, so it succeeds. + assertOriginFirst(coll.find({loc: {$near: [180, 0]}})); + assertOriginFirst(view.find({loc: {$near: [180, 0]}})); + + // Surprisingly, $nearSphere can use a '2d' index to do a spherical-distance query. + // Also surprisingly, this only works with the [x, y] query syntax, not with GeoJSON. + assertOriginLast(coll.find({loc: {$nearSphere: [180, 0]}})); + assertOriginLast(view.find({loc: {$nearSphere: [180, 0]}})); +}); +withIndexes([{loc: '2dsphere'}], () => { + // When '2dsphere' is available but not '2d', we can only satisfy spherical-distance queries. + + // $near [x, y] fails because it means planar distance. + assertNoIndex(coll.find({loc: {$near: [180, 0]}})); + assertNoIndex(view.find({loc: {$near: [180, 0]}})); + + // $near with GeoJSON means spherical distance, so it succeeds. + assertOriginLast(coll.find({loc: {$near: {$geometry: geoJSON([180, 0])}}})); + assertOriginLast(view.find({loc: {$near: {$geometry: geoJSON([180, 0])}}})); + + // $nearSphere succeeds with either syntax. + assertOriginLast(coll.find({loc: {$nearSphere: [180, 0]}})); + assertOriginLast(view.find({loc: {$nearSphere: [180, 0]}})); + + assertOriginLast(coll.find({loc: {$nearSphere: {$geometry: geoJSON([180, 0])}}})); + assertOriginLast(view.find({loc: {$nearSphere: {$geometry: geoJSON([180, 0])}}})); +}); + +// For the rest of the tests, both index types are available. +// With both types of indexes available, all the queries should succeed. +coll.createIndexes([{loc: '2d'}, {loc: '2dsphere'}]); + +// $near [x, y] means planar. +assertOriginFirst(coll.find({loc: {$near: [180, 0]}})); +// GeoJSON and/or $nearSphere means spherical. +assertOriginLast(coll.find({loc: {$near: {$geometry: geoJSON([180, 0])}}})); +assertOriginLast(coll.find({loc: {$nearSphere: [180, 0]}})); +assertOriginLast(coll.find({loc: {$nearSphere: {$geometry: geoJSON([180, 0])}}})); + +// $near [x, y] means planar distance. +assertOriginFirst(view.find({loc: {$near: [180, 0]}})); +// GeoJSON and/or $nearSphere means spherical. +assertOriginLast(view.find({loc: {$near: {$geometry: geoJSON([180, 0])}}})); +assertOriginLast(view.find({loc: {$nearSphere: [180, 0]}})); +assertOriginLast(view.find({loc: {$nearSphere: {$geometry: geoJSON([180, 0])}}})); + +// Test how minDistance / maxDistance are interpreted. +{ + // In a planar-distance query, min/maxDistance make sense even without thinking about units + // (degrees vs radians). You can think of the min/maxDistance as being the same units as the + // [x, y] coordinates in the collection: pixels, miles; it doesn't matter. + assert.eq(getY(coll.find({loc: {$near: [0, 0], $minDistance: 9, $maxDistance: 36}})), + [9, 18, 27, 36]); + assert.eq(getY(view.find({loc: {$near: [0, 0], $minDistance: 9, $maxDistance: 36}})), + [9, 18, 27, 36]); + + // In a spherical-distance query, units do matter. The points in the collection are interpreted + // as degrees [longitude, latitude]. + + // Queries written as GeoJSON use meters. + assert.eq(getY(coll.find({ + loc: { + $near: { + $geometry: geoJSON([0, 0]), + $minDistance: degToMeters(9), + $maxDistance: degToMeters(36) + 1, + } + } + })), + [9, 18, 27, 36]); + assert.eq(getY(coll.find({ + loc: { + $nearSphere: { + $geometry: geoJSON([0, 0]), + $minDistance: degToMeters(9), + $maxDistance: degToMeters(36) + 1, + } + } + })), + [9, 18, 27, 36]); + assert.eq(getY(view.find({ + loc: { + $near: { + $geometry: geoJSON([0, 0]), + $minDistance: degToMeters(9), + $maxDistance: degToMeters(36) + 1, + } + } + })), + [9, 18, 27, 36]); + assert.eq(getY(view.find({ + loc: { + $nearSphere: { + $geometry: geoJSON([0, 0]), + $minDistance: degToMeters(9), + $maxDistance: degToMeters(36) + 1, + } + } + })), + [9, 18, 27, 36]); + + // $nearSphere [x, y] uses radians. + assert.eq( + getY(coll.find( + {loc: {$nearSphere: [0, 0], $minDistance: degToRad(9), $maxDistance: degToRad(36)}})), + [9, 18, 27, 36]); + assert.eq( + getY(view.find( + {loc: {$nearSphere: [0, 0], $minDistance: degToRad(9), $maxDistance: degToRad(36)}})), + [9, 18, 27, 36]); +} + +// Test a few more odd examples: just confirm that the view and collection behave the same way. + +function example(predicate, limit = 0) { + // .find().limit(0) is interpreted as no limit. + assert.eq(coll.find(predicate).limit(limit).itcount(), + view.find(predicate).limit(limit).itcount(), + predicate); + assert.docEq(coll.find(predicate).limit(limit).toArray(), + view.find(predicate).limit(limit).toArray(), + predicate); +} + +// We want the order of results to be predictable: no ties. The documents all have integer +// coordinates, so if we avoid nice numbers like .0 or .5 we know every document has a different +// distance from the query point. +example({loc: {$near: [0, 2.2]}}); +example({loc: {$near: [0, 9.1]}}); + +// $near can be combined with another predicate. +example({loc: {$near: [0, 2.2]}, _id: {$ne: 2}}); + +// $near can have $minDistance and/or $maxDistance. +example({loc: {$near: [0, 30.1], $minDistance: 10}}); +example({loc: {$near: [0, 30.1], $maxDistance: 20}}); +example({loc: {$near: [0, 30.1], $minDistance: 10, $maxDistance: 20}}); + +// $near can have a limit. +example({loc: {$near: [0, 2.2]}}, 10); +example({loc: {$near: [0, 9.1]}}, 30); + +// Try combining several options: +// - another predicate +// - min/max distance +// - limit +example({loc: {$near: [0, 30.1], $minDistance: 10, $maxDistance: 30}, _id: {$ne: 2}}, 20); + +// .sort() throws away the $near order. +assert.docEq(coll.find({loc: {$near: [0, 100]}}).sort({'loc.1': 1}).limit(5).toArray(), + view.find({loc: {$near: [0, 100]}}).sort({'loc.1': 1}).limit(5).toArray()); + +// Test wrapping: insert a point at -179 degrees, and query it at +179. +// It should be within about 2 degrees. +const wrapTestDoc = { + _id: "wrap", + loc: [-179, 0] +}; +assert.commandWorked(coll.insert(wrapTestDoc)); + +// Test all 3 syntaxes on the collection, and on the view. +// These are all just alternative ways to specify the same spherical query. +// Although the results are consistent, the plan is slightly different when running on the view, +// because it runs as an aggregation. + +const assertNearSphereQuery = explain => { + assert(isQueryPlan(explain), explain); + assert(planHasStage(db, explain, 'GEO_NEAR_2DSPHERE'), explain); +}; +const assertNearSphereAgg = explain => { + assert(isAggregationPlan(explain), explain); + let stages = getAggPlanStages(explain, '$geoNearCursor'); + assert.neq(stages, [], explain); +}; + +let query = { + loc: { + $nearSphere: { + $geometry: geoJSON([+179, 0]), + $minDistance: degToMeters(1.9), + $maxDistance: degToMeters(2.1) + } + } +}; +assert.docEq(coll.find(query).toArray(), [wrapTestDoc]); +assert.docEq(view.find(query).toArray(), [wrapTestDoc]); +assertNearSphereQuery(coll.find(query).explain()); +assertNearSphereAgg(view.find(query).explain()); + +query = { + loc: { + $near: { + $geometry: geoJSON([+179, 0]), + $minDistance: degToMeters(1.9), + $maxDistance: degToMeters(2.1) + } + } +}; +assert.docEq(coll.find(query).toArray(), [wrapTestDoc]); +assert.docEq(view.find(query).toArray(), [wrapTestDoc]); +assertNearSphereQuery(coll.find(query).explain()); +assertNearSphereAgg(view.find(query).explain()); + +query = { + loc: {$nearSphere: [+179, 0], $minDistance: degToRad(1.9), $maxDistance: degToRad(2.1)} +}; +assert.docEq(coll.find(query).toArray(), [wrapTestDoc]); +assert.docEq(view.find(query).toArray(), [wrapTestDoc]); +assertNearSphereQuery(coll.find(query).explain()); +assertNearSphereAgg(view.find(query).explain()); +})();
\ No newline at end of file diff --git a/src/mongo/db/commands/find_cmd.cpp b/src/mongo/db/commands/find_cmd.cpp index 31faa7c7bb3..01bf2a8afbf 100644 --- a/src/mongo/db/commands/find_cmd.cpp +++ b/src/mongo/db/commands/find_cmd.cpp @@ -56,6 +56,7 @@ #include "mongo/db/stats/resource_consumption_metrics.h" #include "mongo/db/stats/server_read_concern_metrics.h" #include "mongo/db/storage/storage_engine.h" +#include "mongo/db/storage/storage_parameters_gen.h" #include "mongo/db/transaction_participant.h" #include "mongo/logv2/log.h" #include "mongo/rpc/get_status_from_command_result.h" @@ -119,7 +120,8 @@ std::unique_ptr<FindCommandRequest> parseCmdObjectToFindCommandRequest(Operation boost::intrusive_ptr<ExpressionContext> makeExpressionContext( OperationContext* opCtx, const FindCommandRequest& findCommand, - boost::optional<ExplainOptions::Verbosity> verbosity) { + boost::optional<ExplainOptions::Verbosity> verbosity, + bool isView) { std::unique_ptr<CollatorInterface> collator; if (!findCommand.getCollation().isEmpty()) { collator = uassertStatusOK(CollatorFactoryInterface::get(opCtx->getServiceContext()) @@ -153,9 +155,10 @@ boost::intrusive_ptr<ExpressionContext> makeExpressionContext( std::move(collator), nullptr, // mongoProcessInterface StringMap<ExpressionContext::ResolvedNamespace>{}, - boost::none, // uuid - findCommand.getLet(), // let - CurOp::get(opCtx)->dbProfileLevel() > 0 // mayDbProfile + boost::none, // uuid + findCommand.getLet(), // let + CurOp::get(opCtx)->dbProfileLevel() > 0, // mayDbProfile + isView // omitVariables ); expCtx->tempDir = storageGlobalParams.dbpath + "/_tmp"; expCtx->startExpressionCounters(); @@ -302,25 +305,24 @@ public: // Finish the parsing step by using the FindCommandRequest to create a CanonicalQuery. const ExtensionsCallbackReal extensionsCallback(opCtx, &nss); - auto expCtx = makeExpressionContext(opCtx, *findCommand, verbosity); const bool isExplain = true; - auto cq = uassertStatusOK( - CanonicalQuery::canonicalize(opCtx, - std::move(findCommand), - isExplain, - std::move(expCtx), - extensionsCallback, - MatchExpressionParser::kAllowAllSpecialFeatures)); - if (ctx->getView()) { + auto expCtx = + makeExpressionContext(opCtx, *findCommand, verbosity, true /* isView */); + auto cq = uassertStatusOK( + CanonicalQuery::canonicalize(opCtx, + std::move(findCommand), + isExplain, + std::move(expCtx), + extensionsCallback, + Pipeline::viewFindMatcherFeatures())); + // Relinquish locks. The aggregation command will re-acquire them. ctx.reset(); // Convert the find command into an aggregation using $match (and other stages, as // necessary), if possible. - const auto& findCommand = cq->getFindCommandRequest(); - auto viewAggregationCommand = - uassertStatusOK(query_request_helper::asAggregationCommand(findCommand)); + auto viewAggregationCommand = uassertStatusOK(asAggregationCommand(*cq)); auto viewAggCmd = OpMsgRequest::fromDBAndBody(_dbName, viewAggregationCommand).body; // Create the agg request equivalent of the find operation, with the explain @@ -347,6 +349,15 @@ public: return; } + auto expCtx = makeExpressionContext(opCtx, *findCommand, verbosity, false /* isView */); + auto cq = uassertStatusOK( + CanonicalQuery::canonicalize(opCtx, + std::move(findCommand), + isExplain, + std::move(expCtx), + extensionsCallback, + MatchExpressionParser::kAllowAllSpecialFeatures)); + // The collection may be NULL. If so, getExecutor() should handle it by returning an // execution tree with an EOFStage. const auto& collection = ctx->getCollection(); @@ -445,27 +456,27 @@ public: // Fill out curop information. beginQueryOp(opCtx, nss, _request.body); - // Finish the parsing step by using the FindCommandRequest to create a CanonicalQuery. const ExtensionsCallbackReal extensionsCallback(opCtx, &nss); - auto expCtx = makeExpressionContext(opCtx, *findCommand, boost::none /* verbosity */); - auto cq = uassertStatusOK( - CanonicalQuery::canonicalize(opCtx, - std::move(findCommand), - isExplain, - std::move(expCtx), - extensionsCallback, - MatchExpressionParser::kAllowAllSpecialFeatures)); if (ctx->getView()) { + auto expCtx = makeExpressionContext( + opCtx, *findCommand, boost::none /* verbosity */, true /* isView */); + + // Finish the parsing step by using the FindCommandRequest to create a + // CanonicalQuery. And then convert the find command into an aggregation using + // $match (and other stages, as necessary), if possible. + auto cq = uassertStatusOK( + CanonicalQuery::canonicalize(opCtx, + std::move(findCommand), + isExplain, + std::move(expCtx), + extensionsCallback, + Pipeline::viewFindMatcherFeatures())); + auto viewAggregationCommand = uassertStatusOK(asAggregationCommand(*cq)); + // Relinquish locks. The aggregation command will re-acquire them. ctx.reset(); - // Convert the find command into an aggregation using $match (and other stages, as - // necessary), if possible. - const auto& findCommand = cq->getFindCommandRequest(); - auto viewAggregationCommand = - uassertStatusOK(query_request_helper::asAggregationCommand(findCommand)); - BSONObj aggResult = CommandHelpers::runCommandDirectly( opCtx, OpMsgRequest::fromDBAndBody(_dbName, std::move(viewAggregationCommand))); auto status = getStatusFromCommandResult(aggResult); @@ -478,6 +489,17 @@ public: return; } + auto expCtx = makeExpressionContext( + opCtx, *findCommand, boost::none /* verbosity */, false /* isView */); + // Finish the parsing step by using the FindCommandRequest to create a CanonicalQuery. + auto cq = uassertStatusOK( + CanonicalQuery::canonicalize(opCtx, + std::move(findCommand), + isExplain, + std::move(expCtx), + extensionsCallback, + MatchExpressionParser::kAllowAllSpecialFeatures)); + const auto& collection = ctx->getCollection(); if (cq->getFindCommandRequest().getReadOnce()) { diff --git a/src/mongo/db/matcher/expression_geo.cpp b/src/mongo/db/matcher/expression_geo.cpp index 6db7f2003c5..e90ca0089d2 100644 --- a/src/mongo/db/matcher/expression_geo.cpp +++ b/src/mongo/db/matcher/expression_geo.cpp @@ -446,6 +446,7 @@ GeoNearMatchExpression::GeoNearMatchExpression(StringData path, bool GeoNearMatchExpression::matchesSingleElement(const BSONElement& e, MatchDetails* details) const { + tasserted(5844303, "GeoNearMatchExpression::matchesSingleElement() should never be called"); return true; } diff --git a/src/mongo/db/pipeline/document_source_geo_near.cpp b/src/mongo/db/pipeline/document_source_geo_near.cpp index e10b9e59b50..3c734f76b0f 100644 --- a/src/mongo/db/pipeline/document_source_geo_near.cpp +++ b/src/mongo/db/pipeline/document_source_geo_near.cpp @@ -38,6 +38,8 @@ #include "mongo/db/pipeline/lite_parsed_document_source.h" #include "mongo/logv2/log.h" +#include "mongo/db/storage/storage_parameters_gen.h" + namespace mongo { using boost::intrusive_ptr; @@ -62,7 +64,9 @@ Value DocumentSourceGeoNear::serialize(boost::optional<ExplainOptions::Verbosity result.setField("near", Value(coords)); } - result.setField("distanceField", Value(distanceField->fullPath())); + if (distanceField) { + result.setField("distanceField", Value(distanceField->fullPath())); + } if (maxDistance) { result.setField("maxDistance", Value(*maxDistance)); @@ -120,17 +124,26 @@ void DocumentSourceGeoNear::parseOptions(BSONObj options) { !options["num"]); uassert(50856, "$geoNear no longer supports the 'start' argument.", !options["start"]); - // The "near" and "distanceField" parameters are required. + // The "near" parameter is required. uassert(16605, "$geoNear requires a 'near' option as an Array", options["near"].isABSONObj()); // Array or Object (Object is deprecated) coordsIsArray = options["near"].type() == Array; coords = options["near"].embeddedObject().getOwned(); - uassert(16606, - "$geoNear requires a 'distanceField' option as a String", - options["distanceField"].type() == String); - distanceField.reset(new FieldPath(options["distanceField"].str())); + // "distanceField" is optional. + auto distElem = options["distanceField"]; + if (!feature_flags::gTimeseriesMetricIndexes.isEnabled( + serverGlobalParams.featureCompatibility)) { + // Except without this feature flag, "distanceField" is required. + uassert(16606, + "$geoNear requires a 'distanceField' option, as a String", + distElem.type() == String); + } + if (distElem) { + uassert(5844304, "$geoNear 'distanceField' must be a String", distElem.type() == String); + distanceField = FieldPath(distElem.str()); + } // The remaining fields are optional. if (auto maxDistElem = options["maxDistance"]) { @@ -226,10 +239,10 @@ bool DocumentSourceGeoNear::needsGeoNearPoint() const { } DepsTracker::State DocumentSourceGeoNear::getDependencies(DepsTracker* deps) const { - // TODO (SERVER-35424): Implement better dependency tracking. For example, 'distanceField' is - // produced by this stage, and we could inform the query system that it need not include it in - // its response. For now, assume that we require the entire document as well as the appropriate - // geoNear metadata. + // TODO (SERVER-35424): Implement better dependency tracking. For example, 'distanceField' (if + // specified) is produced by this stage, and we could inform the query system that it need not + // include it in its response. For now, assume that we require the entire document as well as + // the appropriate geoNear metadata. deps->setNeedsMetadata(DocumentMetadataFields::kGeoNearDist, true); deps->setNeedsMetadata(DocumentMetadataFields::kGeoNearPoint, needsGeoNearPoint()); @@ -243,7 +256,14 @@ DocumentSourceGeoNear::DocumentSourceGeoNear(const intrusive_ptr<ExpressionConte boost::optional<DocumentSource::DistributedPlanLogic> DocumentSourceGeoNear::distributedPlanLogic() { // {shardsStage, mergingStage, sortPattern} - return DistributedPlanLogic{this, nullptr, BSON(distanceField->fullPath() << 1)}; + return DistributedPlanLogic{ + this, + nullptr, + // The field name here apparently doesn't matter, because we always look + // in {$meta: sortKey}, which is implicitly set from {$meta: geoNearDistance} + // when available. + BSON("field_name_does_not_matter" << 1), + }; } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_geo_near.h b/src/mongo/db/pipeline/document_source_geo_near.h index d9fbc6e669a..3e941aecb97 100644 --- a/src/mongo/db/pipeline/document_source_geo_near.h +++ b/src/mongo/db/pipeline/document_source_geo_near.h @@ -93,8 +93,8 @@ public: /** * The field in which the computed distance will be stored. */ - FieldPath getDistanceField() const { - return *distanceField; + boost::optional<FieldPath> getDistanceField() const { + return distanceField; } /** @@ -152,10 +152,10 @@ private: void parseOptions(BSONObj options); // These fields describe the command to run. - // 'coords' and 'distanceField' are required; the rest are optional. + // 'coords' is required; the rest are optional. BSONObj coords; // "near" option, but near is a reserved keyword on windows bool coordsIsArray; - std::unique_ptr<FieldPath> distanceField; // Using unique_ptr because FieldPath can't be empty + boost::optional<FieldPath> distanceField; BSONObj query; bool spherical; boost::optional<double> maxDistance; diff --git a/src/mongo/db/pipeline/document_source_geo_near_cursor.cpp b/src/mongo/db/pipeline/document_source_geo_near_cursor.cpp index 8e6885567d0..0d3d5ad4a4f 100644 --- a/src/mongo/db/pipeline/document_source_geo_near_cursor.cpp +++ b/src/mongo/db/pipeline/document_source_geo_near_cursor.cpp @@ -54,7 +54,7 @@ boost::intrusive_ptr<DocumentSourceGeoNearCursor> DocumentSourceGeoNearCursor::c const CollectionPtr& collection, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec, const boost::intrusive_ptr<ExpressionContext>& expCtx, - FieldPath distanceField, + boost::optional<FieldPath> distanceField, boost::optional<FieldPath> locationField, double distanceMultiplier) { return {new DocumentSourceGeoNearCursor(collection, @@ -69,7 +69,7 @@ DocumentSourceGeoNearCursor::DocumentSourceGeoNearCursor( const CollectionPtr& collection, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec, const boost::intrusive_ptr<ExpressionContext>& expCtx, - FieldPath distanceField, + boost::optional<FieldPath> distanceField, boost::optional<FieldPath> locationField, double distanceMultiplier) : DocumentSourceCursor( @@ -94,7 +94,9 @@ Document DocumentSourceGeoNearCursor::transformDoc(Document&& objInput) const { << output.peek().toString()); const auto distance = output.peek().metadata().getGeoNearDistance() * _distanceMultiplier; - output.setNestedField(_distanceField, Value(distance)); + if (_distanceField) { + output.setNestedField(*_distanceField, Value(distance)); + } if (_locationField) { invariant( output.peek().metadata().hasGeoNearPoint(), diff --git a/src/mongo/db/pipeline/document_source_geo_near_cursor.h b/src/mongo/db/pipeline/document_source_geo_near_cursor.h index b3ddf9a834b..d0127667ac5 100644 --- a/src/mongo/db/pipeline/document_source_geo_near_cursor.h +++ b/src/mongo/db/pipeline/document_source_geo_near_cursor.h @@ -63,7 +63,7 @@ public: const CollectionPtr&, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>, const boost::intrusive_ptr<ExpressionContext>&, - FieldPath distanceField, + boost::optional<FieldPath> distanceField, boost::optional<FieldPath> locationField = boost::none, double distanceMultiplier = 1.0); @@ -73,7 +73,7 @@ private: DocumentSourceGeoNearCursor(const CollectionPtr&, std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>, const boost::intrusive_ptr<ExpressionContext>&, - FieldPath distanceField, + boost::optional<FieldPath> distanceField, boost::optional<FieldPath> locationField, double distanceMultiplier); @@ -85,7 +85,7 @@ private: Document transformDoc(Document&& obj) const override final; // The output field in which to store the computed distance. - FieldPath _distanceField; + boost::optional<FieldPath> _distanceField; // The output field to store the point that matched, if specified. boost::optional<FieldPath> _locationField; diff --git a/src/mongo/db/pipeline/expression_context.cpp b/src/mongo/db/pipeline/expression_context.cpp index 69698ab0289..86a20538afc 100644 --- a/src/mongo/db/pipeline/expression_context.cpp +++ b/src/mongo/db/pipeline/expression_context.cpp @@ -92,7 +92,8 @@ ExpressionContext::ExpressionContext( StringMap<ExpressionContext::ResolvedNamespace> resolvedNamespaces, boost::optional<UUID> collUUID, const boost::optional<BSONObj>& letParameters, - bool mayDbProfile) + bool mayDbProfile, + bool omitVariables) : explain(explain), fromMongos(fromMongos), needsMerge(needsMerge), @@ -127,6 +128,9 @@ ExpressionContext::ExpressionContext( } if (letParameters) variables.seedVariablesWithLetParameters(this, *letParameters); + + if (omitVariables) + variables = Variables{}; } ExpressionContext::ExpressionContext( diff --git a/src/mongo/db/pipeline/expression_context.h b/src/mongo/db/pipeline/expression_context.h index 398a162b283..7c8d61ea03d 100644 --- a/src/mongo/db/pipeline/expression_context.h +++ b/src/mongo/db/pipeline/expression_context.h @@ -123,6 +123,10 @@ public: * Constructs an ExpressionContext to be used for Pipeline parsing and evaluation. This version * requires finer-grained parameters but does not require an AggregateCommandRequest. * 'resolvedNamespaces' maps collection names (not full namespaces) to ResolvedNamespaces. + * + * 'omitVariables == true' means the ExpressionContext should not have any variables defined. + * This lets you parse and optimize a query without assuming that top-level variables such as + * NOW are known. */ ExpressionContext(OperationContext* opCtx, const boost::optional<ExplainOptions::Verbosity>& explain, @@ -138,7 +142,8 @@ public: StringMap<ExpressionContext::ResolvedNamespace> resolvedNamespaces, boost::optional<UUID> collUUID, const boost::optional<BSONObj>& letParameters = boost::none, - bool mayDbProfile = true); + bool mayDbProfile = true, + bool omitVariables = false); /** * Constructs an ExpressionContext suitable for use outside of the aggregation system, including diff --git a/src/mongo/db/pipeline/pipeline.cpp b/src/mongo/db/pipeline/pipeline.cpp index 90377d824f4..ae6ab01ae1b 100644 --- a/src/mongo/db/pipeline/pipeline.cpp +++ b/src/mongo/db/pipeline/pipeline.cpp @@ -45,6 +45,7 @@ #include "mongo/db/pipeline/lite_parsed_pipeline.h" #include "mongo/db/query/query_knobs_gen.h" #include "mongo/db/storage/storage_options.h" +#include "mongo/db/storage/storage_parameters_gen.h" #include "mongo/util/fail_point.h" #include "mongo/util/str.h" @@ -134,6 +135,17 @@ using StreamType = StageConstraints::StreamType; constexpr MatchExpressionParser::AllowedFeatureSet Pipeline::kAllowedMatcherFeatures; constexpr MatchExpressionParser::AllowedFeatureSet Pipeline::kGeoNearMatcherFeatures; +MatchExpressionParser::AllowedFeatureSet Pipeline::viewFindMatcherFeatures() { + if (serverGlobalParams.featureCompatibility.isVersionInitialized() && + feature_flags::gTimeseriesMetricIndexes.isEnabled( + serverGlobalParams.featureCompatibility)) { + return Pipeline::kGeoNearMatcherFeatures; + } + + return Pipeline::kAllowedMatcherFeatures; +} + + Pipeline::Pipeline(const intrusive_ptr<ExpressionContext>& pTheCtx) : pCtx(pTheCtx) {} Pipeline::Pipeline(SourceContainer stages, const intrusive_ptr<ExpressionContext>& expCtx) diff --git a/src/mongo/db/pipeline/pipeline.h b/src/mongo/db/pipeline/pipeline.h index b67bb035fa2..61fe21cb548 100644 --- a/src/mongo/db/pipeline/pipeline.h +++ b/src/mongo/db/pipeline/pipeline.h @@ -109,6 +109,14 @@ public: MatchExpressionParser::AllowedFeatures::kGeoNear; /** + * The match expression features allowed when running .find() on a view. + * + * The result can depend on feature flags or FCV. + * If FCV is not known yet, we err on the side of disallowing newer features. + */ + static MatchExpressionParser::AllowedFeatureSet viewFindMatcherFeatures(); + + /** * Parses a Pipeline from a vector of BSONObjs then invokes the optional 'validator' callback * with a reference to the newly created Pipeline. If no validator callback is given, this * method assumes that we're parsing a top-level pipeline. Throws an exception if it failed to diff --git a/src/mongo/db/query/canonical_query.cpp b/src/mongo/db/query/canonical_query.cpp index 87efc4cc259..2cfac7c0643 100644 --- a/src/mongo/db/query/canonical_query.cpp +++ b/src/mongo/db/query/canonical_query.cpp @@ -37,7 +37,9 @@ #include "mongo/db/commands/test_commands_enabled.h" #include "mongo/db/cst/cst_parser.h" #include "mongo/db/jsobj.h" +#include "mongo/db/matcher/expression_always_boolean.h" #include "mongo/db/matcher/expression_array.h" +#include "mongo/db/matcher/expression_geo.h" #include "mongo/db/namespace_string.h" #include "mongo/db/operation_context.h" #include "mongo/db/query/canonical_query_encoder.h" @@ -45,6 +47,7 @@ #include "mongo/db/query/indexability.h" #include "mongo/db/query/projection_parser.h" #include "mongo/db/query/query_planner_common.h" +#include "mongo/db/storage/storage_parameters_gen.h" namespace mongo { namespace { @@ -210,6 +213,46 @@ Status CanonicalQuery::init(OperationContext* opCtx, return status; } + { + // If there is a geo expression, extract it. + if (auto n = countNodes(_root.get(), MatchExpression::GEO_NEAR)) { + tassert(1234501, "isValidNormalized should have caught extra GEO_NEAR", n == 1); + + if (_root->matchType() == MatchExpression::GEO_NEAR) { + std::unique_ptr<MatchExpression> nonGeoNear{ + static_cast<MatchExpression*>(new AlwaysTrueMatchExpression())}; + std::unique_ptr<GeoNearMatchExpression> geoNear{ + static_cast<GeoNearMatchExpression*>(_root->shallowClone().release())}; + _splitGeoNear = SplitGeoNear{std::move(nonGeoNear), std::move(geoNear)}; + } else { + tassert(1234502, + "GEO_NEAR can only happen in a top-level AND", + _root->matchType() == MatchExpression::AND); + + auto nonGeoNear = _root->shallowClone(); + std::unique_ptr<GeoNearMatchExpression> geoNear; + + for (size_t i = 0; i < nonGeoNear->numChildren(); ++i) { + auto& children = *nonGeoNear->getChildVector(); + if (nonGeoNear->getChild(i)->matchType() == MatchExpression::GEO_NEAR) { + // Found it. + geoNear.reset(static_cast<GeoNearMatchExpression*>(children[i].release())); + children.erase(children.begin() + i); + break; + } + } + tassert(1234503, "Expected a GEO_NEAR in here", geoNear); + + // Removing one branch of an $and may have left us with only one remaining branch. + if (nonGeoNear->numChildren() == 1) { + nonGeoNear = std::move(nonGeoNear->getChildVector()->at(0)); + } + + _splitGeoNear = SplitGeoNear{std::move(nonGeoNear), std::move(geoNear)}; + } + } + } + // Validate the projection if there is one. if (!_findCommand->getProjection().isEmpty()) { try { @@ -288,6 +331,10 @@ void CanonicalQuery::setCollator(std::unique_ptr<CollatorInterface> collator) { // The collator associated with the match expression tree is now invalid, since we have reset // the collator owned by the ExpressionContext. _root->setCollator(collatorRaw); + if (_splitGeoNear) { + _splitGeoNear->geoNear->setCollator(collatorRaw); + _splitGeoNear->nonGeoNear->setCollator(collatorRaw); + } } // static @@ -331,6 +378,14 @@ size_t CanonicalQuery::countNodes(const MatchExpression* root, MatchExpression:: return sum; } +boost::optional<GeoNearMatchExpression*> CanonicalQuery::geoNear() const { + if (_splitGeoNear) { + return static_cast<GeoNearMatchExpression*>(_splitGeoNear->geoNear.get()); + } else { + return boost::none; + } +} + /** * Does 'root' have a subtree of type 'subtreeType' with a node of type 'childType' inside? */ @@ -509,6 +564,10 @@ std::string CanonicalQuery::toString() const { // The expression tree puts an endl on for us. ss << "Tree: " << _root->debugString(); + if (_splitGeoNear) { + ss << "NonGeoNear: " << _splitGeoNear->nonGeoNear->debugString(); + ss << "GeoNear: " << _splitGeoNear->geoNear->debugString(); + } ss << "Sort: " << _findCommand->getSort().toString() << '\n'; ss << "Proj: " << _findCommand->getProjection().toString() << '\n'; if (!_findCommand->getCollation().isEmpty()) { @@ -547,4 +606,220 @@ CanonicalQuery::QueryShapeString CanonicalQuery::encodeKey() const { return canonical_query_encoder::encode(*this); } +StatusWith<BSONObj> asAggregationCommand(const CanonicalQuery& cq) { + BSONObjBuilder aggregationBuilder; + + const FindCommandRequest& findCommand = cq.getFindCommandRequest(); + + // The find command will translate away ntoreturn above this layer. + tassert(5746106, "ntoreturn should not be set in the findCommand", !findCommand.getNtoreturn()); + + // First, check if this query has options that are not supported in aggregation. + if (!findCommand.getMin().isEmpty()) { + return {ErrorCodes::InvalidPipelineOperator, + str::stream() << "Option " << FindCommandRequest::kMinFieldName + << " not supported in aggregation."}; + } + if (!findCommand.getMax().isEmpty()) { + return {ErrorCodes::InvalidPipelineOperator, + str::stream() << "Option " << FindCommandRequest::kMaxFieldName + << " not supported in aggregation."}; + } + if (findCommand.getReturnKey()) { + return {ErrorCodes::InvalidPipelineOperator, + str::stream() << "Option " << FindCommandRequest::kReturnKeyFieldName + << " not supported in aggregation."}; + } + if (findCommand.getShowRecordId()) { + return {ErrorCodes::InvalidPipelineOperator, + str::stream() << "Option " << FindCommandRequest::kShowRecordIdFieldName + << " not supported in aggregation."}; + } + if (findCommand.getTailable()) { + return {ErrorCodes::InvalidPipelineOperator, + "Tailable cursors are not supported in aggregation."}; + } + if (findCommand.getNoCursorTimeout()) { + return {ErrorCodes::InvalidPipelineOperator, + str::stream() << "Option " << FindCommandRequest::kNoCursorTimeoutFieldName + << " not supported in aggregation."}; + } + if (findCommand.getAllowPartialResults()) { + return {ErrorCodes::InvalidPipelineOperator, + str::stream() << "Option " << FindCommandRequest::kAllowPartialResultsFieldName + << " not supported in aggregation."}; + } + if (findCommand.getSort()[query_request_helper::kNaturalSortField]) { + return {ErrorCodes::InvalidPipelineOperator, + str::stream() << "Sort option " << query_request_helper::kNaturalSortField + << " not supported in aggregation."}; + } + // The aggregation command normally does not support the 'singleBatch' option, but we make a + // special exception if 'limit' is set to 1. + if (findCommand.getSingleBatch() && findCommand.getLimit().value_or(0) != 1LL) { + return {ErrorCodes::InvalidPipelineOperator, + str::stream() << "Option " << FindCommandRequest::kSingleBatchFieldName + << " not supported in aggregation."}; + } + if (findCommand.getReadOnce()) { + return {ErrorCodes::InvalidPipelineOperator, + str::stream() << "Option " << FindCommandRequest::kReadOnceFieldName + << " not supported in aggregation."}; + } + + if (findCommand.getAllowSpeculativeMajorityRead()) { + return {ErrorCodes::InvalidPipelineOperator, + str::stream() << "Option " + << FindCommandRequest::kAllowSpeculativeMajorityReadFieldName + << " not supported in aggregation."}; + } + + if (findCommand.getRequestResumeToken()) { + return {ErrorCodes::InvalidPipelineOperator, + str::stream() << "Option " << FindCommandRequest::kRequestResumeTokenFieldName + << " not supported in aggregation."}; + } + + if (!findCommand.getResumeAfter().isEmpty()) { + return {ErrorCodes::InvalidPipelineOperator, + str::stream() << "Option " << FindCommandRequest::kResumeAfterFieldName + << " not supported in aggregation."}; + } + + // Now that we've successfully validated this QR, begin building the aggregation command. + aggregationBuilder.append("aggregate", + findCommand.getNamespaceOrUUID().nss() + ? findCommand.getNamespaceOrUUID().nss()->coll() + : ""); + + // Construct an aggregation pipeline that finds the equivalent documents to this query request. + BSONArrayBuilder pipelineBuilder(aggregationBuilder.subarrayStart("pipeline")); + { + if (auto geoNear = cq.geoNear()) { + const auto& args = (*geoNear)->getData(); + + BSONObjBuilder geoStage = pipelineBuilder.subobjStart(); + BSONObjBuilder geoArgs = geoStage.subobjStart("$geoNear"_sd); + + geoArgs.append("key"_sd, (*geoNear)->path()); + + // $near and $nearSphere both support two different syntaxes: + // - GeoJSON: {type: Point, coordinates: [x, y]} + // - legacy coordinate pair: [x, y] + bool spherical = [&]() { + // $nearSphere always uses spherical geometry, but $near can use either + // spherical or planar geometry, depending on how the centroid is specified. + switch (args.centroid->crs) { + case CRS::FLAT: + return false; + case CRS::SPHERE: + return true; + case CRS::STRICT_SPHERE: + tasserted(5844301, + "CRS::STRICT_SPHERE should not be possible in a " + "$near/$nearSphere query"); + case CRS::UNSET: + tasserted(5844302, + "CRS::UNSET should not be possible in a $near/$nearSphere query"); + } + tasserted(5844300, "Unhandled enum value for CRS, in $near/$nearSphere query"); + }(); + geoArgs.append("spherical"_sd, spherical); + + if (args.unitsAreRadians || !spherical) { + // Write the $geoNear as [x, y] coordinates. + BSONArrayBuilder nearBuilder = geoArgs.subarrayStart("near"_sd); + nearBuilder.append(args.centroid->oldPoint.x); + nearBuilder.append(args.centroid->oldPoint.y); + nearBuilder.doneFast(); + } else { + // Write the $geoNear as a GeoJSON point. + BSONObjBuilder nearBuilder = geoArgs.subobjStart("near"_sd); + nearBuilder.append("type"_sd, "Point"_sd); + { + BSONArrayBuilder coordinates = nearBuilder.subarrayStart("coordinates"_sd); + coordinates.append(args.centroid->oldPoint.x); + coordinates.append(args.centroid->oldPoint.y); + coordinates.doneFast(); + } + nearBuilder.doneFast(); + } + + if (args.minDistance) { + geoArgs.append("minDistance"_sd, args.minDistance); + } + if (args.maxDistance) { + geoArgs.append("maxDistance"_sd, args.maxDistance); + } + geoArgs.doneFast(); + geoStage.doneFast(); + } + if (auto filter = cq.rootWithoutGeoNear(); !filter->isTriviallyTrue()) { + BSONObjBuilder matchBuilder = pipelineBuilder.subobjStart(); + matchBuilder.append("$match"_sd, filter->serialize()); + matchBuilder.doneFast(); + } + } + if (!findCommand.getSort().isEmpty()) { + BSONObjBuilder sortBuilder(pipelineBuilder.subobjStart()); + sortBuilder.append("$sort", findCommand.getSort()); + sortBuilder.doneFast(); + } + if (findCommand.getSkip()) { + BSONObjBuilder skipBuilder(pipelineBuilder.subobjStart()); + skipBuilder.append("$skip", *findCommand.getSkip()); + skipBuilder.doneFast(); + } + if (findCommand.getLimit()) { + BSONObjBuilder limitBuilder(pipelineBuilder.subobjStart()); + limitBuilder.append("$limit", *findCommand.getLimit()); + limitBuilder.doneFast(); + } + if (!findCommand.getProjection().isEmpty()) { + BSONObjBuilder projectBuilder(pipelineBuilder.subobjStart()); + projectBuilder.append("$project", findCommand.getProjection()); + projectBuilder.doneFast(); + } + pipelineBuilder.doneFast(); + + // The aggregation 'cursor' option is always set, regardless of the presence of batchSize. + BSONObjBuilder batchSizeBuilder(aggregationBuilder.subobjStart("cursor")); + if (findCommand.getBatchSize()) { + batchSizeBuilder.append(FindCommandRequest::kBatchSizeFieldName, + *findCommand.getBatchSize()); + } + batchSizeBuilder.doneFast(); + + // Other options. + aggregationBuilder.append("collation", findCommand.getCollation()); + int maxTimeMS = findCommand.getMaxTimeMS() ? static_cast<int>(*findCommand.getMaxTimeMS()) : 0; + if (maxTimeMS > 0) { + aggregationBuilder.append(query_request_helper::cmdOptionMaxTimeMS, maxTimeMS); + } + if (!findCommand.getHint().isEmpty()) { + aggregationBuilder.append(FindCommandRequest::kHintFieldName, findCommand.getHint()); + } + if (findCommand.getReadConcern()) { + aggregationBuilder.append("readConcern", *findCommand.getReadConcern()); + } + if (!findCommand.getUnwrappedReadPref().isEmpty()) { + aggregationBuilder.append(FindCommandRequest::kUnwrappedReadPrefFieldName, + findCommand.getUnwrappedReadPref()); + } + if (findCommand.getAllowDiskUse()) { + aggregationBuilder.append(FindCommandRequest::kAllowDiskUseFieldName, + static_cast<bool>(findCommand.getAllowDiskUse())); + } + if (findCommand.getLegacyRuntimeConstants()) { + BSONObjBuilder rtcBuilder( + aggregationBuilder.subobjStart(FindCommandRequest::kLegacyRuntimeConstantsFieldName)); + findCommand.getLegacyRuntimeConstants()->serialize(&rtcBuilder); + rtcBuilder.doneFast(); + } + if (findCommand.getLet()) { + aggregationBuilder.append(FindCommandRequest::kLetFieldName, *findCommand.getLet()); + } + return StatusWith<BSONObj>(aggregationBuilder.obj()); +} + } // namespace mongo diff --git a/src/mongo/db/query/canonical_query.h b/src/mongo/db/query/canonical_query.h index 1e2a29d6def..a0eb4cfce8e 100644 --- a/src/mongo/db/query/canonical_query.h +++ b/src/mongo/db/query/canonical_query.h @@ -45,6 +45,7 @@ namespace mongo { class OperationContext; +class GeoNearMatchExpression; class CanonicalQuery { public: @@ -130,10 +131,32 @@ public: const BSONObj& getQueryObj() const { return _findCommand->getFilter(); } + + /** + * Returns the query with any $near or $nearSphere removed. + * Returns $alwaysTrue if the query contains nothing besides $near or $nearSphere. + */ + MatchExpression* rootWithoutGeoNear() const { + if (_splitGeoNear) { + return _splitGeoNear->nonGeoNear.get(); + } else { + return _root.get(); + } + } + + /** + * Returns the $near or $nearSphere expression, if the query has one. + */ + boost::optional<GeoNearMatchExpression*> geoNear() const; + const FindCommandRequest& getFindCommandRequest() const { return *_findCommand; } + std::unique_ptr<FindCommandRequest> releaseFindCommandRequest() && { + return std::move(_findCommand); + } + /** * Returns the projection, or nullptr if none. */ @@ -256,6 +279,16 @@ private: std::unique_ptr<MatchExpression> _root; + // If _root contains a GEO_NEAR, then we split it into a geo part and a non-geo part. + // GEO_NEAR is special because it also sorts, so it can't be represented as a $match; + // we split it so we can create a separate $geoNear stage instead. + // This is only supported if gTimeseriesMetricIndexes is enabled. + struct SplitGeoNear { + std::unique_ptr<MatchExpression> nonGeoNear; + std::unique_ptr<MatchExpression> geoNear; + }; + boost::optional<SplitGeoNear> _splitGeoNear; + boost::optional<projection_ast::Projection> _proj; boost::optional<SortPattern> _sortPattern; @@ -273,4 +306,11 @@ private: bool _enableSlotBasedExecutionEngine = false; }; +/** + * Converts this CanonicalQuery into an aggregation using $match. If this CanonicalQuery has + * options that cannot be satisfied by aggregation, a non-OK status is returned and 'cmdBuilder' is + * not modified. + */ +StatusWith<BSONObj> asAggregationCommand(const CanonicalQuery& cq); + } // namespace mongo diff --git a/src/mongo/db/query/query_request_helper.cpp b/src/mongo/db/query/query_request_helper.cpp index 7db63a1427b..a8405f00e69 100644 --- a/src/mongo/db/query/query_request_helper.cpp +++ b/src/mongo/db/query/query_request_helper.cpp @@ -38,6 +38,8 @@ #include "mongo/bson/simple_bsonobj_comparator.h" #include "mongo/db/commands/test_commands_enabled.h" #include "mongo/db/dbmessage.h" +#include "mongo/db/matcher/expression_geo.h" +#include "mongo/db/query/canonical_query.h" namespace mongo { @@ -414,158 +416,5 @@ StatusWith<std::unique_ptr<FindCommandRequest>> fromLegacyQuery(NamespaceStringO return std::move(findCommand); } -StatusWith<BSONObj> asAggregationCommand(const FindCommandRequest& findCommand) { - BSONObjBuilder aggregationBuilder; - - // The find command will translate away ntoreturn above this layer. - tassert(5746106, "ntoreturn should not be set in the findCommand", !findCommand.getNtoreturn()); - - // First, check if this query has options that are not supported in aggregation. - if (!findCommand.getMin().isEmpty()) { - return {ErrorCodes::InvalidPipelineOperator, - str::stream() << "Option " << FindCommandRequest::kMinFieldName - << " not supported in aggregation."}; - } - if (!findCommand.getMax().isEmpty()) { - return {ErrorCodes::InvalidPipelineOperator, - str::stream() << "Option " << FindCommandRequest::kMaxFieldName - << " not supported in aggregation."}; - } - if (findCommand.getReturnKey()) { - return {ErrorCodes::InvalidPipelineOperator, - str::stream() << "Option " << FindCommandRequest::kReturnKeyFieldName - << " not supported in aggregation."}; - } - if (findCommand.getShowRecordId()) { - return {ErrorCodes::InvalidPipelineOperator, - str::stream() << "Option " << FindCommandRequest::kShowRecordIdFieldName - << " not supported in aggregation."}; - } - if (findCommand.getTailable()) { - return {ErrorCodes::InvalidPipelineOperator, - "Tailable cursors are not supported in aggregation."}; - } - if (findCommand.getNoCursorTimeout()) { - return {ErrorCodes::InvalidPipelineOperator, - str::stream() << "Option " << FindCommandRequest::kNoCursorTimeoutFieldName - << " not supported in aggregation."}; - } - if (findCommand.getAllowPartialResults()) { - return {ErrorCodes::InvalidPipelineOperator, - str::stream() << "Option " << FindCommandRequest::kAllowPartialResultsFieldName - << " not supported in aggregation."}; - } - if (findCommand.getSort()[query_request_helper::kNaturalSortField]) { - return {ErrorCodes::InvalidPipelineOperator, - str::stream() << "Sort option " << query_request_helper::kNaturalSortField - << " not supported in aggregation."}; - } - // The aggregation command normally does not support the 'singleBatch' option, but we make a - // special exception if 'limit' is set to 1. - if (findCommand.getSingleBatch() && findCommand.getLimit().value_or(0) != 1LL) { - return {ErrorCodes::InvalidPipelineOperator, - str::stream() << "Option " << FindCommandRequest::kSingleBatchFieldName - << " not supported in aggregation."}; - } - if (findCommand.getReadOnce()) { - return {ErrorCodes::InvalidPipelineOperator, - str::stream() << "Option " << FindCommandRequest::kReadOnceFieldName - << " not supported in aggregation."}; - } - - if (findCommand.getAllowSpeculativeMajorityRead()) { - return {ErrorCodes::InvalidPipelineOperator, - str::stream() << "Option " - << FindCommandRequest::kAllowSpeculativeMajorityReadFieldName - << " not supported in aggregation."}; - } - - if (findCommand.getRequestResumeToken()) { - return {ErrorCodes::InvalidPipelineOperator, - str::stream() << "Option " << FindCommandRequest::kRequestResumeTokenFieldName - << " not supported in aggregation."}; - } - - if (!findCommand.getResumeAfter().isEmpty()) { - return {ErrorCodes::InvalidPipelineOperator, - str::stream() << "Option " << FindCommandRequest::kResumeAfterFieldName - << " not supported in aggregation."}; - } - - // Now that we've successfully validated this QR, begin building the aggregation command. - aggregationBuilder.append("aggregate", - findCommand.getNamespaceOrUUID().nss() - ? findCommand.getNamespaceOrUUID().nss()->coll() - : ""); - - // Construct an aggregation pipeline that finds the equivalent documents to this query request. - BSONArrayBuilder pipelineBuilder(aggregationBuilder.subarrayStart("pipeline")); - if (!findCommand.getFilter().isEmpty()) { - BSONObjBuilder matchBuilder(pipelineBuilder.subobjStart()); - matchBuilder.append("$match", findCommand.getFilter()); - matchBuilder.doneFast(); - } - if (!findCommand.getSort().isEmpty()) { - BSONObjBuilder sortBuilder(pipelineBuilder.subobjStart()); - sortBuilder.append("$sort", findCommand.getSort()); - sortBuilder.doneFast(); - } - if (findCommand.getSkip()) { - BSONObjBuilder skipBuilder(pipelineBuilder.subobjStart()); - skipBuilder.append("$skip", *findCommand.getSkip()); - skipBuilder.doneFast(); - } - if (findCommand.getLimit()) { - BSONObjBuilder limitBuilder(pipelineBuilder.subobjStart()); - limitBuilder.append("$limit", *findCommand.getLimit()); - limitBuilder.doneFast(); - } - if (!findCommand.getProjection().isEmpty()) { - BSONObjBuilder projectBuilder(pipelineBuilder.subobjStart()); - projectBuilder.append("$project", findCommand.getProjection()); - projectBuilder.doneFast(); - } - pipelineBuilder.doneFast(); - - // The aggregation 'cursor' option is always set, regardless of the presence of batchSize. - BSONObjBuilder batchSizeBuilder(aggregationBuilder.subobjStart("cursor")); - if (findCommand.getBatchSize()) { - batchSizeBuilder.append(FindCommandRequest::kBatchSizeFieldName, - *findCommand.getBatchSize()); - } - batchSizeBuilder.doneFast(); - - // Other options. - aggregationBuilder.append("collation", findCommand.getCollation()); - int maxTimeMS = findCommand.getMaxTimeMS() ? static_cast<int>(*findCommand.getMaxTimeMS()) : 0; - if (maxTimeMS > 0) { - aggregationBuilder.append(cmdOptionMaxTimeMS, maxTimeMS); - } - if (!findCommand.getHint().isEmpty()) { - aggregationBuilder.append(FindCommandRequest::kHintFieldName, findCommand.getHint()); - } - if (findCommand.getReadConcern()) { - aggregationBuilder.append("readConcern", *findCommand.getReadConcern()); - } - if (!findCommand.getUnwrappedReadPref().isEmpty()) { - aggregationBuilder.append(FindCommandRequest::kUnwrappedReadPrefFieldName, - findCommand.getUnwrappedReadPref()); - } - if (findCommand.getAllowDiskUse()) { - aggregationBuilder.append(FindCommandRequest::kAllowDiskUseFieldName, - static_cast<bool>(findCommand.getAllowDiskUse())); - } - if (findCommand.getLegacyRuntimeConstants()) { - BSONObjBuilder rtcBuilder( - aggregationBuilder.subobjStart(FindCommandRequest::kLegacyRuntimeConstantsFieldName)); - findCommand.getLegacyRuntimeConstants()->serialize(&rtcBuilder); - rtcBuilder.doneFast(); - } - if (findCommand.getLet()) { - aggregationBuilder.append(FindCommandRequest::kLetFieldName, *findCommand.getLet()); - } - return StatusWith<BSONObj>(aggregationBuilder.obj()); -} - } // namespace query_request_helper } // namespace mongo diff --git a/src/mongo/db/query/query_request_helper.h b/src/mongo/db/query/query_request_helper.h index c925c06dd38..063bdd1bac8 100644 --- a/src/mongo/db/query/query_request_helper.h +++ b/src/mongo/db/query/query_request_helper.h @@ -41,6 +41,7 @@ namespace mongo { class QueryMessage; +class CanonicalQuery; class Status; template <typename T> class StatusWith; @@ -93,13 +94,6 @@ std::unique_ptr<FindCommandRequest> makeFromFindCommandForTests( void refreshNSS(const NamespaceString& nss, FindCommandRequest* findCommand); /** - * Converts this FindCommandRequest into an aggregation using $match. If this FindCommandRequest has - * options that cannot be satisfied by aggregation, a non-OK status is returned and 'cmdBuilder' is - * not modified. - */ -StatusWith<BSONObj> asAggregationCommand(const FindCommandRequest& findCommand); - -/** * Helper function to identify text search sort key * Example: {a: {$meta: "textScore"}} */ diff --git a/src/mongo/db/query/query_request_test.cpp b/src/mongo/db/query/query_request_test.cpp index 6eeb8704f26..46f5e716bb8 100644 --- a/src/mongo/db/query/query_request_test.cpp +++ b/src/mongo/db/query/query_request_test.cpp @@ -40,8 +40,10 @@ #include "mongo/db/json.h" #include "mongo/db/namespace_string.h" #include "mongo/db/pipeline/aggregation_request_helper.h" +#include "mongo/db/query/canonical_query.h" #include "mongo/db/query/query_request_helper.h" #include "mongo/db/service_context_test_fixture.h" +#include "mongo/idl/server_parameter_test_util.h" #include "mongo/unittest/unittest.h" namespace mongo { @@ -52,6 +54,29 @@ using unittest::assertGet; static const NamespaceString testns("testdb.testcoll"); +class QueryRequestTest : public ServiceContextTest { +public: + ServiceContext::UniqueOperationContext uniqueTxn = makeOperationContext(); + OperationContext* opCtx = uniqueTxn.get(); + + // Helper wrapper for this test--the real asAggregationCommand() takes a CanonicalQuery, + // but all these examples use FindCommand. For convenience this wrapper canonicalizes + // the FindCommand before calling the real function. This wrapper is limited to this test, + // because exposing it would make it easy to accidentally canonicalize a query twice. + StatusWith<BSONObj> testAsAggregationCommand(const FindCommandRequest& findCommand) { + auto cq = CanonicalQuery::canonicalize(opCtx, + std::make_unique<FindCommandRequest>(findCommand), + false /*isExplain*/, + nullptr /*expCtx*/, + ExtensionsCallbackNoop(), + Pipeline::viewFindMatcherFeatures()); + if (!cq.isOK()) + return cq.getStatus(); + + return asAggregationCommand(*cq.getValue()); + } +}; + TEST(QueryRequestTest, LimitWithNToReturn) { FindCommandRequest findCommand(testns); findCommand.setLimit(1); @@ -1264,9 +1289,9 @@ TEST(QueryRequestTest, ParseMaxTimeMSPositiveInRangeSucceeds) { ASSERT_EQ(parseMaxTimeMSForIDL(maxTimeObj[query_request_helper::cmdOptionMaxTimeMS]), 300); } -TEST(QueryRequestTest, ConvertToAggregationSucceeds) { +TEST_F(QueryRequestTest, ConvertToAggregationSucceeds) { FindCommandRequest findCommand(testns); - auto agg = query_request_helper::asAggregationCommand(findCommand); + auto agg = testAsAggregationCommand(findCommand); ASSERT_OK(agg); auto aggCmd = OpMsgRequest::fromDBAndBody(testns.db(), agg.getValue()).body; @@ -1281,9 +1306,9 @@ TEST(QueryRequestTest, ConvertToAggregationSucceeds) { ASSERT_BSONOBJ_EQ(ar.getValue().getCollation().value_or(BSONObj()), BSONObj()); } -TEST(QueryRequestTest, ConvertToAggregationOmitsExplain) { +TEST_F(QueryRequestTest, ConvertToAggregationOmitsExplain) { FindCommandRequest findCommand(testns); - auto agg = query_request_helper::asAggregationCommand(findCommand); + auto agg = testAsAggregationCommand(findCommand); ASSERT_OK(agg); auto aggCmd = OpMsgRequest::fromDBAndBody(testns.db(), agg.getValue()).body; @@ -1295,10 +1320,10 @@ TEST(QueryRequestTest, ConvertToAggregationOmitsExplain) { ASSERT_BSONOBJ_EQ(ar.getValue().getCollation().value_or(BSONObj()), BSONObj()); } -TEST(QueryRequestTest, ConvertToAggregationWithHintSucceeds) { +TEST_F(QueryRequestTest, ConvertToAggregationWithHintSucceeds) { FindCommandRequest findCommand(testns); findCommand.setHint(fromjson("{a_1: -1}")); - const auto agg = query_request_helper::asAggregationCommand(findCommand); + const auto agg = testAsAggregationCommand(findCommand); ASSERT_OK(agg); auto aggCmd = OpMsgRequest::fromDBAndBody(testns.db(), agg.getValue()).body; @@ -1307,88 +1332,88 @@ TEST(QueryRequestTest, ConvertToAggregationWithHintSucceeds) { ASSERT_BSONOBJ_EQ(findCommand.getHint(), ar.getValue().getHint().value_or(BSONObj())); } -TEST(QueryRequestTest, ConvertToAggregationWithMinFails) { +TEST_F(QueryRequestTest, ConvertToAggregationWithMinFails) { FindCommandRequest findCommand(testns); findCommand.setMin(fromjson("{a: 1}")); - ASSERT_NOT_OK(query_request_helper::asAggregationCommand(findCommand)); + ASSERT_NOT_OK(testAsAggregationCommand(findCommand)); } -TEST(QueryRequestTest, ConvertToAggregationWithMaxFails) { +TEST_F(QueryRequestTest, ConvertToAggregationWithMaxFails) { FindCommandRequest findCommand(testns); findCommand.setMax(fromjson("{a: 1}")); - ASSERT_NOT_OK(query_request_helper::asAggregationCommand(findCommand)); + ASSERT_NOT_OK(testAsAggregationCommand(findCommand)); } -TEST(QueryRequestTest, ConvertToAggregationWithSingleBatchFieldFails) { +TEST_F(QueryRequestTest, ConvertToAggregationWithSingleBatchFieldFails) { FindCommandRequest findCommand(testns); findCommand.setSingleBatch(true); - ASSERT_NOT_OK(query_request_helper::asAggregationCommand(findCommand)); + ASSERT_NOT_OK(testAsAggregationCommand(findCommand)); } -TEST(QueryRequestTest, ConvertToAggregationWithSingleBatchFieldAndLimitFails) { +TEST_F(QueryRequestTest, ConvertToAggregationWithSingleBatchFieldAndLimitFails) { FindCommandRequest findCommand(testns); findCommand.setSingleBatch(true); findCommand.setLimit(7); - ASSERT_NOT_OK(query_request_helper::asAggregationCommand(findCommand)); + ASSERT_NOT_OK(testAsAggregationCommand(findCommand)); } -TEST(QueryRequestTest, ConvertToAggregationWithSingleBatchFieldLimitOneSucceeds) { +TEST_F(QueryRequestTest, ConvertToAggregationWithSingleBatchFieldLimitOneSucceeds) { FindCommandRequest findCommand(testns); findCommand.setSingleBatch(true); findCommand.setLimit(1); - ASSERT_OK(query_request_helper::asAggregationCommand(findCommand)); + ASSERT_OK(testAsAggregationCommand(findCommand)); } -TEST(QueryRequestTest, ConvertToAggregationWithReturnKeyFails) { +TEST_F(QueryRequestTest, ConvertToAggregationWithReturnKeyFails) { FindCommandRequest findCommand(testns); findCommand.setReturnKey(true); - ASSERT_NOT_OK(query_request_helper::asAggregationCommand(findCommand)); + ASSERT_NOT_OK(testAsAggregationCommand(findCommand)); } -TEST(QueryRequestTest, ConvertToAggregationWithShowRecordIdFails) { +TEST_F(QueryRequestTest, ConvertToAggregationWithShowRecordIdFails) { FindCommandRequest findCommand(testns); findCommand.setShowRecordId(true); - ASSERT_NOT_OK(query_request_helper::asAggregationCommand(findCommand)); + ASSERT_NOT_OK(testAsAggregationCommand(findCommand)); } -TEST(QueryRequestTest, ConvertToAggregationWithTailableFails) { +TEST_F(QueryRequestTest, ConvertToAggregationWithTailableFails) { FindCommandRequest findCommand(testns); query_request_helper::setTailableMode(TailableModeEnum::kTailable, &findCommand); - ASSERT_NOT_OK(query_request_helper::asAggregationCommand(findCommand)); + ASSERT_NOT_OK(testAsAggregationCommand(findCommand)); } -TEST(QueryRequestTest, ConvertToAggregationWithNoCursorTimeoutFails) { +TEST_F(QueryRequestTest, ConvertToAggregationWithNoCursorTimeoutFails) { FindCommandRequest findCommand(testns); findCommand.setNoCursorTimeout(true); - ASSERT_NOT_OK(query_request_helper::asAggregationCommand(findCommand)); + ASSERT_NOT_OK(testAsAggregationCommand(findCommand)); } -TEST(QueryRequestTest, ConvertToAggregationWithAwaitDataFails) { +TEST_F(QueryRequestTest, ConvertToAggregationWithAwaitDataFails) { FindCommandRequest findCommand(testns); query_request_helper::setTailableMode(TailableModeEnum::kTailableAndAwaitData, &findCommand); - ASSERT_NOT_OK(query_request_helper::asAggregationCommand(findCommand)); + ASSERT_NOT_OK(testAsAggregationCommand(findCommand)); } -TEST(QueryRequestTest, ConvertToAggregationWithAllowPartialResultsFails) { +TEST_F(QueryRequestTest, ConvertToAggregationWithAllowPartialResultsFails) { FindCommandRequest findCommand(testns); findCommand.setAllowPartialResults(true); - ASSERT_NOT_OK(query_request_helper::asAggregationCommand(findCommand)); + ASSERT_NOT_OK(testAsAggregationCommand(findCommand)); } -TEST(QueryRequestTest, ConvertToAggregationWithRequestResumeTokenFails) { +TEST_F(QueryRequestTest, ConvertToAggregationWithRequestResumeTokenFails) { FindCommandRequest findCommand(testns); findCommand.setRequestResumeToken(true); - ASSERT_NOT_OK(query_request_helper::asAggregationCommand(findCommand)); + ASSERT_NOT_OK(testAsAggregationCommand(findCommand)); } -TEST(QueryRequestTest, ConvertToAggregationWithResumeAfterFails) { +TEST_F(QueryRequestTest, ConvertToAggregationWithResumeAfterFails) { FindCommandRequest findCommand(testns); BSONObj resumeAfter = BSON("$recordId" << 1LL); findCommand.setResumeAfter(resumeAfter); - ASSERT_NOT_OK(query_request_helper::asAggregationCommand(findCommand)); + ASSERT_NOT_OK(testAsAggregationCommand(findCommand)); } -TEST(QueryRequestTest, ConvertToAggregationWithPipeline) { +TEST_F(QueryRequestTest, ConvertToAggregationWithPipeline) { FindCommandRequest findCommand(testns); findCommand.setFilter(BSON("x" << 1)); findCommand.setSort(BSON("y" << -1)); @@ -1396,7 +1421,7 @@ TEST(QueryRequestTest, ConvertToAggregationWithPipeline) { findCommand.setSkip(7); findCommand.setProjection(BSON("z" << 0)); - auto agg = query_request_helper::asAggregationCommand(findCommand); + auto agg = testAsAggregationCommand(findCommand); ASSERT_OK(agg); auto aggCmd = OpMsgRequest::fromDBAndBody(testns.db(), agg.getValue()).body; @@ -1409,7 +1434,7 @@ TEST(QueryRequestTest, ConvertToAggregationWithPipeline) { ASSERT_EQ(ar.getValue().getNamespace(), testns); ASSERT_BSONOBJ_EQ(ar.getValue().getCollation().value_or(BSONObj()), BSONObj()); - std::vector<BSONObj> expectedPipeline{BSON("$match" << BSON("x" << 1)), + std::vector<BSONObj> expectedPipeline{BSON("$match" << BSON("x" << BSON("$eq" << 1))), BSON("$sort" << BSON("y" << -1)), BSON("$skip" << 7), BSON("$limit" << 3), @@ -1417,14 +1442,55 @@ TEST(QueryRequestTest, ConvertToAggregationWithPipeline) { ASSERT(std::equal(expectedPipeline.begin(), expectedPipeline.end(), ar.getValue().getPipeline().begin(), + ar.getValue().getPipeline().end(), SimpleBSONObjComparator::kInstance.makeEqualTo())); } -TEST(QueryRequestTest, ConvertToAggregationWithBatchSize) { +TEST_F(QueryRequestTest, ConvertToAggregationWithGeoNear) { + RAIIServerParameterControllerForTest controller("featureFlagTimeseriesMetricIndexes", true); + + FindCommandRequest findCommand(testns); + findCommand.setFilter(BSON("x" << 1 << "loc" << BSON("$near" << BSON_ARRAY(0 << 0)))); + findCommand.setSort(BSON("y" << -1)); + findCommand.setLimit(3); + findCommand.setSkip(7); + findCommand.setProjection(BSON("z" << 0)); + + auto agg = testAsAggregationCommand(findCommand); + ASSERT_OK(agg); + + auto aggCmd = OpMsgRequest::fromDBAndBody(testns.db(), agg.getValue()).body; + auto ar = aggregation_request_helper::parseFromBSONForTests(testns, aggCmd); + ASSERT_OK(ar.getStatus()); + ASSERT(!ar.getValue().getExplain()); + ASSERT_EQ(ar.getValue().getCursor().getBatchSize().value_or( + aggregation_request_helper::kDefaultBatchSize), + aggregation_request_helper::kDefaultBatchSize); + ASSERT_EQ(ar.getValue().getNamespace(), testns); + ASSERT_BSONOBJ_EQ(ar.getValue().getCollation().value_or(BSONObj()), BSONObj()); + + std::vector<BSONObj> expectedPipeline{ + BSON("$geoNear" << BSON("key" + << "loc" + << "spherical" << false << "near" << BSON_ARRAY(0.0 << 0.0) + << "maxDistance" << std::numeric_limits<double>::max())), + BSON("$match" << BSON("x" << BSON("$eq" << 1))), + BSON("$sort" << BSON("y" << -1)), + BSON("$skip" << 7), + BSON("$limit" << 3), + BSON("$project" << BSON("z" << 0))}; + ASSERT(std::equal(expectedPipeline.begin(), + expectedPipeline.end(), + ar.getValue().getPipeline().begin(), + ar.getValue().getPipeline().end(), + SimpleBSONObjComparator::kInstance.makeEqualTo())); +} + +TEST_F(QueryRequestTest, ConvertToAggregationWithBatchSize) { FindCommandRequest findCommand(testns); findCommand.setBatchSize(4); - auto agg = query_request_helper::asAggregationCommand(findCommand); + auto agg = testAsAggregationCommand(findCommand); ASSERT_OK(agg); auto aggCmd = OpMsgRequest::fromDBAndBody(testns.db(), agg.getValue()).body; @@ -1438,11 +1504,11 @@ TEST(QueryRequestTest, ConvertToAggregationWithBatchSize) { ASSERT_BSONOBJ_EQ(ar.getValue().getCollation().value_or(BSONObj()), BSONObj()); } -TEST(QueryRequestTest, ConvertToAggregationWithMaxTimeMS) { +TEST_F(QueryRequestTest, ConvertToAggregationWithMaxTimeMS) { FindCommandRequest findCommand(testns); findCommand.setMaxTimeMS(9); - auto agg = query_request_helper::asAggregationCommand(findCommand); + auto agg = testAsAggregationCommand(findCommand); ASSERT_OK(agg); const BSONObj cmdObj = agg.getValue(); @@ -1459,10 +1525,11 @@ TEST(QueryRequestTest, ConvertToAggregationWithMaxTimeMS) { ASSERT_BSONOBJ_EQ(ar.getValue().getCollation().value_or(BSONObj()), BSONObj()); } -TEST(QueryRequestTest, ConvertToAggregationWithCollationSucceeds) { +TEST_F(QueryRequestTest, ConvertToAggregationWithCollationSucceeds) { FindCommandRequest findCommand(testns); - findCommand.setCollation(BSON("f" << 1)); - auto agg = query_request_helper::asAggregationCommand(findCommand); + findCommand.setCollation(BSON("locale" + << "fr")); + auto agg = testAsAggregationCommand(findCommand); ASSERT_OK(agg); auto aggCmd = OpMsgRequest::fromDBAndBody(testns.db(), agg.getValue()).body; @@ -1474,28 +1541,30 @@ TEST(QueryRequestTest, ConvertToAggregationWithCollationSucceeds) { aggregation_request_helper::kDefaultBatchSize), aggregation_request_helper::kDefaultBatchSize); ASSERT_EQ(ar.getValue().getNamespace(), testns); - ASSERT_BSONOBJ_EQ(ar.getValue().getCollation().value_or(BSONObj()), BSON("f" << 1)); + ASSERT_BSONOBJ_EQ(ar.getValue().getCollation().value_or(BSONObj()), + BSON("locale" + << "fr")); } -TEST(QueryRequestTest, ConvertToAggregationWithReadOnceFails) { +TEST_F(QueryRequestTest, ConvertToAggregationWithReadOnceFails) { FindCommandRequest findCommand(testns); findCommand.setReadOnce(true); - const auto aggCmd = query_request_helper::asAggregationCommand(findCommand); + const auto aggCmd = testAsAggregationCommand(findCommand); ASSERT_EQ(ErrorCodes::InvalidPipelineOperator, aggCmd.getStatus().code()); } -TEST(QueryRequestTest, ConvertToAggregationWithAllowSpeculativeMajorityReadFails) { +TEST_F(QueryRequestTest, ConvertToAggregationWithAllowSpeculativeMajorityReadFails) { FindCommandRequest findCommand(testns); findCommand.setAllowSpeculativeMajorityRead(true); - const auto aggCmd = query_request_helper::asAggregationCommand(findCommand); + const auto aggCmd = testAsAggregationCommand(findCommand); ASSERT_EQ(ErrorCodes::InvalidPipelineOperator, aggCmd.getStatus().code()); } -TEST(QueryRequestTest, ConvertToAggregationWithLegacyRuntimeConstantsSucceeds) { +TEST_F(QueryRequestTest, ConvertToAggregationWithLegacyRuntimeConstantsSucceeds) { LegacyRuntimeConstants rtc{Date_t::now(), Timestamp(1, 1)}; FindCommandRequest findCommand(testns); findCommand.setLegacyRuntimeConstants(rtc); - auto agg = query_request_helper::asAggregationCommand(findCommand); + auto agg = testAsAggregationCommand(findCommand); ASSERT_OK(agg); auto aggCmd = OpMsgRequest::fromDBAndBody(testns.db(), agg.getValue()).body; @@ -1506,10 +1575,10 @@ TEST(QueryRequestTest, ConvertToAggregationWithLegacyRuntimeConstantsSucceeds) { ASSERT_EQ(ar.getValue().getLegacyRuntimeConstants()->getClusterTime(), rtc.getClusterTime()); } -TEST(QueryRequestTest, ConvertToAggregationWithAllowDiskUseTrueSucceeds) { +TEST_F(QueryRequestTest, ConvertToAggregationWithAllowDiskUseTrueSucceeds) { FindCommandRequest findCommand(testns); findCommand.setAllowDiskUse(true); - const auto agg = query_request_helper::asAggregationCommand(findCommand); + const auto agg = testAsAggregationCommand(findCommand); ASSERT_OK(agg.getStatus()); auto aggCmd = OpMsgRequest::fromDBAndBody(testns.db(), agg.getValue()).body; @@ -1518,10 +1587,10 @@ TEST(QueryRequestTest, ConvertToAggregationWithAllowDiskUseTrueSucceeds) { ASSERT_EQ(true, ar.getValue().getAllowDiskUse()); } -TEST(QueryRequestTest, ConvertToAggregationWithAllowDiskUseFalseSucceeds) { +TEST_F(QueryRequestTest, ConvertToAggregationWithAllowDiskUseFalseSucceeds) { FindCommandRequest findCommand(testns); findCommand.setAllowDiskUse(false); - const auto agg = query_request_helper::asAggregationCommand(findCommand); + const auto agg = testAsAggregationCommand(findCommand); ASSERT_OK(agg.getStatus()); auto aggCmd = OpMsgRequest::fromDBAndBody(testns.db(), agg.getValue()).body; @@ -1642,8 +1711,6 @@ TEST(QueryRequestTest, ParseFromLegacyQueryExplainError) { static_cast<ErrorCodes::Error>(5856600)); } -class QueryRequestTest : public ServiceContextTest {}; - TEST_F(QueryRequestTest, ParseFromUUID) { const CollectionUUID uuid = UUID::gen(); diff --git a/src/mongo/s/commands/cluster_find_cmd.cpp b/src/mongo/s/commands/cluster_find_cmd.cpp index 46374f2e505..861664d9b2c 100644 --- a/src/mongo/s/commands/cluster_find_cmd.cpp +++ b/src/mongo/s/commands/cluster_find_cmd.cpp @@ -38,6 +38,7 @@ #include "mongo/db/matcher/extensions_callback_noop.h" #include "mongo/db/query/cursor_response.h" #include "mongo/db/stats/counters.h" +#include "mongo/db/storage/storage_parameters_gen.h" #include "mongo/db/views/resolved_view.h" #include "mongo/rpc/get_status_from_command_result.h" #include "mongo/s/catalog_cache.h" @@ -197,8 +198,14 @@ public: auto bodyBuilder = result->getBodyBuilder(); bodyBuilder.resetToEmpty(); - auto aggCmdOnView = - uassertStatusOK(query_request_helper::asAggregationCommand(*findCommand)); + auto cq = uassertStatusOK( + CanonicalQuery::canonicalize(opCtx, + std::move(findCommand), + false /*isExplain*/, + nullptr /*expCtx*/, + ExtensionsCallbackNoop(), + Pipeline::viewFindMatcherFeatures())); + auto aggCmdOnView = uassertStatusOK(asAggregationCommand(*cq)); auto viewAggregationCommand = OpMsgRequest::fromDBAndBody(_dbName, aggCmdOnView).body; @@ -264,8 +271,16 @@ public: } catch (const ExceptionFor<ErrorCodes::CommandOnShardedViewNotSupportedOnMongod>& ex) { result->reset(); - auto aggCmdOnView = uassertStatusOK( - query_request_helper::asAggregationCommand(cq->getFindCommandRequest())); + // 'cq' was constructed with kAllowAllSpecialFeatures, but now we want to convert it + // to an aggregation, which implies a stricter set of allowed features. + cq = uassertStatusOK( + CanonicalQuery::canonicalize(opCtx, + std::move(*cq).releaseFindCommandRequest(), + false, /* isExplain */ + expCtx, + ExtensionsCallbackNoop(), + Pipeline::viewFindMatcherFeatures())); + auto aggCmdOnView = uassertStatusOK(asAggregationCommand(*cq)); auto viewAggregationCommand = OpMsgRequest::fromDBAndBody(_dbName, aggCmdOnView).body; |