diff options
28 files changed, 125 insertions, 2246 deletions
diff --git a/buildscripts/resmokeconfig/suites/replica_sets_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_kill_primary_jscore_passthrough.yml index a44a1593231..930cefb51f8 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_kill_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_kill_primary_jscore_passthrough.yml @@ -29,6 +29,7 @@ selector: # TODO SERVER-31242: findAndModify no-op retry should respect the fields option. - jstests/core/crud_api.js + - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml index 2ab62eeb528..e14eeff7f80 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml @@ -290,6 +290,7 @@ selector: # TODO SERVER-31242: findAndModify no-op retry should respect the fields option. - jstests/core/crud_api.js + - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml index 7955ab0d6d6..bbd5d018852 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml @@ -288,6 +288,7 @@ selector: # TODO SERVER-31242: findAndModify no-op retry should respect the fields option. - jstests/core/crud_api.js + - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_terminate_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_terminate_primary_jscore_passthrough.yml index 24bd76896e5..d41f43bc0a1 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_terminate_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_terminate_primary_jscore_passthrough.yml @@ -29,6 +29,7 @@ selector: # TODO SERVER-31242: findAndModify no-op retry should respect the fields option. - jstests/core/crud_api.js + - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js diff --git a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml index 879a2f77736..cfc950209e9 100644 --- a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml @@ -32,6 +32,7 @@ selector: # TODO SERVER-31242: findAndModify no-op retry should respect the fields option. - jstests/core/crud_api.js + - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_pipeline_update.js - jstests/core/find_and_modify_server6865.js diff --git a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml index f350ab37942..82b57754c37 100644 --- a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml @@ -29,6 +29,7 @@ selector: # TODO SERVER-31242: findAndModify no-op retry should respect the fields option. - jstests/core/crud_api.js + - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js diff --git a/jstests/core/find_and_modify.js b/jstests/core/find_and_modify.js index 83e1e7da79d..8f437726c51 100644 --- a/jstests/core/find_and_modify.js +++ b/jstests/core/find_and_modify.js @@ -33,6 +33,32 @@ out = t.findAndModify( {query: {inprogress: false}, sort: {priority: -1}, update: {$set: {inprogress: true}}}); assert.eq(out.priority, 9); +// Use expressions in the 'fields' argument with 'new' false. +out = t.findAndModify({ + query: {inprogress: false}, + sort: {priority: -1}, + 'new': false, + update: {$set: {inprogress: true}, $inc: {value: 1}}, + fields: {priority: 1, inprogress: 1, computedField: {$add: ["$value", 2]}} +}); +assert.eq(out.priority, 8); +assert.eq(out.inprogress, false); +// The projection should have been applied to the pre image of the update. +assert.eq(out.computedField, 2); + +// Use expressions in the 'fields' argument with 'new' true. +out = t.findAndModify({ + query: {inprogress: false}, + sort: {priority: -1}, + update: {$set: {inprogress: true}, $inc: {value: 1}}, + 'new': true, + fields: {priority: 1, inprogress: 1, computedField: {$add: ["$value", 2]}} +}); +assert.eq(out.priority, 7); +assert.eq(out.inprogress, true); +// The projection should have been applied to the update post image. +assert.eq(out.computedField, 3); + // remove lowest priority out = t.findAndModify({sort: {priority: 1}, remove: true}); assert.eq(out.priority, 1); @@ -122,15 +148,6 @@ cmdRes = db.runCommand({ }); assert.commandFailed(cmdRes); -// Cannot use expressions in the projection part of findAndModify. -cmdRes = db.runCommand({ - findAndModify: "coll", - update: {a: 1}, - fields: {b: {$ln: ['c']}}, - upsert: true, -}); -assert.commandFailedWithCode(cmdRes, ErrorCodes.BadValue); - // // SERVER-17372 // diff --git a/jstests/core/find_projection_meta_errors.js b/jstests/core/find_projection_meta_errors.js index bab64b816b6..ea6206c4703 100644 --- a/jstests/core/find_projection_meta_errors.js +++ b/jstests/core/find_projection_meta_errors.js @@ -10,6 +10,5 @@ assert.commandWorked(coll.insert({a: 1})); assert.commandWorked(coll.insert({a: 2})); assert.commandFailedWithCode( - db.runCommand({find: coll.getName(), projection: {score: {$meta: "some garbage"}}}), - ErrorCodes.BadValue); + db.runCommand({find: coll.getName(), projection: {score: {$meta: "some garbage"}}}), 17308); }()); diff --git a/jstests/core/fts_projection.js b/jstests/core/fts_projection.js index f021ce04298..45ca86bcd89 100644 --- a/jstests/core/fts_projection.js +++ b/jstests/core/fts_projection.js @@ -29,15 +29,17 @@ scores[results[1]._id] = results[1].score; // Edge/error cases: // -// Project text score into 2 fields. +// Project text score into 3 fields, one nested. results = t.find({$text: {$search: "textual content -irrelevant"}}, { otherScore: {$meta: "textScore"}, - score: {$meta: "textScore"} + score: {$meta: "textScore"}, + "nestedObj.score": {$meta: "textScore"} }).toArray(); assert.eq(2, results.length); for (var i = 0; i < results.length; ++i) { assert.close(scores[results[i]._id], results[i].score); assert.close(scores[results[i]._id], results[i].otherScore); + assert.close(scores[results[i]._id], results[i].nestedObj.score); } // printjson(results); @@ -66,12 +68,6 @@ assert.neq(-1, results[0].b); // Don't crash if we have no text score. var results = t.find({a: /text/}, {score: {$meta: "textScore"}}).toArray(); -// printjson(results); - -// No textScore proj. with nested fields -assert.throws(function() { - t.find({$text: {$search: "blah"}}, {'x.y': {$meta: "textScore"}}).toArray(); -}); // SERVER-12173 // When $text operator is in $or, should evaluate first diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index 0170303b829..c0f40eebca9 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -1017,7 +1017,6 @@ env.Library( 'exec/pipeline_proxy.cpp', 'exec/plan_stage.cpp', 'exec/projection.cpp', - 'exec/projection_exec.cpp', 'exec/projection_executor.cpp', 'exec/queued_data_stage.cpp', 'exec/record_store_fast_count.cpp', diff --git a/src/mongo/db/exec/SConscript b/src/mongo/db/exec/SConscript index cadce17d459..d8c2926a43d 100644 --- a/src/mongo/db/exec/SConscript +++ b/src/mongo/db/exec/SConscript @@ -79,7 +79,6 @@ env.CppUnitTest( "document_value/value_comparator_test.cpp", "find_projection_executor_test.cpp", "projection_exec_agg_test.cpp", - "projection_exec_test.cpp", "projection_executor_test.cpp", "queued_data_stage_test.cpp", "sort_test.cpp", diff --git a/src/mongo/db/exec/projection.cpp b/src/mongo/db/exec/projection.cpp index 5d50aee547c..d824df21a6c 100644 --- a/src/mongo/db/exec/projection.cpp +++ b/src/mongo/db/exec/projection.cpp @@ -127,8 +127,8 @@ void ProjectionStage::getSimpleInclusionFields(const BSONObj& projObj, FieldSet* // The _id is included by default. bool includeId = true; - // Figure out what fields are in the projection. TODO: we can get this from the - // ParsedProjection...modify that to have this type instead of a vector. + // Figure out what fields are in the projection. We could eventually do this using the + // Projection AST. BSONObjIterator projObjIt(projObj); while (projObjIt.more()) { BSONElement elt = projObjIt.next(); diff --git a/src/mongo/db/exec/projection_exec.cpp b/src/mongo/db/exec/projection_exec.cpp deleted file mode 100644 index bf212a0b44b..00000000000 --- a/src/mongo/db/exec/projection_exec.cpp +++ /dev/null @@ -1,489 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/exec/projection_exec.h" - -#include "mongo/bson/mutable/document.h" -#include "mongo/db/matcher/expression.h" -#include "mongo/db/matcher/expression_parser.h" -#include "mongo/db/query/collation/collator_interface.h" -#include "mongo/db/query/query_request.h" -#include "mongo/db/update/path_support.h" -#include "mongo/util/decimal_counter.h" -#include "mongo/util/str.h" - -namespace mongo { - -using std::string; - -namespace mmb = mongo::mutablebson; - -ProjectionExec::ProjectionExec(OperationContext* opCtx, - const BSONObj& spec, - const MatchExpression* queryExpression, - const CollatorInterface* collator) - : _source(spec), _queryExpression(queryExpression), _collator(collator) { - // Whether we're including or excluding fields. - enum class IncludeExclude { kUninitialized, kInclude, kExclude }; - IncludeExclude includeExclude = IncludeExclude::kUninitialized; - - BSONObjIterator it(_source); - while (it.more()) { - BSONElement e = it.next(); - - if (Object == e.type()) { - BSONObj obj = e.embeddedObject(); - invariant(1 == obj.nFields()); - - BSONElement e2 = obj.firstElement(); - if (e2.fieldNameStringData() == "$slice") { - if (e2.isNumber()) { - int i = e2.numberInt(); - if (i < 0) { - add(e.fieldName(), i, -i); // limit is now positive - } else { - add(e.fieldName(), 0, i); - } - } else { - invariant(e2.type() == Array); - BSONObj arr = e2.embeddedObject(); - invariant(2 == arr.nFields()); - - BSONObjIterator it(arr); - int skip = it.next().numberInt(); - int limit = it.next().numberInt(); - - invariant(limit > 0); - - add(e.fieldName(), skip, limit); - } - } else if (e2.fieldNameStringData() == "$elemMatch") { - _arrayOpType = ARRAY_OP_ELEM_MATCH; - - // Create a MatchExpression for the elemMatch. - BSONObj elemMatchObj = e.wrap(); - invariant(elemMatchObj.isOwned()); - _elemMatchObjs.push_back(elemMatchObj); - boost::intrusive_ptr<ExpressionContext> expCtx( - new ExpressionContext(opCtx, _collator)); - StatusWithMatchExpression statusWithMatcher = - MatchExpressionParser::parse(elemMatchObj, std::move(expCtx)); - invariant(statusWithMatcher.isOK()); - // And store it in _matchers. - _matchers[str::before(e.fieldNameStringData(), '.')] = - statusWithMatcher.getValue().release(); - - add(e.fieldName(), true); - } else if (e2.fieldNameStringData() == "$meta") { - invariant(String == e2.type()); - if (e2.valuestr() == QueryRequest::metaTextScore) { - _meta[e.fieldName()] = META_TEXT_SCORE; - _needsTextScore = true; - } else if (e2.valuestr() == QueryRequest::metaSortKey) { - _meta[e.fieldName()] = META_SORT_KEY; - _needsSortKey = true; - } else if (e2.valuestr() == QueryRequest::metaRecordId) { - _meta[e.fieldName()] = META_RECORDID; - } else if (e2.valuestr() == QueryRequest::metaGeoNearPoint) { - _meta[e.fieldName()] = META_GEONEAR_POINT; - _needsGeoNearPoint = true; - } else if (e2.valuestr() == QueryRequest::metaGeoNearDistance) { - _meta[e.fieldName()] = META_GEONEAR_DIST; - _needsGeoNearDistance = true; - } else { - // This shouldn't happen, should be caught by parsing. - MONGO_UNREACHABLE; - } - } else { - MONGO_UNREACHABLE; - } - } else if ((e.fieldNameStringData() == "_id") && !e.trueValue()) { - _includeID = false; - } else { - add(e.fieldName(), e.trueValue()); - - // If we haven't specified an include/exclude, initialize includeExclude. - if (includeExclude == IncludeExclude::kUninitialized) { - includeExclude = - e.trueValue() ? IncludeExclude::kInclude : IncludeExclude::kExclude; - _include = !e.trueValue(); - } - } - - if (str::contains(e.fieldName(), ".$")) { - _arrayOpType = ARRAY_OP_POSITIONAL; - } - } -} - -ProjectionExec::~ProjectionExec() { - for (FieldMap::const_iterator it = _fields.begin(); it != _fields.end(); ++it) { - delete it->second; - } - - for (Matchers::const_iterator it = _matchers.begin(); it != _matchers.end(); ++it) { - delete it->second; - } -} - -void ProjectionExec::add(const string& field, bool include) { - if (field.empty()) { // this is the field the user referred to - _include = include; - } else { - _include = !include; - - const size_t dot = field.find('.'); - const string subfield = field.substr(0, dot); - const string rest = (dot == string::npos ? "" : field.substr(dot + 1, string::npos)); - - ProjectionExec*& fm = _fields[subfield.c_str()]; - - if (nullptr == fm) { - fm = new ProjectionExec(); - } - - fm->add(rest, include); - } -} - -void ProjectionExec::add(const string& field, int skip, int limit) { - _special = true; // can't include or exclude whole object - - if (field.empty()) { // this is the field the user referred to - _skip = skip; - _limit = limit; - } else { - const size_t dot = field.find('.'); - const string subfield = field.substr(0, dot); - const string rest = (dot == string::npos ? "" : field.substr(dot + 1, string::npos)); - - ProjectionExec*& fm = _fields[subfield.c_str()]; - - if (nullptr == fm) { - fm = new ProjectionExec(); - } - - fm->add(rest, skip, limit); - } -} - -// -// Execution -// - -StatusWith<BSONObj> ProjectionExec::project(const BSONObj& in, - const boost::optional<const double> geoDistance, - Value geoNearPoint, - const BSONObj& sortKey, - const boost::optional<const double> textScore, - const int64_t recordId) const { - BSONObjBuilder bob; - MatchDetails matchDetails; - - // If it's a positional projection we need a MatchDetails. - if (projectRequiresQueryExpression()) { - matchDetails.requestElemMatchKey(); - invariant(nullptr != _queryExpression); - invariant(_queryExpression->matchesBSON(in, &matchDetails)); - } - - Status projStatus = projectHelper(in, &bob, &matchDetails); - if (!projStatus.isOK()) - return projStatus; - else - return {addMeta(std::move(bob), geoDistance, geoNearPoint, sortKey, textScore, recordId)}; -} - -StatusWith<BSONObj> ProjectionExec::projectCovered(const std::vector<IndexKeyDatum>& keyData, - const boost::optional<const double> geoDistance, - Value geoNearPoint, - const BSONObj& sortKey, - const boost::optional<const double> textScore, - const int64_t recordId) const { - invariant(!_include); - BSONObjBuilder bob; - // Go field by field. - if (_includeID) { - boost::optional<BSONElement> elt; - // Sometimes the _id field doesn't exist... - if ((elt = IndexKeyDatum::getFieldDotted(keyData, "_id")) && !elt->eoo()) { - bob.appendAs(elt.get(), "_id"); - } - } - - mmb::Document projectedDoc; - - for (auto&& specElt : _source) { - if (specElt.fieldNameStringData() == "_id") { - continue; - } - - // $meta sortKey is the only meta-projection which is allowed to operate on index keys - // rather than the full document. - auto metaIt = _meta.find(specElt.fieldName()); - if (metaIt != _meta.end()) { - invariant(metaIt->second == META_SORT_KEY); - continue; - } - - // $meta sortKey is also the only element with an Object value in the projection spec - // that can operate on index keys rather than the full document. - invariant(BSONType::Object != specElt.type()); - - boost::optional<BSONElement> keyElt; - // We can project a field that doesn't exist. We just ignore it. - if ((keyElt = IndexKeyDatum::getFieldDotted(keyData, specElt.fieldName())) && - !keyElt->eoo()) { - FieldRef projectedFieldPath{specElt.fieldNameStringData()}; - auto setElementStatus = - pathsupport::setElementAtPath(projectedFieldPath, keyElt.get(), &projectedDoc); - if (!setElementStatus.isOK()) { - return setElementStatus; - } - } - } - - bob.appendElements(projectedDoc.getObject()); - return {addMeta(std::move(bob), geoDistance, geoNearPoint, sortKey, textScore, recordId)}; -} - -BSONObj ProjectionExec::addMeta(BSONObjBuilder bob, - const boost::optional<const double> geoDistance, - Value geoNearPoint, - const BSONObj& sortKey, - const boost::optional<const double> textScore, - const int64_t recordId) const { - for (MetaMap::const_iterator it = _meta.begin(); it != _meta.end(); ++it) { - switch (it->second) { - case META_GEONEAR_DIST: - invariant(geoDistance); - bob.append(it->first, geoDistance.get()); - break; - case META_GEONEAR_POINT: { - invariant(!geoNearPoint.missing()); - geoNearPoint.addToBsonObj(&bob, it->first); - break; - } - case META_TEXT_SCORE: - invariant(textScore); - bob.append(it->first, textScore.get()); - break; - case META_SORT_KEY: { - invariant(!sortKey.isEmpty()); - bob.append(it->first, sortKey); - break; - } - case META_RECORDID: - invariant(recordId != 0); - bob.append(it->first, recordId); - } - } - return bob.obj(); -} - -Status ProjectionExec::projectHelper(const BSONObj& in, - BSONObjBuilder* bob, - const MatchDetails* details) const { - const ArrayOpType& arrayOpType = _arrayOpType; - - BSONObjIterator it(in); - while (it.more()) { - BSONElement elt = it.next(); - - // Case 1: _id - if ("_id" == elt.fieldNameStringData()) { - if (_includeID) { - bob->append(elt); - } - continue; - } - - // Case 2: no array projection for this field. - Matchers::const_iterator matcher = _matchers.find(elt.fieldName()); - if (_matchers.end() == matcher) { - Status s = append(bob, elt, details, arrayOpType); - if (!s.isOK()) { - return s; - } - continue; - } - - // Case 3: field has array projection with $elemMatch specified. - if (ARRAY_OP_ELEM_MATCH != arrayOpType) { - return Status(ErrorCodes::BadValue, "Matchers are only supported for $elemMatch"); - } - - MatchDetails arrayDetails; - arrayDetails.requestElemMatchKey(); - - if (matcher->second->matchesBSON(in, &arrayDetails)) { - FieldMap::const_iterator fieldIt = _fields.find(elt.fieldName()); - if (_fields.end() == fieldIt) { - return Status(ErrorCodes::BadValue, - "$elemMatch specified, but projection field not found."); - } - - BSONArrayBuilder arrBuilder; - BSONObjBuilder subBob; - - if (in.getField(elt.fieldName()).eoo()) { - return Status(ErrorCodes::InternalError, - "$elemMatch called on document element with eoo"); - } - - if (in.getField(elt.fieldName()).Obj().getField(arrayDetails.elemMatchKey()).eoo()) { - return Status(ErrorCodes::InternalError, - "$elemMatch called on array element with eoo"); - } - - arrBuilder.append( - in.getField(elt.fieldName()).Obj().getField(arrayDetails.elemMatchKey())); - subBob.appendArray(matcher->first, arrBuilder.arr()); - Status status = append(bob, subBob.done().firstElement(), details, arrayOpType); - if (!status.isOK()) { - return status; - } - } - } - - return Status::OK(); -} - -void ProjectionExec::appendArray(BSONObjBuilder* bob, const BSONObj& array, bool nested) const { - int skip = nested ? 0 : _skip; - int limit = nested ? -1 : _limit; - - if (skip < 0) { - skip = std::max(0, skip + array.nFields()); - } - - DecimalCounter<size_t> index; - BSONObjIterator it(array); - while (it.more()) { - BSONElement elt = it.next(); - - if (skip) { - skip--; - continue; - } - - if (limit != -1 && (limit-- == 0)) { - break; - } - - switch (elt.type()) { - case Array: { - BSONObjBuilder subBob; - appendArray(&subBob, elt.embeddedObject(), true); - bob->appendArray(StringData{index}, subBob.obj()); - ++index; - break; - } - case Object: { - BSONObjBuilder subBob; - BSONObjIterator jt(elt.embeddedObject()); - while (jt.more()) { - append(&subBob, jt.next()).transitional_ignore(); - } - bob->append(StringData{index}, subBob.obj()); - ++index; - break; - } - default: - if (_include) { - bob->appendAs(elt, StringData{index}); - ++index; - } - } - } -} - -Status ProjectionExec::append(BSONObjBuilder* bob, - const BSONElement& elt, - const MatchDetails* details, - const ArrayOpType arrayOpType) const { - // Skip if the field name matches a computed $meta field. - // $meta projection fields can exist at the top level of - // the result document and the field names cannot be dotted. - if (_meta.find(elt.fieldName()) != _meta.end()) { - return Status::OK(); - } - - FieldMap::const_iterator field = _fields.find(elt.fieldName()); - if (field == _fields.end()) { - if (_include) { - bob->append(elt); - } - return Status::OK(); - } - - ProjectionExec& subfm = *field->second; - if ((subfm._fields.empty() && !subfm._special) || - !(elt.type() == Object || elt.type() == Array)) { - // field map empty, or element is not an array/object - if (subfm._include) { - bob->append(elt); - } - } else if (elt.type() == Object) { - BSONObjBuilder subBob; - BSONObjIterator it(elt.embeddedObject()); - while (it.more()) { - subfm.append(&subBob, it.next(), details, arrayOpType).transitional_ignore(); - } - bob->append(elt.fieldName(), subBob.obj()); - } else { - // Array - BSONObjBuilder matchedBuilder; - if (details && arrayOpType == ARRAY_OP_POSITIONAL) { - // $ positional operator specified - if (!details->hasElemMatchKey()) { - str::stream error; - error << "positional operator (" << elt.fieldName() - << ".$) requires corresponding field" - << " in query specifier"; - return Status(ErrorCodes::BadValue, error); - } - - if (elt.embeddedObject()[details->elemMatchKey()].eoo()) { - return Status(ErrorCodes::BadValue, "positional operator element mismatch"); - } - - // append as the first and only element in the projected array - matchedBuilder.appendAs(elt.embeddedObject()[details->elemMatchKey()], "0"); - } else { - // append exact array; no subarray matcher specified - subfm.appendArray(&matchedBuilder, elt.embeddedObject()); - } - bob->appendArray(elt.fieldName(), matchedBuilder.obj()); - } - - return Status::OK(); -} - -} // namespace mongo diff --git a/src/mongo/db/exec/projection_exec.h b/src/mongo/db/exec/projection_exec.h deleted file mode 100644 index 784fe2fb209..00000000000 --- a/src/mongo/db/exec/projection_exec.h +++ /dev/null @@ -1,261 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include "boost/optional.hpp" - -#include "mongo/db/exec/document_value/value.h" -#include "mongo/db/exec/working_set.h" -#include "mongo/db/jsobj.h" -#include "mongo/db/matcher/expression.h" -#include "mongo/db/matcher/expression_parser.h" -#include "mongo/util/string_map.h" - -namespace mongo { - -class CollatorInterface; - -/** - * A fully-featured executor for find projection. - */ -class ProjectionExec { -public: - /** - * A .find() projection can have an array operation, either an elemMatch or positional (or - * neither). - */ - enum ArrayOpType { ARRAY_OP_NORMAL = 0, ARRAY_OP_ELEM_MATCH, ARRAY_OP_POSITIONAL }; - - /** - * Projections based on data computed while answering a query, or other metadata about a - * document / query. - */ - enum MetaProjection { - META_GEONEAR_DIST, - META_GEONEAR_POINT, - META_RECORDID, - META_SORT_KEY, - META_TEXT_SCORE, - }; - - /** - * TODO: document why we like StringMap so much here - */ - typedef StringMap<ProjectionExec*> FieldMap; - typedef StringMap<MatchExpression*> Matchers; - typedef StringMap<MetaProjection> MetaMap; - - ProjectionExec(OperationContext* opCtx, - const BSONObj& spec, - const MatchExpression* queryExpression, - const CollatorInterface* collator); - - ~ProjectionExec(); - - /** - * Indicates whether 'sortKey' must be provided for 'computeReturnKeyProjection()' or - * 'project()'. - */ - bool needsSortKey() const { - return _needsSortKey; - } - - /** - * Indicates whether 'geoDistance' must be provided for 'project()'. - */ - bool needsGeoNearDistance() const { - return _needsGeoNearDistance; - } - - /** - * Indicates whether 'geoNearPoint' must be provided for 'project()'. - */ - bool needsGeoNearPoint() const { - return _needsGeoNearPoint; - } - - /** - * Indicates whether 'textScore' is going to be used in 'project()'. - */ - bool needsTextScore() const { - return _needsTextScore; - } - - /** - * Returns false if there are no meta fields to project. - */ - bool hasMetaFields() const { - return !_meta.empty(); - } - - /** - * Performs a projection given a BSONObj source. Meta fields must be provided if necessary. - * Their necessity can be queried via the 'needs*' functions. - */ - StatusWith<BSONObj> project(const BSONObj& in, - const boost::optional<const double> geoDistance = boost::none, - Value geoNearPoint = Value{}, - const BSONObj& sortKey = BSONObj(), - const boost::optional<const double> textScore = boost::none, - const int64_t recordId = 0) const; - - /** - * Performs a projection given index 'KeyData' to directly retrieve results. This function - * handles projections which do not qualify for the ProjectionNodeCovered fast-path but are - * still covered by indices. - */ - StatusWith<BSONObj> projectCovered( - const std::vector<IndexKeyDatum>& keyData, - const boost::optional<const double> geoDistance = boost::none, - Value geoNearPoint = Value{}, - const BSONObj& sortKey = BSONObj(), - const boost::optional<const double> textScore = boost::none, - const int64_t recordId = 0) const; - - /** - * Determines if calls to the project method require that this object was created with the full - * query expression. We may need it for MatchDetails. - */ - bool projectRequiresQueryExpression() const { - return ARRAY_OP_POSITIONAL == _arrayOpType; - } - - -private: - /** - * Adds meta fields to the end of a projection. - */ - BSONObj addMeta(BSONObjBuilder bob, - const boost::optional<const double> geoDistance, - Value geoNearPoint, - const BSONObj& sortKey, - const boost::optional<const double> textScore, - const int64_t recordId) const; - - // - // Initialization - // - - ProjectionExec() = default; - - /** - * Add 'field' as a field name that is included or excluded as part of the projection. - */ - void add(const std::string& field, bool include); - - /** - * Add 'field' as a field name that is sliced as part of the projection. - */ - void add(const std::string& field, int skip, int limit); - - // - // Execution - // - - /** - * Apply the projection that 'this' represents to the object 'in'. 'details' is the result - * of a match evaluation of the full query on the object 'in'. This is only required - * if the projection is positional. - * - * If the projection is successfully computed, returns Status::OK() and stuff the result in - * 'bob'. - * Otherwise, returns error. - */ - Status projectHelper(const BSONObj& in, - BSONObjBuilder* bob, - const MatchDetails* details = nullptr) const; - - /** - * Appends the element 'e' to the builder 'bob', possibly descending into sub-fields of 'e' - * if needed. - */ - Status append(BSONObjBuilder* bob, - const BSONElement& elt, - const MatchDetails* details = nullptr, - const ArrayOpType arrayOpType = ARRAY_OP_NORMAL) const; - - /** - * Like append, but for arrays. - * Deals with slice and calls appendArray to preserve the array-ness. - */ - void appendArray(BSONObjBuilder* bob, const BSONObj& array, bool nested = false) const; - - // True if default at this level is to include. - bool _include = true; - - // True if this level can't be skipped or included without recursing. - bool _special = false; - - // We must group projections with common prefixes together. - // TODO: benchmark std::vector<pair> vs map - // - // Projection is a rooted tree. If we have {a.b: 1, a.c: 1} we don't want to - // double-traverse the document when we're projecting it. Instead, we have an entry in - // _fields for 'a' with two sub projections: b:1 and c:1. - FieldMap _fields; - - // The raw projection spec. that is passed into init(...) - BSONObj _source; - - // Should we include the _id field? - bool _includeID = true; - - // Arguments from the $slice operator. - int _skip = 0; - int _limit = -1; - - // Used for $elemMatch and positional operator ($) - Matchers _matchers; - - // The matchers above point into BSONObjs and this is where those objs live. - std::vector<BSONObj> _elemMatchObjs; - - ArrayOpType _arrayOpType = ARRAY_OP_NORMAL; - - // The full query expression. Used when we need MatchDetails. - const MatchExpression* _queryExpression = nullptr; - - // Projections that aren't sourced from the document or index keys. - MetaMap _meta; - - // After parsing in the constructor, these fields will indicate the neccesity of metadata - // for $meta projection. - bool _needsSortKey = false; - bool _needsGeoNearDistance = false; - bool _needsGeoNearPoint = false; - bool _needsTextScore = false; - - // The collator this projection should use to compare strings. Needed for projection operators - // that perform matching (e.g. elemMatch projection). If null, the collation is a simple binary - // compare. - const CollatorInterface* _collator = nullptr; -}; - -} // namespace mongo diff --git a/src/mongo/db/exec/projection_exec_test.cpp b/src/mongo/db/exec/projection_exec_test.cpp deleted file mode 100644 index f15094fd3aa..00000000000 --- a/src/mongo/db/exec/projection_exec_test.cpp +++ /dev/null @@ -1,310 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -/** - * This file contains tests for mongo/db/exec/projection_exec.cpp - */ - -#include "boost/optional.hpp" -#include "boost/optional/optional_io.hpp" -#include <memory> - -#include "mongo/db/exec/projection_exec.h" - -#include "mongo/db/json.h" -#include "mongo/db/matcher/expression_parser.h" -#include "mongo/db/pipeline/expression_context_for_test.h" -#include "mongo/db/query/collation/collator_interface_mock.h" -#include "mongo/stdx/variant.h" -#include "mongo/unittest/unittest.h" - -using namespace mongo; -using namespace std::string_literals; - -namespace { - -/** - * Utility function to create a MatchExpression. - */ -std::unique_ptr<MatchExpression> parseMatchExpression(const BSONObj& obj) { - boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); - StatusWithMatchExpression status = MatchExpressionParser::parse(obj, std::move(expCtx)); - ASSERT_TRUE(status.isOK()); - return std::move(status.getValue()); -} - -/** - * Test encapsulation for single call to ProjectionExec::project() or - * ProjectionExec::projectCovered(). - */ -boost::optional<std::string> project( - const char* specStr, - const char* queryStr, - const stdx::variant<const char*, const IndexKeyDatum> objStrOrDatum, - const boost::optional<const CollatorInterface&> collator = boost::none, - const BSONObj& sortKey = BSONObj(), - const double textScore = 0.0) { - // Create projection exec object. - BSONObj spec = fromjson(specStr); - BSONObj query = fromjson(queryStr); - std::unique_ptr<MatchExpression> queryExpression = parseMatchExpression(query); - QueryTestServiceContext serviceCtx; - auto opCtx = serviceCtx.makeOperationContext(); - ProjectionExec exec(opCtx.get(), spec, queryExpression.get(), collator.get_ptr()); - - auto objStr = stdx::get_if<const char*>(&objStrOrDatum); - auto projected = objStr - ? exec.project(fromjson(*objStr), boost::none, Value{}, sortKey, textScore) - : exec.projectCovered({stdx::get<const IndexKeyDatum>(objStrOrDatum)}, - boost::none, - Value{}, - sortKey, - textScore); - - if (!projected.isOK()) - return boost::none; - else - return boost::make_optional(projected.getValue().toString()); -} - -// -// position $ -// - -TEST(ProjectionExecTest, TransformPositionalDollar) { - // Valid position $ projections. - ASSERT_EQ(boost::make_optional("{ a: [ 10 ] }"s), - project("{'a.$': 1}", "{a: 10}", "{a: [10, 20, 30]}")); - ASSERT_EQ(boost::make_optional("{ a: [ 20 ] }"s), - project("{'a.$': 1}", "{a: 20}", "{a: [10, 20, 30]}")); - ASSERT_EQ(boost::make_optional("{ a: [ 5 ] }"s), - project("{'a.$': 1}", "{a: {$gt: 4}}", "{a: [5]}")); - - // Invalid position $ projections. - ASSERT_EQ(boost::none, project("{'a.$': 1}", "{a: {$size: 1}}", "{a: [5]}")); -} - -// -// $elemMatch -// - -TEST(ProjectionExecTest, TransformElemMatch) { - const char* s = "{a: [{x: 1, y: 10}, {x: 1, y: 20}, {x: 2, y: 10}]}"; - - // Valid $elemMatch projections. - ASSERT_EQ(boost::make_optional("{ a: [ { x: 1, y: 10 } ] }"s), - project("{a: {$elemMatch: {x: 1}}}", "{}", s)); - ASSERT_EQ(boost::make_optional("{ a: [ { x: 1, y: 20 } ] }"s), - project("{a: {$elemMatch: {x: 1, y: 20}}}", "{}", s)); - ASSERT_EQ(boost::make_optional("{ a: [ { x: 2, y: 10 } ] }"s), - project("{a: {$elemMatch: {x: 2}}}", "{}", s)); - ASSERT_EQ(boost::make_optional("{}"s), project("{a: {$elemMatch: {x: 3}}}", "{}", s)); - - // $elemMatch on unknown field z - ASSERT_EQ(boost::make_optional("{}"s), project("{a: {$elemMatch: {z: 1}}}", "{}", s)); -} - -TEST(ProjectionExecTest, ElemMatchProjectionRespectsCollator) { - CollatorInterfaceMock collator(CollatorInterfaceMock::MockType::kReverseString); - ASSERT_EQ(boost::make_optional("{ a: [ \"zdd\" ] }"s), - project("{a: {$elemMatch: {$gte: 'abc'}}}", - "{}", - "{a: ['zaa', 'zbb', 'zdd', 'zee']}", - collator)); -} - -// -// $slice -// - -TEST(ProjectionExecTest, TransformSliceCount) { - // Valid $slice projections using format {$slice: count}. - ASSERT_EQ(boost::make_optional("{ a: [ 4, 6, 8 ] }"s), - project("{a: {$slice: -10}}", "{}", "{a: [4, 6, 8]}")); - ASSERT_EQ(boost::make_optional("{ a: [ 4, 6, 8 ] }"s), - project("{a: {$slice: -3}}", "{}", "{a: [4, 6, 8]}")); - ASSERT_EQ(boost::make_optional("{ a: [ 8 ] }"s), - project("{a: {$slice: -1}}", "{}", "{a: [4, 6, 8]}")); - ASSERT_EQ(boost::make_optional("{ a: [] }"s), - project("{a: {$slice: 0}}", "{}", "{a: [4, 6, 8]}")); - ASSERT_EQ(boost::make_optional("{ a: [ 4 ] }"s), - project("{a: {$slice: 1}}", "{}", "{a: [4, 6, 8]}")); - ASSERT_EQ(boost::make_optional("{ a: [ 4, 6, 8 ] }"s), - project("{a: {$slice: 3}}", "{}", "{a: [4, 6, 8]}")); - ASSERT_EQ(boost::make_optional("{ a: [ 4, 6, 8 ] }"s), - project("{a: {$slice: 10}}", "{}", "{a: [4, 6, 8]}")); -} - -TEST(ProjectionExecTest, TransformSliceSkipLimit) { - // Valid $slice projections using format {$slice: [skip, limit]}. - // Non-positive limits are rejected at the query parser and therefore not handled by - // the projection execution stage. In fact, it will abort on an invalid limit. - ASSERT_EQ(boost::make_optional("{ a: [ 4, 6, 8 ] }"s), - project("{a: {$slice: [-10, 10]}}", "{}", "{a: [4, 6, 8]}")); - ASSERT_EQ(boost::make_optional("{ a: [ 4, 6, 8 ] }"s), - project("{a: {$slice: [-3, 5]}}", "{}", "{a: [4, 6, 8]}")); - ASSERT_EQ(boost::make_optional("{ a: [ 8 ] }"s), - project("{a: {$slice: [-1, 1]}}", "{}", "{a: [4, 6, 8]}")); - ASSERT_EQ(boost::make_optional("{ a: [ 4, 6 ] }"s), - project("{a: {$slice: [0, 2]}}", "{}", "{a: [4, 6, 8]}")); - ASSERT_EQ(boost::make_optional("{ a: [ 4 ] }"s), - project("{a: {$slice: [0, 1]}}", "{}", "{a: [4, 6, 8]}")); - ASSERT_EQ(boost::make_optional("{ a: [ 6 ] }"s), - project("{a: {$slice: [1, 1]}}", "{}", "{a: [4, 6, 8]}")); - ASSERT_EQ(boost::make_optional("{ a: [] }"s), - project("{a: {$slice: [3, 5]}}", "{}", "{a: [4, 6, 8]}")); - ASSERT_EQ(boost::make_optional("{ a: [] }"s), - project("{a: {$slice: [10, 10]}}", "{}", "{a: [4, 6, 8]}")); -} - -// -// Dotted projections. -// - -TEST(ProjectionExecTest, TransformCoveredDottedProjection) { - BSONObj keyPattern = fromjson("{a: 1, 'b.c': 1, 'b.d': 1, 'b.f.g': 1, 'b.f.h': 1}"); - BSONObj keyData = fromjson("{'': 1, '': 2, '': 3, '': 4, '': 5}"); - ASSERT_EQ(boost::make_optional("{ b: { c: 2, d: 3, f: { g: 4, h: 5 } } }"s), - project("{'b.c': 1, 'b.d': 1, 'b.f.g': 1, 'b.f.h': 1}", - "{}", - IndexKeyDatum(keyPattern, keyData, 0, SnapshotId{}))); -} - -TEST(ProjectionExecTest, TransformNonCoveredDottedProjection) { - ASSERT_EQ(boost::make_optional("{ b: { c: 2, d: 3, f: { g: 4, h: 5 } } }"s), - project("{'b.c': 1, 'b.d': 1, 'b.f.g': 1, 'b.f.h': 1}", - "{}", - "{a: 1, b: {c: 2, d: 3, f: {g: 4, h: 5}}}")); -} - -// -// $meta -// $meta projections add computed values to the projected object. -// - -TEST(ProjectionExecTest, TransformMetaTextScore) { - // Query {} is ignored. - ASSERT_EQ(boost::make_optional("{ a: \"hello\", b: 100.0 }"s), - project("{b: {$meta: 'textScore'}}", - "{}", - "{a: 'hello'}", - boost::none, // collator - BSONObj(), // sortKey - 100.0)); // textScore - // Projected meta field should overwrite existing field. - ASSERT_EQ(boost::make_optional("{ a: \"hello\", b: 100.0 }"s), - project("{b: {$meta: 'textScore'}}", - "{}", - "{a: 'hello', b: -1}", - boost::none, // collator - BSONObj(), // sortKey - 100.0)); // textScore -} - -TEST(ProjectionExecTest, TransformMetaSortKey) { - // Query {} is ignored. - ASSERT_EQ(boost::make_optional("{ a: \"hello\", b: { : 99 } }"s), - project("{b: {$meta: 'sortKey'}}", - "{}", - "{a: 'hello'}", - boost::none, // collator - BSON("" << 99))); // sortKey - - // Projected meta field should overwrite existing field. - ASSERT_EQ(boost::make_optional("{ a: { : 99 } }"s), - project("{a: {$meta: 'sortKey'}}", - "{}", - "{a: 'hello'}", - boost::none, // collator - BSON("" << 99))); // sortKey -} - -TEST(ProjectionExecTest, TransformMetaSortKeyCoveredNormal) { - ASSERT_EQ(boost::make_optional("{ a: 5, b: { : 5 } }"s), - project("{_id: 0, a: 1, b: {$meta: 'sortKey'}}", - "{}", - IndexKeyDatum(BSON("a" << 1), BSON("" << 5), 0, SnapshotId{}), - boost::none, // collator - BSON("" << 5))); // sortKey -} - -TEST(ProjectionExecTest, TransformMetaSortKeyCoveredOverwrite) { - ASSERT_EQ(boost::make_optional("{ a: { : 5 } }"s), - project("{_id: 0, a: 1, a: {$meta: 'sortKey'}}", - "{}", - IndexKeyDatum(BSON("a" << 1), BSON("" << 5), 0, SnapshotId{}), - boost::none, // collator - BSON("" << 5))); // sortKey -} - -TEST(ProjectionExecTest, TransformMetaSortKeyCoveredAdditionalData) { - ASSERT_EQ(boost::make_optional("{ a: 5, c: 6, b: { : 5 } }"s), - project("{_id: 0, a: 1, b: {$meta: 'sortKey'}, c: 1}", - "{}", - IndexKeyDatum( - BSON("a" << 1 << "c" << 1), BSON("" << 5 << "" << 6), 0, SnapshotId{}), - boost::none, // collator - BSON("" << 5))); // sortKey -} - -TEST(ProjectionExecTest, TransformMetaSortKeyCoveredCompound) { - ASSERT_EQ(boost::make_optional("{ a: 5, b: { : 5, : 6 } }"s), - project("{_id: 0, a: 1, b: {$meta: 'sortKey'}}", - "{}", - IndexKeyDatum( - BSON("a" << 1 << "c" << 1), BSON("" << 5 << "" << 6), 0, SnapshotId{}), - boost::none, // collator - BSON("" << 5 << "" << 6))); // sortKey -} - -TEST(ProjectionExecTest, TransformMetaSortKeyCoveredCompound2) { - ASSERT_EQ(boost::make_optional("{ a: 5, c: 4, b: { : 5, : 6 } }"s), - project("{_id: 0, a: 1, c: 1, b: {$meta: 'sortKey'}}", - "{}", - IndexKeyDatum(BSON("a" << 1 << "b" << 1 << "c" << 1), - BSON("" << 5 << "" << 6 << "" << 4), - 0, - SnapshotId{}), - boost::none, // collator - BSON("" << 5 << "" << 6))); // sortKey -} - -TEST(ProjectionExecTest, TransformMetaSortKeyCoveredCompound3) { - ASSERT_EQ(boost::make_optional("{ c: 4, d: 9000, b: { : 6, : 4 } }"s), - project("{_id: 0, c: 1, d: 1, b: {$meta: 'sortKey'}}", - "{}", - IndexKeyDatum(BSON("a" << 1 << "b" << 1 << "c" << 1 << "d" << 1), - BSON("" << 5 << "" << 6 << "" << 4 << "" << 9000), - 0, - SnapshotId{}), - boost::none, // collator - BSON("" << 6 << "" << 4))); // sortKey -} - -} // namespace diff --git a/src/mongo/db/exec/projection_executor_test.cpp b/src/mongo/db/exec/projection_executor_test.cpp index a1ac7467510..0caa4f3d007 100644 --- a/src/mongo/db/exec/projection_executor_test.cpp +++ b/src/mongo/db/exec/projection_executor_test.cpp @@ -32,6 +32,7 @@ #include "mongo/db/exec/document_value/document_value_test_util.h" #include "mongo/db/exec/projection_executor.h" #include "mongo/db/pipeline/aggregation_context_fixture.h" +#include "mongo/db/query/collation/collator_interface_mock.h" #include "mongo/db/query/projection_ast_util.h" #include "mongo/db/query/projection_parser.h" #include "mongo/unittest/unittest.h" @@ -134,6 +135,9 @@ TEST_F(ProjectionExecutorTest, CanProjectFindPositional) { auto executor = buildProjectionExecutor(getExpCtx(), &proj, {}); ASSERT_DOCUMENT_EQ(Document{fromjson("{a: {b: [3]}}")}, executor->applyTransformation(Document{fromjson("{a: {b: [1,2,3,4]}}")})); + + ASSERT_DOCUMENT_EQ(Document{fromjson("{a: {b: [4]}}")}, + executor->applyTransformation(Document{fromjson("{a: {b: [4, 3, 2]}}")})); } TEST_F(ProjectionExecutorTest, CanProjectFindElemMatchWithInclusion) { @@ -144,6 +148,37 @@ TEST_F(ProjectionExecutorTest, CanProjectFindElemMatchWithInclusion) { executor->applyTransformation(Document{fromjson("{a: [{b: 1}, {b: 2}, {b: 3}]}")})); } +TEST_F(ProjectionExecutorTest, CanProjectFindElemMatch) { + + const BSONObj obj = fromjson("{a: [{b: 3, c: 1}, {b: 1, c: 2}, {b: 1, c: 3}]}"); + { + auto proj = parseWithDefaultPolicies(fromjson("{a: {$elemMatch: {b: 1}}}")); + auto executor = buildProjectionExecutor(getExpCtx(), &proj, {}); + ASSERT_DOCUMENT_EQ(Document{fromjson("{a: [{b: 1, c: 2}]}")}, + executor->applyTransformation(Document{obj})); + } + + { + auto proj = parseWithDefaultPolicies(fromjson("{a: {$elemMatch: {b: 1, c: 3}}}")); + auto executor = buildProjectionExecutor(getExpCtx(), &proj, {}); + ASSERT_DOCUMENT_EQ(Document{fromjson("{a: [{b: 1, c: 3}]}")}, + executor->applyTransformation(Document{obj})); + } +} + +TEST_F(ProjectionExecutorTest, ElemMatchRespectsCollator) { + CollatorInterfaceMock collator(CollatorInterfaceMock::MockType::kReverseString); + getExpCtx()->setCollator(&collator); + + auto proj = parseWithDefaultPolicies(fromjson("{a: {$elemMatch: {$gte: 'abc'}}}")); + auto executor = buildProjectionExecutor(getExpCtx(), &proj, {}); + + + ASSERT_DOCUMENT_EQ( + Document{fromjson("{ a: [ \"zdd\" ] }")}, + executor->applyTransformation(Document{fromjson("{a: ['zaa', 'zbb', 'zdd', 'zee']}")})); +} + TEST_F(ProjectionExecutorTest, CanProjectFindElemMatchWithExclusion) { auto proj = parseWithFindFeaturesEnabled(fromjson("{a: {$elemMatch: {b: {$gte: 3}}}, c: 0}")); auto executor = buildProjectionExecutor(getExpCtx(), &proj, {}); @@ -156,11 +191,27 @@ TEST_F(ProjectionExecutorTest, CanProjectFindSliceWithInclusion) { auto proj = parseWithFindFeaturesEnabled(fromjson("{'a.b': {$slice: [1,2]}, c: 1}")); auto executor = buildProjectionExecutor(getExpCtx(), &proj, {}); ASSERT_DOCUMENT_EQ( + Document{fromjson("{a: {b: [1,2,3]}, c: 'abc'}")}, + executor->applyTransformation(Document{fromjson("{a: {b: [1,2,3]}, c: 'abc'}")})); +} + +TEST_F(ProjectionExecutorTest, CanProjectFindSliceSkipLimitWithInclusion) { + auto proj = parseWithFindFeaturesEnabled(fromjson("{'a.b': {$slice: [1,2]}, c: 1}")); + auto executor = buildProjectionExecutor(getExpCtx(), &proj, {}); + ASSERT_DOCUMENT_EQ( Document{fromjson("{a: {b: [2,3]}, c: 'abc'}")}, executor->applyTransformation(Document{fromjson("{a: {b: [1,2,3,4]}, c: 'abc'}")})); } -TEST_F(ProjectionExecutorTest, CanProjectFindSliceWithExclusion) { +TEST_F(ProjectionExecutorTest, CanProjectFindSliceBasicWithExclusion) { + auto proj = parseWithFindFeaturesEnabled(fromjson("{'a.b': {$slice: 3}, c: 0}")); + auto executor = buildProjectionExecutor(getExpCtx(), &proj, {}); + ASSERT_DOCUMENT_EQ( + Document{fromjson("{a: {b: [1,2,3]}}")}, + executor->applyTransformation(Document{fromjson("{a: {b: [1,2,3,4]}, c: 'abc'}")})); +} + +TEST_F(ProjectionExecutorTest, CanProjectFindSliceSkipLimitWithExclusion) { auto proj = parseWithFindFeaturesEnabled(fromjson("{'a.b': {$slice: [1,2]}, c: 0}")); auto executor = buildProjectionExecutor(getExpCtx(), &proj, {}); ASSERT_DOCUMENT_EQ( diff --git a/src/mongo/db/query/SConscript b/src/mongo/db/query/SConscript index c687aea2b82..4e21c1b2417 100644 --- a/src/mongo/db/query/SConscript +++ b/src/mongo/db/query/SConscript @@ -20,7 +20,6 @@ env.Library( "canonical_query.cpp", "canonical_query_encoder.cpp", "index_tag.cpp", - "parsed_projection.cpp", "plan_cache.cpp", "plan_cache_indexability.cpp", "plan_enumerator.cpp", @@ -265,7 +264,6 @@ env.CppUnitTest( "lru_key_value_test.cpp", 'map_reduce_output_format_test.cpp', "parsed_distinct_test.cpp", - "parsed_projection_test.cpp", "plan_cache_indexability_test.cpp", "plan_cache_test.cpp", "planner_access_test.cpp", diff --git a/src/mongo/db/query/canonical_query.cpp b/src/mongo/db/query/canonical_query.cpp index 5722bc90f6b..9f4795cecca 100644 --- a/src/mongo/db/query/canonical_query.cpp +++ b/src/mongo/db/query/canonical_query.cpp @@ -240,7 +240,6 @@ Status CanonicalQuery::init(OperationContext* opCtx, // Validate the projection if there is one. if (!_qr->getProj().isEmpty()) { - Status newParserStatus = Status::OK(); try { _proj.emplace(projection_ast::parse(expCtx, _qr->getProj(), @@ -248,31 +247,7 @@ Status CanonicalQuery::init(OperationContext* opCtx, _qr->getFilter(), ProjectionPolicies::findProjectionPolicies())); } catch (const DBException& e) { - newParserStatus = e.toStatus(); - } - - ParsedProjection* pp = nullptr; - Status projStatus = ParsedProjection::make(opCtx, _qr->getProj(), _root.get(), &pp); - - std::unique_ptr<ParsedProjection> projDeleter(pp); - pp = nullptr; - - // The query system is in the process of migrating from one projection - // implementation/language to another. If there's a projection that the old parser rejects - // but the new parser accepts, then the client is attempting to use a feature only available - // as part of the new language, so we fail to parse. - if (newParserStatus.isOK() && !projStatus.isOK()) { - return projStatus.withContext(str::stream() - << "projection " << _qr->getProj() - << " is supported by new parser but not the old parser"); - } - - if (!projStatus.isOK()) { - return projStatus; - } - - if (!newParserStatus.isOK()) { - return newParserStatus; + return e.toStatus(); } _metadataDeps = _proj->metadataDeps(); diff --git a/src/mongo/db/query/canonical_query.h b/src/mongo/db/query/canonical_query.h index 317544c4944..59d83b4d6a6 100644 --- a/src/mongo/db/query/canonical_query.h +++ b/src/mongo/db/query/canonical_query.h @@ -31,12 +31,10 @@ #include "mongo/base/status.h" -#include "mongo/db/dbmessage.h" #include "mongo/db/jsobj.h" #include "mongo/db/matcher/expression.h" #include "mongo/db/matcher/extensions_callback_noop.h" #include "mongo/db/query/collation/collator_interface.h" -#include "mongo/db/query/parsed_projection.h" #include "mongo/db/query/projection.h" #include "mongo/db/query/query_request.h" diff --git a/src/mongo/db/query/get_executor.cpp b/src/mongo/db/query/get_executor.cpp index f610b940db4..8f0623def21 100644 --- a/src/mongo/db/query/get_executor.cpp +++ b/src/mongo/db/query/get_executor.cpp @@ -659,20 +659,6 @@ StatusWith<unique_ptr<PlanStage>> applyProjection(OperationContext* opCtx, cq->getQueryObj(), ProjectionPolicies::findProjectionPolicies()); - { - // The query system is in the process of migrating from one projection - // implementation/language to another. If there's a projection that the old parser rejects - // but the new parser accepts, then the client is attempting to use a feature only available - // as part of the new language, so we fail to parse. - // TODO SERVER-42423: Remove this. - ParsedProjection* rawParsedProj; - Status ppStatus = ParsedProjection::make(opCtx, projObj, cq->root(), &rawParsedProj); - if (!ppStatus.isOK()) { - return ppStatus; - } - std::unique_ptr<ParsedProjection> projDeleter(rawParsedProj); - } - // ProjectionExec requires the MatchDetails from the query expression when the projection // uses the positional operator. Since the query may no longer match the newly-updated // document, we forbid this case. diff --git a/src/mongo/db/query/parsed_projection.cpp b/src/mongo/db/query/parsed_projection.cpp deleted file mode 100644 index 165a27e66bb..00000000000 --- a/src/mongo/db/query/parsed_projection.cpp +++ /dev/null @@ -1,401 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/query/parsed_projection.h" - -#include "mongo/bson/simple_bsonobj_comparator.h" -#include "mongo/db/query/query_request.h" - -namespace mongo { - -using std::string; -using std::unique_ptr; - -/** - * Parses the projection 'spec' and checks its validity with respect to the query 'query'. - * Puts covering information into 'out'. - * - * Does not take ownership of 'query'. - * - * Returns Status::OK() if it's a valid spec. - * Returns a Status indicating how it's invalid otherwise. - */ -// static -Status ParsedProjection::make(OperationContext* opCtx, - const BSONObj& spec, - const MatchExpression* const query, - ParsedProjection** out) { - // Whether we're including or excluding fields. - enum class IncludeExclude { kUninitialized, kInclude, kExclude }; - IncludeExclude includeExclude = IncludeExclude::kUninitialized; - - bool requiresDocument = false; - bool wantTextScore = false; - bool wantGeoNearPoint = false; - bool wantGeoNearDistance = false; - bool wantSortKey = false; - - // Until we see a positional or elemMatch operator we're normal. - ArrayOpType arrayOpType = ARRAY_OP_NORMAL; - - // Fill out the returned obj. - unique_ptr<ParsedProjection> pp(new ParsedProjection()); - pp->_hasId = true; - - for (auto&& elem : spec) { - if (Object == elem.type()) { - BSONObj obj = elem.embeddedObject(); - if (1 != obj.nFields()) { - return Status(ErrorCodes::BadValue, ">1 field in obj: " + obj.toString()); - } - - BSONElement e2 = obj.firstElement(); - if (e2.fieldNameStringData() == "$slice") { - if (e2.isNumber()) { - // This is A-OK. - } else if (e2.type() == Array) { - BSONObj arr = e2.embeddedObject(); - if (2 != arr.nFields()) { - return Status(ErrorCodes::BadValue, "$slice array wrong size"); - } - - BSONObjIterator it(arr); - // Skip over 'skip'. - it.next(); - int limit = it.next().numberInt(); - if (limit <= 0) { - return Status(ErrorCodes::BadValue, "$slice limit must be positive"); - } - } else { - return Status(ErrorCodes::BadValue, - "$slice only supports numbers and [skip, limit] arrays"); - } - - // Projections with $slice aren't covered. - requiresDocument = true; - pp->_arrayFields.push_back(elem.fieldNameStringData()); - } else if (e2.fieldNameStringData() == "$elemMatch") { - // Validate $elemMatch arguments and dependencies. - if (Object != e2.type()) { - return Status(ErrorCodes::BadValue, - "elemMatch: Invalid argument, object required."); - } - - if (ARRAY_OP_POSITIONAL == arrayOpType) { - return Status(ErrorCodes::BadValue, - "Cannot specify positional operator and $elemMatch."); - } - - if (str::contains(elem.fieldName(), '.')) { - return Status(ErrorCodes::BadValue, - "Cannot use $elemMatch projection on a nested field."); - } - - arrayOpType = ARRAY_OP_ELEM_MATCH; - - // Create a MatchExpression for the elemMatch. - BSONObj elemMatchObj = elem.wrap(); - invariant(elemMatchObj.isOwned()); - - // We pass a null pointer instead of threading through the CollatorInterface. This - // is ok because the parsed MatchExpression is not used after being created. We are - // only parsing here in order to ensure that the elemMatch projection is valid. - // - // Match expression extensions such as $text, $where, $geoNear, $near, and - // $nearSphere are not allowed in $elemMatch projections. $expr and $jsonSchema are - // not allowed because the matcher is not applied to the root of the document. - const CollatorInterface* collator = nullptr; - boost::intrusive_ptr<ExpressionContext> expCtx( - new ExpressionContext(opCtx, collator)); - StatusWithMatchExpression statusWithMatcher = - MatchExpressionParser::parse(elemMatchObj, - std::move(expCtx), - ExtensionsCallbackNoop(), - MatchExpressionParser::kBanAllSpecialFeatures); - if (!statusWithMatcher.isOK()) { - return statusWithMatcher.getStatus(); - } - - // Projections with $elemMatch aren't covered. - requiresDocument = true; - pp->_arrayFields.push_back(elem.fieldNameStringData()); - } else if (e2.fieldNameStringData() == "$meta") { - // Field for meta must be top level. We can relax this at some point. - if (str::contains(elem.fieldName(), '.')) { - return Status(ErrorCodes::BadValue, "field for $meta cannot be nested"); - } - - // Make sure the argument to $meta is something we recognize. - // e.g. {x: {$meta: "textScore"}} - if (String != e2.type()) { - return Status(ErrorCodes::BadValue, "unexpected argument to $meta in proj"); - } - - if (e2.valuestr() != QueryRequest::metaTextScore && - e2.valuestr() != QueryRequest::metaRecordId && - e2.valuestr() != QueryRequest::metaGeoNearDistance && - e2.valuestr() != QueryRequest::metaGeoNearPoint && - e2.valuestr() != QueryRequest::metaSortKey) { - return Status(ErrorCodes::BadValue, "unsupported $meta operator: " + e2.str()); - } - - // This clobbers everything else. - if (e2.valuestr() == QueryRequest::metaTextScore) { - wantTextScore = true; - } else if (e2.valuestr() == QueryRequest::metaGeoNearDistance) { - wantGeoNearDistance = true; - } else if (e2.valuestr() == QueryRequest::metaGeoNearPoint) { - wantGeoNearPoint = true; - } else if (e2.valuestr() == QueryRequest::metaSortKey) { - wantSortKey = true; - } - - // Of the $meta projections, only sortKey can be covered. - if (e2.valuestr() != QueryRequest::metaSortKey) { - requiresDocument = true; - } - pp->_metaFields.push_back(elem.fieldNameStringData()); - } else { - return Status(ErrorCodes::BadValue, - string("Unsupported projection option: ") + elem.toString()); - } - } else if ((elem.fieldNameStringData() == "_id") && !elem.trueValue()) { - pp->_hasId = false; - } else { - pp->_hasDottedFieldPath = pp->_hasDottedFieldPath || - elem.fieldNameStringData().find('.') != std::string::npos; - - if (elem.trueValue()) { - pp->_includedFields.push_back(elem.fieldNameStringData()); - } else { - pp->_excludedFields.push_back(elem.fieldNameStringData()); - } - - // If we haven't specified an include/exclude, initialize includeExclude. We expect - // further include/excludes to match it. - if (includeExclude == IncludeExclude::kUninitialized) { - includeExclude = - elem.trueValue() ? IncludeExclude::kInclude : IncludeExclude::kExclude; - } else if ((includeExclude == IncludeExclude::kInclude && !elem.trueValue()) || - (includeExclude == IncludeExclude::kExclude && elem.trueValue())) { - return Status(ErrorCodes::BadValue, - "Projection cannot have a mix of inclusion and exclusion."); - } - } - - if (_isPositionalOperator(elem.fieldName())) { - // Validate the positional op. - if (!elem.trueValue()) { - return Status(ErrorCodes::BadValue, - "Cannot exclude array elements with the positional operator."); - } - - if (ARRAY_OP_POSITIONAL == arrayOpType) { - return Status(ErrorCodes::BadValue, - "Cannot specify more than one positional proj. per query."); - } - - if (ARRAY_OP_ELEM_MATCH == arrayOpType) { - return Status(ErrorCodes::BadValue, - "Cannot specify positional operator and $elemMatch."); - } - - StringData after = str::after(elem.fieldNameStringData(), ".$"); - if (after.find(".$"_sd) != std::string::npos) { - str::stream ss; - ss << "Positional projection '" << elem.fieldName() << "' contains " - << "the positional operator more than once."; - return Status(ErrorCodes::BadValue, ss); - } - - StringData matchfield = str::before(elem.fieldNameStringData(), '.'); - if (query && !_hasPositionalOperatorMatch(query, matchfield)) { - str::stream ss; - ss << "Positional projection '" << elem.fieldName() << "' does not " - << "match the query document."; - return Status(ErrorCodes::BadValue, ss); - } - - arrayOpType = ARRAY_OP_POSITIONAL; - pp->_arrayFields.push_back(elem.fieldNameStringData()); - } - } - - // If includeExclude is uninitialized or set to exclude fields, then we can't use an index - // because we don't know what fields we're missing. - if (includeExclude == IncludeExclude::kUninitialized || - includeExclude == IncludeExclude::kExclude) { - requiresDocument = true; - } - - pp->_isInclusionProjection = (includeExclude == IncludeExclude::kInclude); - - // The positional operator uses the MatchDetails from the query - // expression to know which array element was matched. - pp->_requiresMatchDetails = arrayOpType == ARRAY_OP_POSITIONAL; - - // Save the raw spec. It should be owned by the QueryRequest. - verify(spec.isOwned()); - pp->_source = spec; - pp->_requiresDocument = requiresDocument; - - // Add meta-projections. - pp->_wantTextScore = wantTextScore; - pp->_wantGeoNearPoint = wantGeoNearPoint; - pp->_wantGeoNearDistance = wantGeoNearDistance; - pp->_wantSortKey = wantSortKey; - - // If it's possible to compute the projection in a covered fashion, populate _requiredFields - // so the planner can perform projection analysis. - if (!pp->_requiresDocument) { - if (pp->_hasId) { - pp->_requiredFields.push_back("_id"); - } - - // The only way we could be here is if spec is only simple non-dotted-field inclusions or - // the $meta sortKey projection. Therefore we can iterate over spec to get the fields - // required. - BSONObjIterator srcIt(spec); - while (srcIt.more()) { - BSONElement elt = srcIt.next(); - // We've already handled the _id field before entering this loop. - if (pp->_hasId && (elt.fieldNameStringData() == "_id")) { - continue; - } - // $meta sortKey should not be checked as a part of _requiredFields, since it can - // potentially produce a covered projection as long as the sort key is covered. - if (BSONType::Object == elt.type()) { - dassert(SimpleBSONObjComparator::kInstance.evaluate(elt.Obj() == - BSON("$meta" - << "sortKey"))); - continue; - } - if (elt.trueValue()) { - pp->_requiredFields.push_back(elt.fieldName()); - } - } - } - - *out = pp.release(); - return Status::OK(); -} - -namespace { - -bool isPrefixOf(StringData first, StringData second) { - if (first.size() >= second.size()) { - return false; - } - - return second.startsWith(first) && second[first.size()] == '.'; -} - -} // namespace - -bool ParsedProjection::isFieldRetainedExactly(StringData path) const { - // If a path, or a parent or child of the path, is contained in _metaFields or in _arrayFields, - // our output likely does not preserve that field. - for (auto&& metaField : _metaFields) { - if (path == metaField || isPrefixOf(path, metaField) || isPrefixOf(metaField, path)) { - return false; - } - } - - for (auto&& arrayField : _arrayFields) { - if (path == arrayField || isPrefixOf(path, arrayField) || isPrefixOf(arrayField, path)) { - return false; - } - } - - if (path == "_id" || isPrefixOf("_id", path)) { - return _hasId; - } - - if (!_isInclusionProjection) { - // If we are an exclusion projection, and the path, or a parent or child of the path, is - // contained in _excludedFields, our output likely does not preserve that field. - for (auto&& excluded : _excludedFields) { - if (path == excluded || isPrefixOf(excluded, path) || isPrefixOf(path, excluded)) { - return false; - } - } - } else { - // If we are an inclusion projection, we may include parents of this path, but we cannot - // include children. - bool fieldIsIncluded = false; - // In a projection with several statements, the last one takes precedence. For example, the - // projection {a: 1, a.b: 1} preserves 'a.b', but not 'a'. - // TODO SERVER-6527: Simplify this when projections are no longer order-dependent. - for (auto&& included : _includedFields) { - if (path == included || isPrefixOf(included, path)) { - fieldIsIncluded = true; - } else if (isPrefixOf(path, included)) { - fieldIsIncluded = false; - } - } - - if (!fieldIsIncluded) { - return false; - } - } - - - return true; -} - -// static -bool ParsedProjection::_isPositionalOperator(const char* fieldName) { - return str::contains(fieldName, ".$") && !str::contains(fieldName, ".$ref") && - !str::contains(fieldName, ".$id") && !str::contains(fieldName, ".$db"); -} - -// static -bool ParsedProjection::_hasPositionalOperatorMatch(const MatchExpression* const query, - StringData matchfield) { - if (query->getCategory() == MatchExpression::MatchCategory::kLogical) { - for (unsigned int i = 0; i < query->numChildren(); ++i) { - if (_hasPositionalOperatorMatch(query->getChild(i), matchfield)) { - return true; - } - } - } else { - StringData queryPath = query->path(); - // We have to make a distinction between match expressions that are - // initialized with an empty field/path name "" and match expressions - // for which the path is not meaningful (eg. $where). - if (!queryPath.rawData()) { - return false; - } - StringData pathPrefix = str::before(queryPath, '.'); - return pathPrefix == matchfield; - } - return false; -} - -} // namespace mongo diff --git a/src/mongo/db/query/parsed_projection.h b/src/mongo/db/query/parsed_projection.h deleted file mode 100644 index 1d9063acf05..00000000000 --- a/src/mongo/db/query/parsed_projection.h +++ /dev/null @@ -1,198 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include "mongo/db/jsobj.h" -#include "mongo/db/matcher/expression_parser.h" -#include "mongo/util/str.h" - -namespace mongo { - -class ParsedProjection { -public: - // TODO: this is duplicated in here and in the proj exec code. When we have - // ProjectionExpression we can remove dups. - enum ArrayOpType { ARRAY_OP_NORMAL = 0, ARRAY_OP_ELEM_MATCH, ARRAY_OP_POSITIONAL }; - - /** - * Parses the projection 'spec' and checks its validity with respect to the query 'query'. - * Puts covering information into 'out'. - * - * Returns Status::OK() if it's a valid spec. - * Returns a Status indicating how it's invalid otherwise. - */ - static Status make(OperationContext* opCtx, - const BSONObj& spec, - const MatchExpression* const query, - ParsedProjection** out); - - /** - * Returns true if the projection requires match details from the query, - * and false otherwise. - */ - bool requiresMatchDetails() const { - return _requiresMatchDetails; - } - - /** - * Is the full document required to compute this projection? - */ - bool requiresDocument() const { - return _requiresDocument; - } - - /** - * If requiresDocument() == false, what fields are required to compute - * the projection? - * - * Returned StringDatas are owned by, and have the lifetime of, the ParsedProjection. - */ - const std::vector<StringData>& getRequiredFields() const { - return _requiredFields; - } - - /** - * Get the raw BSONObj proj spec obj - */ - const BSONObj& getProjObj() const { - return _source; - } - - bool wantTextScore() const { - return _wantTextScore; - } - - /** - * Does the projection want geoNear metadata? If so any geoNear stage should include them. - */ - bool wantGeoNearDistance() const { - return _wantGeoNearDistance; - } - - bool wantGeoNearPoint() const { - return _wantGeoNearPoint; - } - - bool wantSortKey() const { - return _wantSortKey; - } - - /** - * Returns true if the element at 'path' is preserved entirely after this projection is applied, - * and false otherwise. For example, the projection {a: 1} will preserve the element located at - * 'a.b', and the projection {'a.b': 0} will not preserve the element located at 'a'. - */ - bool isFieldRetainedExactly(StringData path) const; - - /** - * Returns true if the project contains any paths with multiple path pieces (e.g. returns true - * for {_id: 0, "a.b": 1} and returns false for {_id: 0, a: 1, b: 1}). - */ - bool hasDottedFieldPath() const { - return _hasDottedFieldPath; - } - -private: - /** - * Must go through ::make - */ - ParsedProjection() = default; - - /** - * Returns true if field name refers to a positional projection. - */ - static bool _isPositionalOperator(const char* fieldName); - - /** - * Returns true if the MatchExpression 'query' queries against - * the field named by 'matchfield'. This deeply traverses logical - * nodes in the matchfield and returns true if any of the children - * have the field (so if 'query' is {$and: [{a: 1}, {b: 1}]} and - * 'matchfield' is "b", the return value is true). - * - * Does not take ownership of 'query'. - */ - static bool _hasPositionalOperatorMatch(const MatchExpression* const query, - StringData matchfield); - - // Track fields needed by the projection so that the query planner can perform projection - // analysis and possibly give us a covered projection. - // - // StringDatas are owned by the ParsedProjection. - // - // The order of the fields is the order they were in the projection object. - std::vector<StringData> _requiredFields; - - // _hasId determines whether the _id field of the input is included in the output. - bool _hasId = false; - - // Tracks the fields that have been explicitly included and excluded, respectively, in this - // projection. - // - // StringDatas are owned by the ParsedProjection. - // - // The ordering of the paths is the order that they appeared within the projection, and should - // be maintained. - std::vector<StringData> _includedFields; - std::vector<StringData> _excludedFields; - - // Tracks fields referenced within the projection that are meta or array projections, - // respectively. - // - // StringDatas are owned by the ParsedProjection. - // - // The order of the fields is not significant. - std::vector<StringData> _metaFields; - std::vector<StringData> _arrayFields; - - // Tracks whether this projection is an inclusion projection, i.e., {a: 1}, or an exclusion - // projection, i.e., {a: 0}. The projection {_id: 0} is ambiguous but will result in this field - // being set to false. - bool _isInclusionProjection = false; - - bool _requiresMatchDetails = false; - - bool _requiresDocument = true; - - BSONObj _source; - - bool _wantTextScore = false; - - bool _wantGeoNearDistance = false; - - bool _wantGeoNearPoint = false; - - // Whether this projection includes a sortKey meta-projection. - bool _wantSortKey = false; - - bool _hasDottedFieldPath = false; -}; - -} // namespace mongo diff --git a/src/mongo/db/query/parsed_projection_test.cpp b/src/mongo/db/query/parsed_projection_test.cpp deleted file mode 100644 index 8994723a62d..00000000000 --- a/src/mongo/db/query/parsed_projection_test.cpp +++ /dev/null @@ -1,481 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/query/parsed_projection.h" - -#include "mongo/db/json.h" -#include "mongo/db/matcher/expression_always_boolean.h" -#include "mongo/db/matcher/expression_parser.h" -#include "mongo/db/query/query_test_service_context.h" -#include "mongo/unittest/unittest.h" -#include <memory> - -namespace { - -using std::string; -using std::unique_ptr; -using std::vector; - -using namespace mongo; - -// -// creation function -// - -unique_ptr<ParsedProjection> createParsedProjection(const BSONObj& query, const BSONObj& projObj) { - QueryTestServiceContext serviceCtx; - auto opCtx = serviceCtx.makeOperationContext(); - const CollatorInterface* collator = nullptr; - const boost::intrusive_ptr<ExpressionContext> expCtx( - new ExpressionContext(opCtx.get(), collator)); - StatusWithMatchExpression statusWithMatcher = - MatchExpressionParser::parse(query, std::move(expCtx)); - ASSERT(statusWithMatcher.isOK()); - std::unique_ptr<MatchExpression> queryMatchExpr = std::move(statusWithMatcher.getValue()); - ParsedProjection* out = nullptr; - Status status = ParsedProjection::make(opCtx.get(), projObj, queryMatchExpr.get(), &out); - if (!status.isOK()) { - FAIL(str::stream() << "failed to parse projection " << projObj << " (query: " << query - << "): " << status.toString()); - } - ASSERT(out); - return unique_ptr<ParsedProjection>(out); -} - -unique_ptr<ParsedProjection> createParsedProjection(const char* queryStr, const char* projStr) { - BSONObj query = fromjson(queryStr); - BSONObj projObj = fromjson(projStr); - return createParsedProjection(query, projObj); -} - -// -// Failure to create a parsed projection is expected -// - -void assertInvalidProjection(const char* queryStr, const char* projStr) { - BSONObj query = fromjson(queryStr); - BSONObj projObj = fromjson(projStr); - QueryTestServiceContext serviceCtx; - auto opCtx = serviceCtx.makeOperationContext(); - const CollatorInterface* collator = nullptr; - const boost::intrusive_ptr<ExpressionContext> expCtx( - new ExpressionContext(opCtx.get(), collator)); - StatusWithMatchExpression statusWithMatcher = - MatchExpressionParser::parse(query, std::move(expCtx)); - ASSERT(statusWithMatcher.isOK()); - std::unique_ptr<MatchExpression> queryMatchExpr = std::move(statusWithMatcher.getValue()); - ParsedProjection* out = nullptr; - Status status = ParsedProjection::make(opCtx.get(), projObj, queryMatchExpr.get(), &out); - std::unique_ptr<ParsedProjection> destroy(out); - ASSERT(!status.isOK()); -} - -// canonical_query.cpp will invoke ParsedProjection::make only when -// the projection spec is non-empty. This test case is included for -// completeness and do not reflect actual usage. -TEST(ParsedProjectionTest, MakeId) { - unique_ptr<ParsedProjection> parsedProj(createParsedProjection("{}", "{}")); - ASSERT(parsedProj->requiresDocument()); -} - -TEST(ParsedProjectionTest, MakeEmpty) { - unique_ptr<ParsedProjection> parsedProj(createParsedProjection("{}", "{_id: 0}")); - ASSERT(parsedProj->requiresDocument()); -} - -TEST(ParsedProjectionTest, MakeSingleField) { - unique_ptr<ParsedProjection> parsedProj(createParsedProjection("{}", "{a: 1}")); - ASSERT(!parsedProj->requiresDocument()); - const vector<StringData>& fields = parsedProj->getRequiredFields(); - ASSERT_EQUALS(fields.size(), 2U); - ASSERT_EQUALS(fields[0], "_id"); - ASSERT_EQUALS(fields[1], "a"); -} - -TEST(ParsedProjectionTest, MakeSingleFieldCovered) { - unique_ptr<ParsedProjection> parsedProj(createParsedProjection("{}", "{_id: 0, a: 1}")); - ASSERT(!parsedProj->requiresDocument()); - const vector<StringData>& fields = parsedProj->getRequiredFields(); - ASSERT_EQUALS(fields.size(), 1U); - ASSERT_EQUALS(fields[0], "a"); -} - -TEST(ParsedProjectionTest, MakeSingleFieldIDCovered) { - unique_ptr<ParsedProjection> parsedProj(createParsedProjection("{}", "{_id: 1}")); - ASSERT(!parsedProj->requiresDocument()); - const vector<StringData>& fields = parsedProj->getRequiredFields(); - ASSERT_EQUALS(fields.size(), 1U); - ASSERT_EQUALS(fields[0], "_id"); -} - -// boolean support is undocumented -TEST(ParsedProjectionTest, MakeSingleFieldCoveredBoolean) { - unique_ptr<ParsedProjection> parsedProj(createParsedProjection("{}", "{_id: 0, a: true}")); - ASSERT(!parsedProj->requiresDocument()); - const vector<StringData>& fields = parsedProj->getRequiredFields(); - ASSERT_EQUALS(fields.size(), 1U); - ASSERT_EQUALS(fields[0], "a"); -} - -// boolean support is undocumented -TEST(ParsedProjectionTest, MakeSingleFieldCoveredIdBoolean) { - unique_ptr<ParsedProjection> parsedProj(createParsedProjection("{}", "{_id: false, a: 1}")); - ASSERT(!parsedProj->requiresDocument()); - const vector<StringData>& fields = parsedProj->getRequiredFields(); - ASSERT_EQUALS(fields.size(), 1U); - ASSERT_EQUALS(fields[0], "a"); -} - -// -// Positional operator validation -// - -TEST(ParsedProjectionTest, InvalidPositionalOperatorProjections) { - assertInvalidProjection("{}", "{'a.$': 1}"); - assertInvalidProjection("{a: 1}", "{'b.$': 1}"); - assertInvalidProjection("{a: 1}", "{'a.$': 0}"); - assertInvalidProjection("{a: 1}", "{'a.$.d.$': 1}"); - assertInvalidProjection("{a: 1}", "{'a.$.$': 1}"); - assertInvalidProjection("{a: 1}", "{'a.$.$': 1}"); - assertInvalidProjection("{a: 1, b: 1, c: 1}", "{'abc.$': 1}"); - assertInvalidProjection("{$or: [{a: 1}, {$or: [{b: 1}, {c: 1}]}]}", "{'d.$': 1}"); - assertInvalidProjection("{a: [1, 2, 3]}", "{'.$': 1}"); -} - -TEST(ParsedProjectionTest, InvalidElemMatchTextProjection) { - assertInvalidProjection("{}", "{a: {$elemMatch: {$text: {$search: 'str'}}}}"); -} - -TEST(ParsedProjectionTest, InvalidElemMatchWhereProjection) { - assertInvalidProjection("{}", "{a: {$elemMatch: {$where: 'this.a == this.b'}}}"); -} - -TEST(ParsedProjectionTest, InvalidElemMatchGeoNearProjection) { - assertInvalidProjection( - "{}", - "{a: {$elemMatch: {$nearSphere: {$geometry: {type: 'Point', coordinates: [0, 0]}}}}}"); -} - -TEST(ParsedProjectionTest, InvalidElemMatchExprProjection) { - assertInvalidProjection("{}", "{a: {$elemMatch: {$expr: 5}}}"); -} - -TEST(ParsedProjectionTest, ValidPositionalOperatorProjections) { - createParsedProjection("{a: 1}", "{'a.$': 1}"); - createParsedProjection("{a: 1}", "{'a.foo.bar.$': 1}"); - createParsedProjection("{a: 1}", "{'a.foo.bar.$.x.y': 1}"); - createParsedProjection("{'a.b.c': 1}", "{'a.b.c.$': 1}"); - createParsedProjection("{'a.b.c': 1}", "{'a.e.f.$': 1}"); - createParsedProjection("{a: {b: 1}}", "{'a.$': 1}"); - createParsedProjection("{a: 1, b: 1}}", "{'a.$': 1}"); - createParsedProjection("{a: 1, b: 1}}", "{'b.$': 1}"); - createParsedProjection("{$and: [{a: 1}, {b: 1}]}", "{'a.$': 1}"); - createParsedProjection("{$and: [{a: 1}, {b: 1}]}", "{'b.$': 1}"); - createParsedProjection("{$or: [{a: 1}, {b: 1}]}", "{'a.$': 1}"); - createParsedProjection("{$or: [{a: 1}, {b: 1}]}", "{'b.$': 1}"); - createParsedProjection("{$and: [{$or: [{a: 1}, {$and: [{b: 1}, {c: 1}]}]}]}", "{'c.d.f.$': 1}"); - // Fields with empty name can be projected using the positional $ operator. - createParsedProjection("{'': [1, 2, 3]}", "{'.$': 1}"); -} - -// Some match expressions (eg. $where) do not override MatchExpression::path() -// In this test case, we use an internal match expression implementation ALWAYS_FALSE -// to achieve the same effect. -// Projection parser should handle this the same way as an empty path. -TEST(ParsedProjectionTest, InvalidPositionalProjectionDefaultPathMatchExpression) { - QueryTestServiceContext serviceCtx; - auto opCtx = serviceCtx.makeOperationContext(); - unique_ptr<MatchExpression> queryMatchExpr(new AlwaysFalseMatchExpression()); - ASSERT(nullptr == queryMatchExpr->path().rawData()); - - ParsedProjection* out = nullptr; - BSONObj projObj = fromjson("{'a.$': 1}"); - Status status = ParsedProjection::make(opCtx.get(), projObj, queryMatchExpr.get(), &out); - ASSERT(!status.isOK()); - std::unique_ptr<ParsedProjection> destroy(out); - - // Projecting onto empty field should fail. - BSONObj emptyFieldProjObj = fromjson("{'.$': 1}"); - status = ParsedProjection::make(opCtx.get(), emptyFieldProjObj, queryMatchExpr.get(), &out); - ASSERT(!status.isOK()); -} - -TEST(ParsedProjectionTest, ParsedProjectionDefaults) { - auto parsedProjection = createParsedProjection("{}", "{}"); - - ASSERT_FALSE(parsedProjection->wantSortKey()); - ASSERT_TRUE(parsedProjection->requiresDocument()); - ASSERT_FALSE(parsedProjection->requiresMatchDetails()); - ASSERT_FALSE(parsedProjection->wantGeoNearDistance()); - ASSERT_FALSE(parsedProjection->wantGeoNearPoint()); -} - -TEST(ParsedProjectionTest, SortKeyMetaProjection) { - auto parsedProjection = createParsedProjection("{}", "{foo: {$meta: 'sortKey'}}"); - - ASSERT_BSONOBJ_EQ(parsedProjection->getProjObj(), fromjson("{foo: {$meta: 'sortKey'}}")); - ASSERT_TRUE(parsedProjection->wantSortKey()); - ASSERT_TRUE(parsedProjection->requiresDocument()); - - ASSERT_FALSE(parsedProjection->requiresMatchDetails()); - ASSERT_FALSE(parsedProjection->wantGeoNearDistance()); - ASSERT_FALSE(parsedProjection->wantGeoNearPoint()); -} - -TEST(ParsedProjectionTest, SortKeyMetaProjectionCovered) { - auto parsedProjection = createParsedProjection("{}", "{a: 1, foo: {$meta: 'sortKey'}, _id: 0}"); - - ASSERT_BSONOBJ_EQ(parsedProjection->getProjObj(), - fromjson("{a: 1, foo: {$meta: 'sortKey'}, _id: 0}")); - ASSERT_TRUE(parsedProjection->wantSortKey()); - - ASSERT_FALSE(parsedProjection->requiresDocument()); - ASSERT_FALSE(parsedProjection->requiresMatchDetails()); - ASSERT_FALSE(parsedProjection->wantGeoNearDistance()); - ASSERT_FALSE(parsedProjection->wantGeoNearPoint()); -} - -TEST(ParsedProjectionTest, SortKeyMetaAndSlice) { - auto parsedProjection = - createParsedProjection("{}", "{a: 1, foo: {$meta: 'sortKey'}, _id: 0, b: {$slice: 1}}"); - - ASSERT_BSONOBJ_EQ(parsedProjection->getProjObj(), - fromjson("{a: 1, foo: {$meta: 'sortKey'}, _id: 0, b: {$slice: 1}}")); - ASSERT_TRUE(parsedProjection->wantSortKey()); - ASSERT_TRUE(parsedProjection->requiresDocument()); - - ASSERT_FALSE(parsedProjection->requiresMatchDetails()); - ASSERT_FALSE(parsedProjection->wantGeoNearDistance()); - ASSERT_FALSE(parsedProjection->wantGeoNearPoint()); -} - -TEST(ParsedProjectionTest, SortKeyMetaAndElemMatch) { - auto parsedProjection = createParsedProjection( - "{}", "{a: 1, foo: {$meta: 'sortKey'}, _id: 0, b: {$elemMatch: {a: 1}}}"); - - ASSERT_BSONOBJ_EQ(parsedProjection->getProjObj(), - fromjson("{a: 1, foo: {$meta: 'sortKey'}, _id: 0, b: {$elemMatch: {a: 1}}}")); - ASSERT_TRUE(parsedProjection->wantSortKey()); - ASSERT_TRUE(parsedProjection->requiresDocument()); - - ASSERT_FALSE(parsedProjection->requiresMatchDetails()); - ASSERT_FALSE(parsedProjection->wantGeoNearDistance()); - ASSERT_FALSE(parsedProjection->wantGeoNearPoint()); -} - -TEST(ParsedProjectionTest, SortKeyMetaAndExclusion) { - auto parsedProjection = createParsedProjection("{}", "{a: 0, foo: {$meta: 'sortKey'}, _id: 0}"); - - ASSERT_BSONOBJ_EQ(parsedProjection->getProjObj(), - fromjson("{a: 0, foo: {$meta: 'sortKey'}, _id: 0}")); - ASSERT_TRUE(parsedProjection->wantSortKey()); - ASSERT_TRUE(parsedProjection->requiresDocument()); - - ASSERT_FALSE(parsedProjection->requiresMatchDetails()); - ASSERT_FALSE(parsedProjection->wantGeoNearDistance()); - ASSERT_FALSE(parsedProjection->wantGeoNearPoint()); -} - -// -// Cases for ParsedProjection::isFieldRetainedExactly(). -// - -TEST(ParsedProjectionTest, InclusionProjectionPreservesChild) { - auto parsedProjection = createParsedProjection("{}", "{a: 1}"); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("a.b")); -} - -TEST(ParsedProjectionTest, InclusionProjectionDoesNotPreserveParent) { - auto parsedProjection = createParsedProjection("{}", "{'a.b': 1}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a")); -} - -TEST(ParsedProjectionTest, InclusionProjectionPreservesField) { - auto parsedProjection = createParsedProjection("{}", "{a: 1}"); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("a")); -} - -TEST(ParsedProjectionTest, InclusionProjectionOrderingDeterminesPreservation) { - auto parsedProjection = createParsedProjection("{}", "{a: 1, 'a.b': 1}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a")); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("a.b")); - - parsedProjection = createParsedProjection("{}", "{'a.b': 1, a: 1}"); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("a")); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("a.b")); -} - -TEST(ParsedProjectionTest, ExclusionProjectionDoesNotPreserveParent) { - auto parsedProjection = createParsedProjection("{}", "{'a.b': 0}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a")); -} - -TEST(ParsedProjectionTest, ExclusionProjectionDoesNotPreserveChild) { - auto parsedProjection = createParsedProjection("{}", "{a: 0}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a.b")); -} - -TEST(ParsedProjectionTest, ExclusionProjectionDoesNotPreserveField) { - auto parsedProjection = createParsedProjection("{}", "{a: 0}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a")); -} - -TEST(ParsedProjectionTest, InclusionProjectionDoesNotPreserveNonIncludedFields) { - auto parsedProjection = createParsedProjection("{}", "{a: 1}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("c")); -} - -TEST(ParsedProjectionTest, ExclusionProjectionPreservesNonExcludedFields) { - auto parsedProjection = createParsedProjection("{}", "{a: 0}"); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("c")); -} - -TEST(ParsedProjectionTest, PositionalProjectionDoesNotPreserveField) { - auto parsedProjection = createParsedProjection("{a: {$elemMatch: {$eq: 0}}}", "{'a.$': 1}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a")); -} - -TEST(ParsedProjectionTest, PositionalProjectionDoesNotPreserveChild) { - auto parsedProjection = createParsedProjection("{a: {$elemMatch: {$eq: 0}}}", "{'a.$': 1}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a.b")); -} - -TEST(ParsedProjectionTest, PositionalProjectionDoesNotPreserveParent) { - auto parsedProjection = - createParsedProjection("{'a.b': {$elemMatch: {$eq: 0}}}", "{'a.b.$': 1}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a")); -} - -TEST(ParsedProjectionTest, MetaProjectionDoesNotPreserveField) { - auto parsedProjection = createParsedProjection("{}", "{a: {$meta: 'textScore'}}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a")); -} - -TEST(ParsedProjectionTest, MetaProjectionDoesNotPreserveChild) { - auto parsedProjection = createParsedProjection("{}", "{a: {$meta: 'textScore'}}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a.b")); -} - -TEST(ParsedProjectionTest, IdExclusionProjectionPreservesOtherFields) { - auto parsedProjection = createParsedProjection("{}", "{_id: 0}"); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("a")); -} - -TEST(ParsedProjectionTest, IdInclusionProjectionDoesNotPreserveOtherFields) { - auto parsedProjection = createParsedProjection("{}", "{_id: 1}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a")); -} - -TEST(ParsedProjectionTest, IdSubfieldExclusionProjectionPreservesId) { - auto parsedProjection = createParsedProjection("{}", "{'_id.a': 0}"); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("b")); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("_id")); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("_id.a")); -} - -TEST(ParsedProjectionTest, IdSubfieldInclusionProjectionPreservesId) { - auto parsedProjection = createParsedProjection("{}", "{'_id.a': 1}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("b")); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("_id")); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("_id.a")); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("_id.b")); -} - -TEST(ParsedProjectionTest, IdExclusionWithExclusionProjectionDoesNotPreserveId) { - auto parsedProjection = createParsedProjection("{}", "{_id: 0, a: 0}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("_id")); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("b")); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a")); -} - -TEST(ParsedProjectionTest, IdInclusionWithInclusionProjectionPreservesId) { - auto parsedProjection = createParsedProjection("{}", "{_id: 1, a: 1}"); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("_id")); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("b")); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("a")); -} - -TEST(ParsedProjectionTest, IdExclusionWithInclusionProjectionDoesNotPreserveId) { - auto parsedProjection = createParsedProjection("{}", "{_id: 0, a: 1}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("_id")); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("b")); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("a")); -} - -TEST(ParsedProjectionTest, PositionalProjectionDoesNotPreserveFields) { - auto parsedProjection = createParsedProjection("{a: 1}", "{'a.$': 1}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("b")); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a")); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a.b")); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("_id")); -} - -TEST(ParsedProjectionTest, PositionalProjectionWithIdExclusionDoesNotPreserveFields) { - auto parsedProjection = createParsedProjection("{a: 1}", "{_id: 0, 'a.$': 1}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("b")); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a")); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a.b")); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("_id")); -} - -TEST(ParsedProjectionTest, PositionalProjectionWithIdInclusionPreservesId) { - auto parsedProjection = createParsedProjection("{a: 1}", "{_id: 1, 'a.$': 1}"); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("b")); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a")); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("a.b")); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("_id")); -} - -TEST(ParsedProjectionTest, ProjectionOfFieldSimilarToIdIsNotSpecial) { - auto parsedProjection = createParsedProjection("{}", "{_idimpostor: 0}"); - ASSERT_TRUE(parsedProjection->isFieldRetainedExactly("_id")); - ASSERT_FALSE(parsedProjection->isFieldRetainedExactly("_idimpostor")); -} - -// -// DBRef projections -// - -TEST(ParsedProjectionTest, DBRefProjections) { - // non-dotted - createParsedProjection(BSONObj(), BSON("$ref" << 1)); - createParsedProjection(BSONObj(), BSON("$id" << 1)); - createParsedProjection(BSONObj(), BSON("$ref" << 1)); - // dotted before - createParsedProjection("{}", "{'a.$ref': 1}"); - createParsedProjection("{}", "{'a.$id': 1}"); - createParsedProjection("{}", "{'a.$db': 1}"); - // dotted after - createParsedProjection("{}", "{'$id.a': 1}"); - // position operator on $id - // $ref and $db hold the collection and database names respectively, - // so these fields cannot be arrays. - createParsedProjection("{'a.$id': {$elemMatch: {x: 1}}}", "{'a.$id.$': 1}"); -} -} // unnamed namespace diff --git a/src/mongo/db/query/projection_ast_test.cpp b/src/mongo/db/query/projection_ast_test.cpp index ac3592a8d3b..4afb6a3e98a 100644 --- a/src/mongo/db/query/projection_ast_test.cpp +++ b/src/mongo/db/query/projection_ast_test.cpp @@ -475,6 +475,8 @@ TEST_F(ProjectionASTTest, ParserErrorsOnPositionalProjectionNotMatchingQuery) { ASSERT_THROWS_CODE(parseWithFindFeaturesEnabled(fromjson("{'a.$': 1}"), fromjson("{b: 1}")), DBException, 31277); + ASSERT_THROWS_CODE( + parseWithFindFeaturesEnabled(fromjson("{'a.$': 1}"), boost::none), DBException, 51050); } TEST_F(ProjectionASTTest, ParserErrorsOnSubfieldPrefixedByDbRefField) { diff --git a/src/mongo/db/query/projection_parser.cpp b/src/mongo/db/query/projection_parser.cpp index 727ec0df089..08e2b987d67 100644 --- a/src/mongo/db/query/projection_parser.cpp +++ b/src/mongo/db/query/projection_parser.cpp @@ -337,10 +337,11 @@ void parseInclusion(ParseContext* ctx, StringData matchField = fullPathToParent ? fullPathToParent->front() : str::before(elem.fieldNameStringData(), '.'); + uassert(51050, "Projections with a positional operator require a matcher", ctx->query); uassert(31277, str::stream() << "Positional projection '" << elem.fieldName() << "' does not " << "match the query document.", - ctx->query && hasPositionalOperatorMatch(ctx->query, matchField)); + hasPositionalOperatorMatch(ctx->query, matchField)); // Check that the path does not end with ".$." which can be interpreted as the // positional projection. diff --git a/src/mongo/db/query/projection_test.cpp b/src/mongo/db/query/projection_test.cpp index 12e33792114..c1381fb7c4d 100644 --- a/src/mongo/db/query/projection_test.cpp +++ b/src/mongo/db/query/projection_test.cpp @@ -29,8 +29,6 @@ #include <memory> -#include "mongo/db/query/parsed_projection.h" - #include "mongo/db/json.h" #include "mongo/db/matcher/expression_always_boolean.h" #include "mongo/db/matcher/expression_parser.h" diff --git a/src/mongo/embedded/stitch_support/stitch_support.cpp b/src/mongo/embedded/stitch_support/stitch_support.cpp index 3ec1908891b..40257b99f7e 100644 --- a/src/mongo/embedded/stitch_support/stitch_support.cpp +++ b/src/mongo/embedded/stitch_support/stitch_support.cpp @@ -35,11 +35,12 @@ #include "mongo/base/initializer.h" #include "mongo/bson/bsonobj.h" #include "mongo/db/client.h" -#include "mongo/db/exec/projection_exec.h" +#include "mongo/db/exec/projection_executor.h" #include "mongo/db/matcher/matcher.h" #include "mongo/db/ops/parsed_update.h" #include "mongo/db/query/collation/collator_factory_interface.h" -#include "mongo/db/query/parsed_projection.h" +#include "mongo/db/query/projection.h" +#include "mongo/db/query/projection_parser.h" #include "mongo/db/service_context.h" #include "mongo/db/update/update_driver.h" #include "mongo/util/assert_util.h" @@ -139,21 +140,6 @@ struct ServiceContextDestructor { }; using EmbeddedServiceContextPtr = std::unique_ptr<mongo::ServiceContext, ServiceContextDestructor>; - -ProjectionExec makeProjectionExecChecked(OperationContext* opCtx, - const BSONObj& spec, - const MatchExpression* queryExpression, - const CollatorInterface* collator) { - /** - * ParsedProjction::make performs necessary checks to ensure a projection spec is valid however - * we are not interested in the ParsedProjection object it produces. - */ - ParsedProjection* dummy; - uassertStatusOK(ParsedProjection::make(opCtx, spec, queryExpression, &dummy)); - delete dummy; - return ProjectionExec(opCtx, spec, queryExpression, collator); -} - } // namespace } // namespace mongo @@ -192,27 +178,34 @@ struct stitch_support_v1_projection { const mongo::BSONObj& pattern, stitch_support_v1_matcher* matcher, stitch_support_v1_collator* collator) - : client(std::move(client)), - opCtx(this->client->makeOperationContext()), - projectionExec(mongo::makeProjectionExecChecked( - opCtx.get(), - pattern.getOwned(), - matcher ? matcher->matcher.getMatchExpression() : nullptr, - collator ? collator->collator.get() : nullptr)), - matcher(matcher) { - uassert(51050, - "Projections with a positional operator require a matcher", - matcher || !projectionExec.projectRequiresQueryExpression()); + : client(std::move(client)), opCtx(this->client->makeOperationContext()), matcher(matcher) { + + auto expCtx = mongo::make_intrusive<mongo::ExpressionContext>( + opCtx.get(), collator ? collator->collator.get() : nullptr); + const auto policies = mongo::ProjectionPolicies::findProjectionPolicies(); + auto proj = + mongo::projection_ast::parse(expCtx, + pattern, + matcher ? matcher->matcher.getMatchExpression() : nullptr, + matcher ? *matcher->matcher.getQuery() : mongo::BSONObj(), + policies); + uassert(51051, "$textScore, $sortKey, $recordId and $geoNear are not allowed in this " "context", - !projectionExec.hasMetaFields()); + !proj.metadataDeps().any()); + + this->requiresMatch = proj.requiresMatchDetails(); + this->projectionExec = + mongo::projection_executor::buildProjectionExecutor(expCtx, &proj, policies); } mongo::ServiceContext::UniqueClient client; mongo::ServiceContext::UniqueOperationContext opCtx; - mongo::ProjectionExec projectionExec; + std::unique_ptr<mongo::parsed_aggregation_projection::ParsedAggregationProjection> + projectionExec; + bool requiresMatch = false; stitch_support_v1_matcher* matcher; }; @@ -513,8 +506,8 @@ stitch_support_v1_projection_apply(stitch_support_v1_projection* const projectio return enterCXX(mongo::getStatusImpl(status), [&]() { mongo::BSONObj document(mongo::fromInterfaceType(documentBSON)); - auto outputResult = projection->projectionExec.project(document); - auto outputObj = uassertStatusOK(outputResult); + auto outputObj = + projection->projectionExec->applyTransformation(mongo::Document{document}).toBson(); auto outputSize = static_cast<size_t>(outputObj.objsize()); auto output = new (std::nothrow) char[outputSize]; @@ -530,7 +523,7 @@ stitch_support_v1_projection_apply(stitch_support_v1_projection* const projectio bool MONGO_API_CALL stitch_support_v1_projection_requires_match(stitch_support_v1_projection* const projection) { return [projection]() noexcept { - return projection->projectionExec.projectRequiresQueryExpression(); + return projection->requiresMatch; } (); } diff --git a/src/mongo/embedded/stitch_support/stitch_support_test.cpp b/src/mongo/embedded/stitch_support/stitch_support_test.cpp index 21cd1229fc2..c336cdc55b1 100644 --- a/src/mongo/embedded/stitch_support/stitch_support_test.cpp +++ b/src/mongo/embedded/stitch_support/stitch_support_test.cpp @@ -433,8 +433,9 @@ TEST_F(StitchSupportTest, CheckProjectionProducesExpectedStatus) { checkProjectionStatus("{'a.$.c': 1}", "{_id: 1, a: [{b: 2, c: 100}, {b: 1, c: 200}]}")); ASSERT_EQ("$textScore, $sortKey, $recordId and $geoNear are not allowed in this context", checkProjectionStatus("{a: {$meta: 'textScore'}}", "{_id: 1, a: 100, b: 200}")); - ASSERT_EQ("Unsupported projection option: a: { b: 0 }", - checkProjectionStatus("{a: {b: 0}}", "{_id: 1, a: {b: 200}}")); + + ASSERT_EQ("Cannot do inclusion on field c in exclusion projection", + checkProjectionStatus("{a: 0, c: 1}", "{_id: 1, a: {b: 200}}")); } TEST_F(StitchSupportTest, CheckProjectionCollatesRespectfully) { |