diff options
-rw-r--r-- | buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml | 2 | ||||
-rw-r--r-- | buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml | 2 | ||||
-rw-r--r-- | jstests/core/update_arrayFilters.js | 113 | ||||
-rw-r--r-- | src/mongo/db/commands/find_and_modify.cpp | 1 | ||||
-rw-r--r-- | src/mongo/db/ops/SConscript | 20 | ||||
-rw-r--r-- | src/mongo/db/ops/array_filter.cpp | 121 | ||||
-rw-r--r-- | src/mongo/db/ops/array_filter.h | 72 | ||||
-rw-r--r-- | src/mongo/db/ops/array_filter_test.cpp | 183 | ||||
-rw-r--r-- | src/mongo/db/ops/parsed_update.cpp | 31 | ||||
-rw-r--r-- | src/mongo/db/ops/parsed_update.h | 9 | ||||
-rw-r--r-- | src/mongo/db/ops/update_request.h | 41 | ||||
-rw-r--r-- | src/mongo/db/ops/write_ops.h | 1 | ||||
-rw-r--r-- | src/mongo/db/ops/write_ops_exec.cpp | 1 | ||||
-rw-r--r-- | src/mongo/db/ops/write_ops_parsers.cpp | 6 | ||||
-rw-r--r-- | src/mongo/db/ops/write_ops_parsers_test.cpp | 23 | ||||
-rw-r--r-- | src/mongo/db/query/find_and_modify_request.cpp | 51 | ||||
-rw-r--r-- | src/mongo/db/query/find_and_modify_request.h | 9 | ||||
-rw-r--r-- | src/mongo/db/query/find_and_modify_request_test.cpp | 37 |
18 files changed, 716 insertions, 7 deletions
diff --git a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml index 9aa7fb56f7c..af187f8de38 100644 --- a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml @@ -73,6 +73,8 @@ selector: - jstests/core/mr1.js - jstests/core/mr3.js - jstests/core/mr4.js + # TODO SERVER-28577 mongos doesn't support arrayFilters option to update and findAndModify. + - jstests/core/update_arrayFilters.js exclude_with_any_tags: # Tests tagged with the following will fail because they assume collections are not sharded. - assumes_no_implicit_collection_creation_after_drop diff --git a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml index 9791e68b1ad..e525d11f69a 100644 --- a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml @@ -48,6 +48,8 @@ selector: # TODO: SERVER-27269: mongos can't establish cursor if view has $collStats and views another view. - jstests/core/views/views_coll_stats.js - jstests/core/killop_drop_collection.js # Uses fsyncLock. + # TODO SERVER-28577 mongos doesn't support arrayFilters option to update and findAndModify. + - jstests/core/update_arrayFilters.js executor: diff --git a/jstests/core/update_arrayFilters.js b/jstests/core/update_arrayFilters.js new file mode 100644 index 00000000000..ccee3c696db --- /dev/null +++ b/jstests/core/update_arrayFilters.js @@ -0,0 +1,113 @@ +// Tests for the arrayFilters option to update and findAndModify. +// TODO SERVER-28576: Implement these tests in terms of the shell helpers. +(function() { + "use strict"; + + let coll = db.update_arrayFilters; + coll.drop(); + + // + // Update. + // + + // Non-array arrayFilters fail to parse. + let res = db.runCommand( + {update: coll.getName(), updates: [{q: {}, u: {$set: {a: 5}}, arrayFilters: {i: 0}}]}); + assert.commandFailedWithCode(res, ErrorCodes.TypeMismatch); + + // Non-object array filter fail to parse. + res = db.runCommand( + {update: coll.getName(), updates: [{q: {}, u: {$set: {a: 5}}, arrayFilters: ["bad"]}]}); + assert.commandFailedWithCode(res, ErrorCodes.TypeMismatch); + + // Bad array filter fails to parse. + res = db.runCommand({ + update: coll.getName(), + updates: [{q: {}, u: {$set: {a: 5}}, arrayFilters: [{i: 0, j: 0}]}] + }); + assert(res.hasOwnProperty("writeErrors"), tojson(res)); + assert.eq(res.writeErrors.length, 1, tojson(res.writeErrors)); + assert.writeError(res.writeErrors[0], ErrorCodes.FailedToParse); + assert.neq(-1, + res.writeErrors[0].errmsg.indexOf( + "Each array filter must use a single top-level field name"), + "update failed for a reason other than failing to parse array filters"); + + // Multiple array filters with the same id fails to parse. + res = db.runCommand({ + update: coll.getName(), + updates: [{q: {}, u: {$set: {a: 5}}, arrayFilters: [{i: 0}, {j: 0}, {i: 1}]}] + }); + assert(res.hasOwnProperty("writeErrors"), tojson(res)); + assert.eq(res.writeErrors.length, 1, tojson(res.writeErrors)); + assert.writeError(res.writeErrors[0], ErrorCodes.FailedToParse); + assert.neq( + -1, + res.writeErrors[0].errmsg.indexOf( + "Found multiple array filters with the same top-level field name"), + "update failed for a reason other than multiple array filters with the same top-level field name"); + + // Good value for arrayFilters succeeds. + res = db.runCommand({ + update: coll.getName(), + updates: [{q: {}, u: {$set: {a: 5}}, arrayFilters: [{i: 0}, {j: 0}]}] + }); + assert(!res.hasOwnProperty("writeErrors"), tojson(res)); + + // + // FindAndModify. + // + + // Non-array arrayFilters fail to parse. + res = db.runCommand( + {findAndModify: coll.getName(), query: {}, update: {$set: {a: 5}}, arrayFilters: {i: 0}}); + assert.commandFailedWithCode(res, ErrorCodes.TypeMismatch); + + // Non-object array filter fail to parse. + res = db.runCommand( + {findAndModify: coll.getName(), query: {}, update: {$set: {a: 5}}, arrayFilters: ["bad"]}); + assert.commandFailedWithCode(res, ErrorCodes.TypeMismatch); + + // arrayFilters option not allowed with remove=true. + res = db.runCommand( + {findAndModify: coll.getName(), query: {}, remove: true, arrayFilters: [{i: 0}]}); + assert.commandFailedWithCode(res, ErrorCodes.FailedToParse); + assert.neq( + -1, + res.errmsg.indexOf("Cannot specify arrayFilters and remove=true"), + "findAndModify failed for a reason other than specifying arrayFilters with remove=true"); + + // Bad array filter fails to parse. + res = db.runCommand({ + findAndModify: coll.getName(), + query: {}, + update: {$set: {a: 5}}, + arrayFilters: [{i: 0, j: 0}] + }); + assert.commandFailedWithCode(res, ErrorCodes.FailedToParse); + assert.neq(-1, + res.errmsg.indexOf("Each array filter must use a single top-level field name"), + "findAndModify failed for a reason other than failing to parse array filters"); + + // Multiple array filters with the same id fails to parse. + res = db.runCommand({ + findAndModify: coll.getName(), + query: {}, + update: {$set: {a: 5}}, + arrayFilters: [{i: 0}, {j: 0}, {i: 1}] + }); + assert.commandFailedWithCode(res, ErrorCodes.FailedToParse); + assert.neq( + -1, + res.errmsg.indexOf("Found multiple array filters with the same top-level field name"), + "findAndModify failed for a reason other than multiple array filters with the same top-level field name"); + + // Good value for arrayFilters succeeds. + res = db.runCommand({ + findAndModify: coll.getName(), + query: {}, + update: {$set: {a: 5}}, + arrayFilters: [{i: 0}, {j: 0}] + }); + assert.commandWorked(res); +})();
\ No newline at end of file diff --git a/src/mongo/db/commands/find_and_modify.cpp b/src/mongo/db/commands/find_and_modify.cpp index cd66033e427..33df2bda36b 100644 --- a/src/mongo/db/commands/find_and_modify.cpp +++ b/src/mongo/db/commands/find_and_modify.cpp @@ -144,6 +144,7 @@ void makeUpdateRequest(const FindAndModifyRequest& args, requestOut->setUpdates(args.getUpdateObj()); requestOut->setSort(args.getSort()); requestOut->setCollation(args.getCollation()); + requestOut->setArrayFilters(args.getArrayFilters()); requestOut->setUpsert(args.isUpsert()); requestOut->setReturnDocs(args.shouldReturnNew() ? UpdateRequest::RETURN_NEW : UpdateRequest::RETURN_OLD); diff --git a/src/mongo/db/ops/SConscript b/src/mongo/db/ops/SConscript index 157fa1ffd99..43a97cc2c47 100644 --- a/src/mongo/db/ops/SConscript +++ b/src/mongo/db/ops/SConscript @@ -266,6 +266,25 @@ env.CppUnitTest( ) env.Library( + target='array_filter', + source=[ + 'array_filter.cpp', + ], + LIBDEPS=[ + "$BUILD_DIR/mongo/db/matcher/expressions", + ], +) + +env.CppUnitTest( + target='array_filter_test', + source='array_filter_test.cpp', + LIBDEPS=[ + 'array_filter', + '$BUILD_DIR/mongo/db/query/collation/collator_interface_mock', + ], +) + +env.Library( target="write_ops", source=[ "delete.cpp", @@ -278,6 +297,7 @@ env.Library( "write_ops_exec.cpp", ], LIBDEPS=[ + 'array_filter', 'update_driver', '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/concurrency/lock_manager', diff --git a/src/mongo/db/ops/array_filter.cpp b/src/mongo/db/ops/array_filter.cpp new file mode 100644 index 00000000000..55c42c7e0a1 --- /dev/null +++ b/src/mongo/db/ops/array_filter.cpp @@ -0,0 +1,121 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/ops/array_filter.h" + +#include "mongo/db/matcher/expression_parser.h" + +#include <boost/regex.hpp> + +namespace mongo { + +namespace { + +// The array filter must begin with a lowercase letter and contain no special characters. +boost::regex idRegex("^[a-z][a-zA-Z0-9]*$"); + +/** + * Finds the top-level field that 'expr' is over. The must be unique and not the empty string. + */ +StatusWith<StringData> parseId(MatchExpression* expr) { + if (expr->isArray() || expr->isLeaf()) { + auto firstDotPos = expr->path().find('.'); + if (firstDotPos == std::string::npos) { + return expr->path(); + } + return expr->path().substr(0, firstDotPos); + } else if (expr->isLogical()) { + if (expr->numChildren() == 0) { + return Status(ErrorCodes::FailedToParse, + "No top-level field name found in array filter."); + } + + StringData id; + for (size_t i = 0; i < expr->numChildren(); ++i) { + auto statusWithId = parseId(expr->getChild(i)); + if (!statusWithId.isOK()) { + return statusWithId.getStatus(); + } + + if (id == StringData()) { + id = statusWithId.getValue(); + continue; + } + + if (id != statusWithId.getValue()) { + return Status( + ErrorCodes::FailedToParse, + str::stream() + << "Each array filter must use a single top-level field name, found '" + << id + << "' and '" + << statusWithId.getValue() + << "'"); + } + } + return id; + } + + MONGO_UNREACHABLE; +} + +} // namespace + +// static +StatusWith<std::unique_ptr<ArrayFilter>> ArrayFilter::parse( + BSONObj rawArrayFilter, + const ExtensionsCallback& extensionsCallback, + const CollatorInterface* collator) { + StatusWithMatchExpression statusWithFilter = + MatchExpressionParser::parse(rawArrayFilter, extensionsCallback, collator); + if (!statusWithFilter.isOK()) { + return statusWithFilter.getStatus(); + } + auto filter = std::move(statusWithFilter.getValue()); + + auto statusWithId = parseId(filter.get()); + if (!statusWithId.isOK()) { + return statusWithId.getStatus(); + } + auto id = statusWithId.getValue().toString(); + if (!boost::regex_match(id, idRegex)) { + return Status(ErrorCodes::BadValue, + str::stream() + << "The top-level field name in an array filter must be an alphanumeric " + "string beginning with a lowercase letter, found '" + << id + << "'"); + } + + auto arrayFilter = stdx::make_unique<ArrayFilter>(std::move(id), std::move(filter)); + return {std::move(arrayFilter)}; +} + +} // namespace mongo diff --git a/src/mongo/db/ops/array_filter.h b/src/mongo/db/ops/array_filter.h new file mode 100644 index 00000000000..fc9685ce520 --- /dev/null +++ b/src/mongo/db/ops/array_filter.h @@ -0,0 +1,72 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/matcher/expression.h" +#include "mongo/db/matcher/extensions_callback.h" + +namespace mongo { + +/** + * A filter specifying which array elements an update modifier should apply to. For example, the + * array filter with id "i" and filter {i: 0} specifies that an update {$set: {"a.$[i]"}} should + * only apply to elements of "a" which are equal to 0. + */ +class ArrayFilter { + +public: + /** + * Parses 'rawArrayFilter' to an ArrayFilter. This succeeds if 'rawArrayFilter' is a filter over + * a single top-level field, which begins with a lowercase letter and contains no special + * characters. Otherwise, a non-OK status is returned. Callers must maintain ownership of + * 'rawArrayFilter'. + */ + static StatusWith<std::unique_ptr<ArrayFilter>> parse( + BSONObj rawArrayFilter, + const ExtensionsCallback& extensionsCallback, + const CollatorInterface* collator); + + ArrayFilter(std::string id, std::unique_ptr<MatchExpression> filter) + : _id(std::move(id)), _filter(std::move(filter)) {} + + StringData getId() const { + return _id; + } + + MatchExpression* getFilter() const { + return _filter.get(); + } + +private: + // The top-level field that _filter is over. + const std::string _id; + const std::unique_ptr<MatchExpression> _filter; +}; + +} // namespace mongo diff --git a/src/mongo/db/ops/array_filter_test.cpp b/src/mongo/db/ops/array_filter_test.cpp new file mode 100644 index 00000000000..d79ffc2e683 --- /dev/null +++ b/src/mongo/db/ops/array_filter_test.cpp @@ -0,0 +1,183 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/ops/array_filter.h" + +#include "mongo/db/json.h" +#include "mongo/db/matcher/extensions_callback_disallow_extensions.h" +#include "mongo/db/query/collation/collator_interface_mock.h" +#include "mongo/unittest/unittest.h" + +namespace { + +using namespace mongo; +using unittest::assertGet; + +TEST(ArrayFilterTest, ParseBasic) { + const CollatorInterface* collator = nullptr; + auto rawArrayFilter = fromjson("{i: 0}"); + auto arrayFilter = assertGet( + ArrayFilter::parse(rawArrayFilter, ExtensionsCallbackDisallowExtensions(), collator)); + ASSERT_EQ(arrayFilter->getId(), "i"); + ASSERT_TRUE(arrayFilter->getFilter()->matchesBSON(fromjson("{i: 0}"))); + ASSERT_FALSE(arrayFilter->getFilter()->matchesBSON(fromjson("{i: 1}"))); +} + +TEST(ArrayFilterTest, ParseDottedField) { + const CollatorInterface* collator = nullptr; + auto rawArrayFilter = fromjson("{'i.a': 0, 'i.b': 1}"); + auto arrayFilter = assertGet( + ArrayFilter::parse(rawArrayFilter, ExtensionsCallbackDisallowExtensions(), collator)); + ASSERT_EQ(arrayFilter->getId(), "i"); + ASSERT_TRUE(arrayFilter->getFilter()->matchesBSON(fromjson("{i: {a: 0, b: 1}}"))); + ASSERT_FALSE(arrayFilter->getFilter()->matchesBSON(fromjson("{i: {a: 0, b: 0}}"))); +} + +TEST(ArrayFilterTest, ParseLogicalQuery) { + const CollatorInterface* collator = nullptr; + auto rawArrayFilter = fromjson("{$and: [{i: {$gte: 0}}, {i: {$lte: 0}}]}"); + auto arrayFilter = assertGet( + ArrayFilter::parse(rawArrayFilter, ExtensionsCallbackDisallowExtensions(), collator)); + ASSERT_EQ(arrayFilter->getId(), "i"); + ASSERT_TRUE(arrayFilter->getFilter()->matchesBSON(fromjson("{i: 0}"))); + ASSERT_FALSE(arrayFilter->getFilter()->matchesBSON(fromjson("{i: 1}"))); +} + +TEST(ArrayFilterTest, ParseElemMatch) { + const CollatorInterface* collator = nullptr; + auto rawArrayFilter = fromjson("{i: {$elemMatch: {a: 0}}}"); + auto arrayFilter = assertGet( + ArrayFilter::parse(rawArrayFilter, ExtensionsCallbackDisallowExtensions(), collator)); + ASSERT_EQ(arrayFilter->getId(), "i"); + ASSERT_TRUE(arrayFilter->getFilter()->matchesBSON(fromjson("{i: [{a: 0}]}"))); + ASSERT_FALSE(arrayFilter->getFilter()->matchesBSON(fromjson("{i: [{a: 1}]}"))); +} + +TEST(ArrayFilterTest, ParseCollation) { + CollatorInterfaceMock collator(CollatorInterfaceMock::MockType::kAlwaysEqual); + auto rawArrayFilter = fromjson("{i: 'abc'}"); + auto arrayFilter = assertGet( + ArrayFilter::parse(rawArrayFilter, ExtensionsCallbackDisallowExtensions(), &collator)); + ASSERT_EQ(arrayFilter->getId(), "i"); + ASSERT_TRUE(arrayFilter->getFilter()->matchesBSON(fromjson("{i: 'cba'}"))); + ASSERT_FALSE(arrayFilter->getFilter()->matchesBSON(fromjson("{i: 0}"))); +} + +TEST(ArrayFilterTest, ParseIdContainsNumbersAndCapitals) { + const CollatorInterface* collator = nullptr; + auto rawArrayFilter = fromjson("{iA3: 0}"); + auto arrayFilter = assertGet( + ArrayFilter::parse(rawArrayFilter, ExtensionsCallbackDisallowExtensions(), collator)); + ASSERT_EQ(arrayFilter->getId(), "iA3"); + ASSERT_TRUE(arrayFilter->getFilter()->matchesBSON(fromjson("{'iA3': 0}"))); + ASSERT_FALSE(arrayFilter->getFilter()->matchesBSON(fromjson("{'iA3': 1}"))); +} + +TEST(ArrayFilterTest, BadMatchExpressionFailsToParse) { + const CollatorInterface* collator = nullptr; + auto rawArrayFilter = fromjson("{$and: 0}"); + auto status = + ArrayFilter::parse(rawArrayFilter, ExtensionsCallbackDisallowExtensions(), collator); + ASSERT_NOT_OK(status.getStatus()); +} + +TEST(ArrayFilterTest, EmptyMatchExpressionFailsToParse) { + const CollatorInterface* collator = nullptr; + auto rawArrayFilter = fromjson("{}"); + auto status = + ArrayFilter::parse(rawArrayFilter, ExtensionsCallbackDisallowExtensions(), collator); + ASSERT_NOT_OK(status.getStatus()); +} + +TEST(ArrayFilterTest, NestedEmptyMatchExpressionFailsToParse) { + const CollatorInterface* collator = nullptr; + auto rawArrayFilter = fromjson("{$or: [{i: 0}, {$and: [{}]}]}"); + auto status = + ArrayFilter::parse(rawArrayFilter, ExtensionsCallbackDisallowExtensions(), collator); + ASSERT_NOT_OK(status.getStatus()); +} + +TEST(ArrayFilterTest, EmptyFieldNameFailsToParse) { + const CollatorInterface* collator = nullptr; + auto rawArrayFilter = fromjson("{'': 0}"); + auto status = + ArrayFilter::parse(rawArrayFilter, ExtensionsCallbackDisallowExtensions(), collator); + ASSERT_NOT_OK(status.getStatus()); +} + +TEST(ArrayFilterTest, EmptyElemMatchFieldNameFailsToParse) { + const CollatorInterface* collator = nullptr; + auto rawArrayFilter = fromjson("{'': {$elemMatch: {a: 0}}}"); + auto status = + ArrayFilter::parse(rawArrayFilter, ExtensionsCallbackDisallowExtensions(), collator); + ASSERT_NOT_OK(status.getStatus()); +} + +TEST(ArrayFilterTest, EmptyTopLevelFieldNameFailsToParse) { + const CollatorInterface* collator = nullptr; + auto rawArrayFilter = fromjson("{'.i': 0}"); + auto status = + ArrayFilter::parse(rawArrayFilter, ExtensionsCallbackDisallowExtensions(), collator); + ASSERT_NOT_OK(status.getStatus()); +} + +TEST(ArrayFilterTest, MultipleTopLevelFieldsFailsToParse) { + const CollatorInterface* collator = nullptr; + auto rawArrayFilter = fromjson("{$and: [{i: 0}, {j: 0}]}"); + auto status = + ArrayFilter::parse(rawArrayFilter, ExtensionsCallbackDisallowExtensions(), collator); + ASSERT_NOT_OK(status.getStatus()); +} + +TEST(ArrayFilterTest, SpecialCharactersInFieldNameFailsToParse) { + const CollatorInterface* collator = nullptr; + auto rawArrayFilter = fromjson("{'i&': 0}"); + auto status = + ArrayFilter::parse(rawArrayFilter, ExtensionsCallbackDisallowExtensions(), collator); + ASSERT_NOT_OK(status.getStatus()); +} + +TEST(ArrayFilterTest, FieldNameStartingWithNumberFailsToParse) { + const CollatorInterface* collator = nullptr; + auto rawArrayFilter = fromjson("{'3i': 0}"); + auto status = + ArrayFilter::parse(rawArrayFilter, ExtensionsCallbackDisallowExtensions(), collator); + ASSERT_NOT_OK(status.getStatus()); +} + +TEST(ArrayFilterTest, FieldNameStartingWithCapitalFailsToParse) { + const CollatorInterface* collator = nullptr; + auto rawArrayFilter = fromjson("{'Ai': 0}"); + auto status = + ArrayFilter::parse(rawArrayFilter, ExtensionsCallbackDisallowExtensions(), collator); + ASSERT_NOT_OK(status.getStatus()); +} + +} // namespace diff --git a/src/mongo/db/ops/parsed_update.cpp b/src/mongo/db/ops/parsed_update.cpp index 6df14116c1f..a1e0a258e8a 100644 --- a/src/mongo/db/ops/parsed_update.cpp +++ b/src/mongo/db/ops/parsed_update.cpp @@ -67,11 +67,16 @@ Status ParsedUpdate::parseRequest() { _collator = std::move(collator.getValue()); } + Status status = parseArrayFilters(); + if (!status.isOK()) { + return status; + } + // We parse the update portion before the query portion because the dispostion of the update // may determine whether or not we need to produce a CanonicalQuery at all. For example, if // the update involves the positional-dollar operator, we must have a CanonicalQuery even if // it isn't required for query execution. - Status status = parseUpdate(); + status = parseUpdate(); if (!status.isOK()) return status; status = parseQuery(); @@ -138,6 +143,30 @@ Status ParsedUpdate::parseUpdate() { return _driver.parse(_request->getUpdates(), _request->isMulti()); } +Status ParsedUpdate::parseArrayFilters() { + const ExtensionsCallbackReal extensionsCallback(_opCtx, &_request->getNamespaceString()); + + for (auto rawArrayFilter : _request->getArrayFilters()) { + auto arrayFilterStatus = + ArrayFilter::parse(rawArrayFilter, extensionsCallback, _collator.get()); + if (!arrayFilterStatus.isOK()) { + return arrayFilterStatus.getStatus(); + } + auto arrayFilter = std::move(arrayFilterStatus.getValue()); + + if (_arrayFilters.find(arrayFilter->getId()) != _arrayFilters.end()) { + return Status(ErrorCodes::FailedToParse, + str::stream() + << "Found multiple array filters with the same top-level field name " + << arrayFilter->getId()); + } + + _arrayFilters[arrayFilter->getId()] = std::move(arrayFilter); + } + + return Status::OK(); +} + PlanExecutor::YieldPolicy ParsedUpdate::yieldPolicy() const { if (_request->isGod()) { return PlanExecutor::YIELD_MANUAL; diff --git a/src/mongo/db/ops/parsed_update.h b/src/mongo/db/ops/parsed_update.h index 7f844a1a166..8c448bd8e58 100644 --- a/src/mongo/db/ops/parsed_update.h +++ b/src/mongo/db/ops/parsed_update.h @@ -30,6 +30,7 @@ #include "mongo/base/disallow_copying.h" #include "mongo/base/status.h" +#include "mongo/db/ops/array_filter.h" #include "mongo/db/ops/update_driver.h" #include "mongo/db/query/collation/collator_interface.h" #include "mongo/db/query/plan_executor.h" @@ -137,6 +138,11 @@ private: */ Status parseUpdate(); + /** + * Parses the array filters portion of the update request. + */ + Status parseArrayFilters(); + // Unowned pointer to the transactional context. OperationContext* _opCtx; @@ -146,6 +152,9 @@ private: // The collator for the parsed update. Owned here. std::unique_ptr<CollatorInterface> _collator; + // The array filters for the parsed update. Owned here. + std::map<StringData, std::unique_ptr<ArrayFilter>> _arrayFilters; + // Driver for processing updates on matched documents. UpdateDriver _driver; diff --git a/src/mongo/db/ops/update_request.h b/src/mongo/db/ops/update_request.h index f6aa0e31d10..5d3bf8b9d14 100644 --- a/src/mongo/db/ops/update_request.h +++ b/src/mongo/db/ops/update_request.h @@ -109,6 +109,14 @@ public: return _updates; } + inline void setArrayFilters(const std::vector<BSONObj>& arrayFilters) { + _arrayFilters = arrayFilters; + } + + inline const std::vector<BSONObj>& getArrayFilters() const { + return _arrayFilters; + } + // Please see documentation on the private members matching these names for // explanations of the following fields. @@ -185,12 +193,30 @@ public: } const std::string toString() const { - return str::stream() << " query: " << _query << " projection: " << _proj - << " sort: " << _sort << " collation: " << _collation - << " updated: " << _updates << " god: " << _god - << " upsert: " << _upsert << " multi: " << _multi - << " fromMigration: " << _fromMigration - << " isExplain: " << _isExplain; + StringBuilder builder; + builder << " query: " << _query; + builder << " projection: " << _proj; + builder << " sort: " << _sort; + builder << " collation: " << _collation; + builder << " updates: " << _updates; + + builder << " arrayFilters: ["; + bool first = true; + for (auto arrayFilter : _arrayFilters) { + if (!first) { + builder << ", "; + } + first = false; + builder << arrayFilter; + } + builder << "]"; + + builder << " god: " << _god; + builder << " upsert: " << _upsert; + builder << " multi: " << _multi; + builder << " fromMigration: " << _fromMigration; + builder << " isExplain: " << _isExplain; + return builder.str(); } private: @@ -211,6 +237,9 @@ private: // Contains the modifiers to apply to matched objects, or a replacement document. BSONObj _updates; + // Filters to specify which array elements should be updated. + std::vector<BSONObj> _arrayFilters; + // Flags controlling the update. // God bypasses _id checking and index generation. It is only used on behalf of system diff --git a/src/mongo/db/ops/write_ops.h b/src/mongo/db/ops/write_ops.h index 0b4a5cf7f94..44f7921ce66 100644 --- a/src/mongo/db/ops/write_ops.h +++ b/src/mongo/db/ops/write_ops.h @@ -63,6 +63,7 @@ struct UpdateOp : ParsedWriteOp { BSONObj query; BSONObj update; BSONObj collation; + std::vector<BSONObj> arrayFilters; bool multi = false; bool upsert = false; }; diff --git a/src/mongo/db/ops/write_ops_exec.cpp b/src/mongo/db/ops/write_ops_exec.cpp index a29a42e4a52..355c484f4a9 100644 --- a/src/mongo/db/ops/write_ops_exec.cpp +++ b/src/mongo/db/ops/write_ops_exec.cpp @@ -499,6 +499,7 @@ static WriteResult::SingleResult performSingleUpdateOp(OperationContext* opCtx, request.setQuery(op.query); request.setCollation(op.collation); request.setUpdates(op.update); + request.setArrayFilters(op.arrayFilters); request.setMulti(op.multi); request.setUpsert(op.upsert); request.setYieldPolicy(PlanExecutor::YIELD_AUTO); // ParsedUpdate overrides this for $isolated. diff --git a/src/mongo/db/ops/write_ops_parsers.cpp b/src/mongo/db/ops/write_ops_parsers.cpp index e67de9c2e7a..1cbe9dea328 100644 --- a/src/mongo/db/ops/write_ops_parsers.cpp +++ b/src/mongo/db/ops/write_ops_parsers.cpp @@ -171,6 +171,12 @@ UpdateOp parseUpdateCommand(StringData dbName, const BSONObj& cmd) { } else if (fieldName == "collation") { checkBSONType(Object, field); update.collation = field.Obj(); + } else if (fieldName == "arrayFilters") { + checkBSONType(Array, field); + for (auto arrayFilter : field.Obj()) { + checkBSONType(Object, arrayFilter); + update.arrayFilters.push_back(arrayFilter.Obj()); + } } else if (fieldName == "multi") { checkBSONType(Bool, field); update.multi = field.Bool(); diff --git a/src/mongo/db/ops/write_ops_parsers_test.cpp b/src/mongo/db/ops/write_ops_parsers_test.cpp index 488a67ce509..8f4a7539762 100644 --- a/src/mongo/db/ops/write_ops_parsers_test.cpp +++ b/src/mongo/db/ops/write_ops_parsers_test.cpp @@ -117,6 +117,24 @@ TEST(CommandWriteOpsParsers, BadCollationFieldInDeleteDoc) { ASSERT_THROWS_CODE(parseInsertCommand("foo", cmd), UserException, ErrorCodes::FailedToParse); } +TEST(CommandWriteOpsParsers, BadArrayFiltersFieldInUpdateDoc) { + auto cmd = BSON("update" + << "bar" + << "updates" + << BSON_ARRAY("q" << BSONObj() << "u" << BSONObj() << "arrayFilters" + << "bad")); + ASSERT_THROWS_CODE(parseInsertCommand("foo", cmd), UserException, ErrorCodes::FailedToParse); +} + +TEST(CommandWriteOpsParsers, BadArrayFiltersElementInUpdateDoc) { + auto cmd = BSON( + "update" + << "bar" + << "updates" + << BSON_ARRAY("q" << BSONObj() << "u" << BSONObj() << "arrayFilters" << BSON_ARRAY("bad"))); + ASSERT_THROWS_CODE(parseInsertCommand("foo", cmd), UserException, ErrorCodes::FailedToParse); +} + TEST(CommandWriteOpsParsers, SingleInsert) { const auto ns = NamespaceString("test", "foo"); const BSONObj obj = BSON("x" << 1); @@ -155,11 +173,14 @@ TEST(CommandWriteOpsParsers, Update) { const BSONObj update = BSON("$inc" << BSON("x" << 1)); const BSONObj collation = BSON("locale" << "en_US"); + const BSONObj arrayFilter = BSON("i" << 0); for (bool upsert : {false, true}) { for (bool multi : {false, true}) { auto cmd = BSON("update" << ns.coll() << "updates" << BSON_ARRAY(BSON("q" << query << "u" << update << "collation" << collation + << "arrayFilters" + << BSON_ARRAY(arrayFilter) << "upsert" << upsert << "multi" @@ -172,6 +193,8 @@ TEST(CommandWriteOpsParsers, Update) { ASSERT_BSONOBJ_EQ(op.updates[0].query, query); ASSERT_BSONOBJ_EQ(op.updates[0].update, update); ASSERT_BSONOBJ_EQ(op.updates[0].collation, collation); + ASSERT_EQ(op.updates[0].arrayFilters.size(), 1u); + ASSERT_BSONOBJ_EQ(op.updates[0].arrayFilters[0], arrayFilter); ASSERT_EQ(op.updates[0].upsert, upsert); ASSERT_EQ(op.updates[0].multi, multi); } diff --git a/src/mongo/db/query/find_and_modify_request.cpp b/src/mongo/db/query/find_and_modify_request.cpp index e7ba7bab801..362a137b919 100644 --- a/src/mongo/db/query/find_and_modify_request.cpp +++ b/src/mongo/db/query/find_and_modify_request.cpp @@ -42,6 +42,7 @@ const char kCmdName[] = "findAndModify"; const char kQueryField[] = "query"; const char kSortField[] = "sort"; const char kCollationField[] = "collation"; +const char kArrayFiltersField[] = "arrayFilters"; const char kRemoveField[] = "remove"; const char kUpdateField[] = "update"; const char kNewField[] = "new"; @@ -49,6 +50,7 @@ const char kFieldProjectionField[] = "fields"; const char kUpsertField[] = "upsert"; const char kWriteConcernField[] = "writeConcern"; +const std::vector<BSONObj> emptyArrayFilters{}; } // unnamed namespace FindAndModifyRequest::FindAndModifyRequest(NamespaceString fullNs, BSONObj query, BSONObj updateObj) @@ -97,6 +99,14 @@ BSONObj FindAndModifyRequest::toBSON() const { builder.append(kCollationField, _collation.get()); } + if (_arrayFilters) { + BSONArrayBuilder arrayBuilder(builder.subarrayStart(kArrayFiltersField)); + for (auto arrayFilter : _arrayFilters.get()) { + arrayBuilder.append(arrayFilter); + } + arrayBuilder.doneFast(); + } + if (_shouldReturnNew) { builder.append(kNewField, _shouldReturnNew.get()); } @@ -128,6 +138,28 @@ StatusWith<FindAndModifyRequest> FindAndModifyRequest::parseFromBSON(NamespaceSt } } + std::vector<BSONObj> arrayFilters; + bool arrayFiltersSet = false; + { + BSONElement arrayFiltersElt; + Status arrayFiltersEltStatus = + bsonExtractTypedField(cmdObj, kArrayFiltersField, BSONType::Array, &arrayFiltersElt); + if (!arrayFiltersEltStatus.isOK() && (arrayFiltersEltStatus != ErrorCodes::NoSuchKey)) { + return arrayFiltersEltStatus; + } + if (arrayFiltersEltStatus.isOK()) { + arrayFiltersSet = true; + for (auto arrayFilter : arrayFiltersElt.Obj()) { + if (arrayFilter.type() != BSONType::Object) { + return {ErrorCodes::TypeMismatch, + str::stream() << "Each array filter must be an object, found " + << arrayFilter.type()}; + } + arrayFilters.push_back(arrayFilter.Obj()); + } + } + } + bool shouldReturnNew = cmdObj[kNewField].trueValue(); bool isUpsert = cmdObj[kUpsertField].trueValue(); bool isRemove = cmdObj[kRemoveField].trueValue(); @@ -151,6 +183,10 @@ StatusWith<FindAndModifyRequest> FindAndModifyRequest::parseFromBSON(NamespaceSt "Cannot specify both new=true and remove=true;" " 'remove' always returns the deleted document"}; } + + if (arrayFiltersSet) { + return {ErrorCodes::FailedToParse, "Cannot specify arrayFilters and remove=true"}; + } } FindAndModifyRequest request(std::move(fullNs), query, updateObj); @@ -158,6 +194,7 @@ StatusWith<FindAndModifyRequest> FindAndModifyRequest::parseFromBSON(NamespaceSt request.setFieldProjection(fields); request.setSort(sort); request.setCollation(collation); + request.setArrayFilters(std::move(arrayFilters)); if (!isRemove) { request.setShouldReturnNew(shouldReturnNew); @@ -179,6 +216,13 @@ void FindAndModifyRequest::setCollation(BSONObj collation) { _collation = collation.getOwned(); } +void FindAndModifyRequest::setArrayFilters(const std::vector<BSONObj>& arrayFilters) { + _arrayFilters = std::vector<BSONObj>(); + for (auto arrayFilter : arrayFilters) { + _arrayFilters->emplace_back(arrayFilter.getOwned()); + } +} + void FindAndModifyRequest::setShouldReturnNew(bool shouldReturnNew) { dassert(!_isRemove); _shouldReturnNew = shouldReturnNew; @@ -217,6 +261,13 @@ BSONObj FindAndModifyRequest::getCollation() const { return _collation.value_or(BSONObj()); } +const std::vector<BSONObj>& FindAndModifyRequest::getArrayFilters() const { + if (_arrayFilters) { + return _arrayFilters.get(); + } + return emptyArrayFilters; +} + bool FindAndModifyRequest::shouldReturnNew() const { return _shouldReturnNew.value_or(false); } diff --git a/src/mongo/db/query/find_and_modify_request.h b/src/mongo/db/query/find_and_modify_request.h index 2f023154aa4..30e512e300b 100644 --- a/src/mongo/db/query/find_and_modify_request.h +++ b/src/mongo/db/query/find_and_modify_request.h @@ -71,6 +71,7 @@ public: * query: <document>, * sort: <document>, * collation: <document>, + * arrayFilters: <array>, * remove: <boolean>, * update: <document>, * new: <boolean>, @@ -95,6 +96,7 @@ public: BSONObj getUpdateObj() const; BSONObj getSort() const; BSONObj getCollation() const; + const std::vector<BSONObj>& getArrayFilters() const; bool shouldReturnNew() const; bool isUpsert() const; bool isRemove() const; @@ -136,6 +138,12 @@ public: void setCollation(BSONObj collation); /** + * Sets the array filters for the update, which determine which array elements should be + * modified. + */ + void setArrayFilters(const std::vector<BSONObj>& arrayFilters); + + /** * Sets the write concern for this request. */ void setWriteConcern(WriteConcernOptions writeConcern); @@ -157,6 +165,7 @@ private: boost::optional<BSONObj> _fieldProjection; boost::optional<BSONObj> _sort; boost::optional<BSONObj> _collation; + boost::optional<std::vector<BSONObj>> _arrayFilters; boost::optional<bool> _shouldReturnNew; boost::optional<WriteConcernOptions> _writeConcern; diff --git a/src/mongo/db/query/find_and_modify_request_test.cpp b/src/mongo/db/query/find_and_modify_request_test.cpp index 57b3f3bacc0..617498aa37c 100644 --- a/src/mongo/db/query/find_and_modify_request_test.cpp +++ b/src/mongo/db/query/find_and_modify_request_test.cpp @@ -170,6 +170,24 @@ TEST(FindAndModifyRequest, UpdateWithCollation) { ASSERT_BSONOBJ_EQ(expectedObj, request.toBSON()); } +TEST(FindAndModifyRequest, UpdateWithArrayFilters) { + const BSONObj query(BSON("x" << 1)); + const BSONObj update(BSON("y" << 1)); + const std::vector<BSONObj> arrayFilters{BSON("i" << 0)}; + + auto request = FindAndModifyRequest::makeUpdate(NamespaceString("test.user"), query, update); + request.setArrayFilters(arrayFilters); + + BSONObj expectedObj(fromjson(R"json({ + findAndModify: 'user', + query: { x: 1 }, + update: { y: 1 }, + arrayFilters: [ { i: 0 } ] + })json")); + + ASSERT_BSONOBJ_EQ(expectedObj, request.toBSON()); +} + TEST(FindAndModifyRequest, UpdateWithWriteConcern) { const BSONObj query(BSON("x" << 1)); const BSONObj update(BSON("y" << 1)); @@ -194,6 +212,7 @@ TEST(FindAndModifyRequest, UpdateWithFullSpec) { const BSONObj sort(BSON("z" << -1)); const BSONObj collation(BSON("locale" << "en_US")); + const std::vector<BSONObj> arrayFilters{BSON("i" << 0)}; const BSONObj field(BSON("x" << 1 << "y" << 1)); const WriteConcernOptions writeConcern(2, WriteConcernOptions::SyncMode::FSYNC, 150); @@ -202,6 +221,7 @@ TEST(FindAndModifyRequest, UpdateWithFullSpec) { request.setShouldReturnNew(true); request.setSort(sort); request.setCollation(collation); + request.setArrayFilters(arrayFilters); request.setWriteConcern(writeConcern); request.setUpsert(true); @@ -213,6 +233,7 @@ TEST(FindAndModifyRequest, UpdateWithFullSpec) { fields: { x: 1, y: 1 }, sort: { z: -1 }, collation: { locale: 'en_US' }, + arrayFilters: [ { i: 0 } ], new: true, writeConcern: { w: 2, fsync: true, wtimeout: 150 } })json")); @@ -347,6 +368,7 @@ TEST(FindAndModifyRequest, ParseWithUpdateOnlyRequiredFields) { ASSERT_BSONOBJ_EQ(BSONObj(), request.getFields()); ASSERT_BSONOBJ_EQ(BSONObj(), request.getSort()); ASSERT_BSONOBJ_EQ(BSONObj(), request.getCollation()); + ASSERT_EQUALS(0u, request.getArrayFilters().size()); ASSERT_EQUALS(false, request.shouldReturnNew()); } @@ -358,6 +380,7 @@ TEST(FindAndModifyRequest, ParseWithUpdateFullSpec) { fields: { x: 1, y: 1 }, sort: { z: -1 }, collation: {locale: 'en_US' }, + arrayFilters: [ { i: 0 } ], new: true })json")); @@ -375,6 +398,8 @@ TEST(FindAndModifyRequest, ParseWithUpdateFullSpec) { ASSERT_BSONOBJ_EQ(BSON("locale" << "en_US"), request.getCollation()); + ASSERT_EQUALS(1u, request.getArrayFilters().size()); + ASSERT_BSONOBJ_EQ(BSON("i" << 0), request.getArrayFilters()[0]); ASSERT_EQUALS(true, request.shouldReturnNew()); } @@ -472,6 +497,18 @@ TEST(FindAndModifyRequest, ParseWithRemoveAndReturnNew) { ASSERT_NOT_OK(parseStatus.getStatus()); } +TEST(FindAndModifyRequest, ParseWithRemoveAndArrayFilters) { + BSONObj cmdObj(fromjson(R"json({ + findAndModify: 'user', + query: { x: 1 }, + remove: true, + arrayFilters: [ { i: 0 } ] + })json")); + + auto parseStatus = FindAndModifyRequest::parseFromBSON(NamespaceString("a.b"), cmdObj); + ASSERT_NOT_OK(parseStatus.getStatus()); +} + TEST(FindAndModifyRequest, ParseWithCollationTypeMismatch) { BSONObj cmdObj(fromjson(R"json({ query: { x: 1 }, |