diff options
Diffstat (limited to 'src/mongo')
-rw-r--r-- | src/mongo/db/commands/SConscript | 13 | ||||
-rw-r--r-- | src/mongo/db/commands/map_reduce.idl | 115 | ||||
-rw-r--r-- | src/mongo/db/commands/map_reduce_global_variable_scope.h | 61 | ||||
-rw-r--r-- | src/mongo/db/commands/map_reduce_javascript_code.h | 64 | ||||
-rw-r--r-- | src/mongo/db/commands/map_reduce_out_options.cpp | 109 | ||||
-rw-r--r-- | src/mongo/db/commands/map_reduce_out_options.h | 92 | ||||
-rw-r--r-- | src/mongo/db/commands/map_reduce_parse_test.cpp | 159 |
7 files changed, 613 insertions, 0 deletions
diff --git a/src/mongo/db/commands/SConscript b/src/mongo/db/commands/SConscript index 2d57cb8e0ed..2f59b34cefb 100644 --- a/src/mongo/db/commands/SConscript +++ b/src/mongo/db/commands/SConscript @@ -502,6 +502,19 @@ if has_option('use-cpu-profiler'): ) env.CppUnitTest( + target="map_reduce_parse_test", + source=[ + "map_reduce_out_options.cpp", + "map_reduce_parse_test.cpp", + env.Idlc('map_reduce.idl')[0], + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/db/write_concern_options', + '$BUILD_DIR/mongo/idl/idl_parser', + ] +) + +env.CppUnitTest( target="db_commands_test", source=[ "index_filter_commands_test.cpp", diff --git a/src/mongo/db/commands/map_reduce.idl b/src/mongo/db/commands/map_reduce.idl new file mode 100644 index 00000000000..ceb11f3f258 --- /dev/null +++ b/src/mongo/db/commands/map_reduce.idl @@ -0,0 +1,115 @@ +# Copyright (C) 2019-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +global: + cpp_namespace: "mongo" + cpp_includes: + - "mongo/db/commands/map_reduce_out_options.h" + - "mongo/db/commands/map_reduce_javascript_code.h" + - "mongo/db/commands/map_reduce_global_variable_scope.h" + - "mongo/db/write_concern_options.h" + +imports: + - "mongo/idl/basic_types.idl" + +types: + mapReduceOutOptionsType: + bson_serialization_type: any + description: "Holds the subdocument with out options" + cpp_type: "mongo::MapReduceOutOptions" + serializer: mongo::MapReduceOutOptions::serializeToBSON + deserializer: mongo::MapReduceOutOptions::parseFromBSON + + mapReduceJavascriptCodeType: + bson_serialization_type: any + description: "Holds Javascript code passed as command input" + cpp_type: "mongo::MapReduceJavascriptCode" + serializer: mongo::MapReduceJavascriptCode::serializeToBSON + deserializer: mongo::MapReduceJavascriptCode::parseFromBSON + + mapReduceGlobalVariableScopeType: + bson_serialization_type: any + description: "Holds a mapping of Javascript global variables" + cpp_type: "mongo::MapReduceGlobalVariableScope" + serializer: mongo::MapReduceGlobalVariableScope::serializeToBSON + deserializer: mongo::MapReduceGlobalVariableScope::parseFromBSON + + +commands: + MapReduce: + description: "The MapReduce command." + namespace: concatenate_with_db + strict: true + fields: + map: + description: "Javascript code to run as the map operation which associates a value + with a key and emits the key and value pair." + type: mapReduceJavascriptCodeType + reduce: + description: "Javascript code to run as the map operation which reduces all the + values associated with a particular key to a single value." + type: mapReduceJavascriptCodeType + out: + description: "Out Options sub-object or string representing a collection." + type: mapReduceOutOptionsType + cpp_name: outOptions + query: + description: "Query object in match language to use as a filter applied before the + map step." + type: object + optional: true + sort: + description: "Sort specification to apply before the map step." + type: object + optional: true + collation: + description: "collation specification which takes effect for 'query' and 'sort'." + type: object + optional: true + limit: + description: "Document count limit to apply before the map step." + type: safeInt64 + optional: true + finalize: + description: "Javascript code to run after the reduce operation." + type: mapReduceJavascriptCodeType + optional: true + scope: + description: "Javascript global variable mapping for map, reduce and finalize." + type: mapReduceGlobalVariableScopeType + optional: true + verbose: + description: "Specifies whether to include the timing information in the result + information." + type: bool + optional: true + bypassDocumentValidation: + description: "Causes the out portion of the operation to ignore the output + collection's document validation." + type: bool + optional: true diff --git a/src/mongo/db/commands/map_reduce_global_variable_scope.h b/src/mongo/db/commands/map_reduce_global_variable_scope.h new file mode 100644 index 00000000000..436a6c8c4f7 --- /dev/null +++ b/src/mongo/db/commands/map_reduce_global_variable_scope.h @@ -0,0 +1,61 @@ +/** + * Copyright (C) 2019-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonelement.h" +#include "mongo/bson/bsonobj.h" + +namespace mongo { + +/** + * Parsed MapReduce Global variable scope. + */ +class MapReduceGlobalVariableScope { +public: + static MapReduceGlobalVariableScope parseFromBSON(const BSONElement& element) { + uassert(ErrorCodes::BadValue, "'scope' must be an object", element.type() == Object); + return MapReduceGlobalVariableScope(element.embeddedObject()); + } + + MapReduceGlobalVariableScope() = default; + MapReduceGlobalVariableScope(const BSONObj& obj) : obj(obj.getOwned()) {} + + void serializeToBSON(StringData fieldName, BSONObjBuilder* builder) const { + builder->append(fieldName, obj); + } + +private: + // Initializers for global variables. These will be directly executed as Javascript. This is + // left as a BSONObj for that API. + BSONObj obj; +}; + +} // namespace mongo diff --git a/src/mongo/db/commands/map_reduce_javascript_code.h b/src/mongo/db/commands/map_reduce_javascript_code.h new file mode 100644 index 00000000000..0ab0752ab45 --- /dev/null +++ b/src/mongo/db/commands/map_reduce_javascript_code.h @@ -0,0 +1,64 @@ +/** + * Copyright (C) 2019-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <string> + +#include "mongo/bson/bsonelement.h" +#include "mongo/bson/bsonobj.h" + +namespace mongo { + +/** + * Code to run as a component of MapReduce. + */ +class MapReduceJavascriptCode { +public: + static MapReduceJavascriptCode parseFromBSON(const BSONElement& element) { + uassert(ErrorCodes::BadValue, + "'scope' must be a string, code or 'code with scope'", + element.type() == String || element.type() == Code || element.type() == CodeWScope); + return MapReduceJavascriptCode(element); + } + + MapReduceJavascriptCode() = default; + MapReduceJavascriptCode(const BSONElement& element) : code(element.wrap()) {} + + void serializeToBSON(StringData fieldName, BSONObjBuilder* builder) const { + (*builder) << fieldName << code[0]; + } + +private: + // We have to save a local copy of the BSON for reserialization since it's difficult to rebuild + // once it has been turned into a code type. + BSONObj code; +}; + +} // namespace mongo diff --git a/src/mongo/db/commands/map_reduce_out_options.cpp b/src/mongo/db/commands/map_reduce_out_options.cpp new file mode 100644 index 00000000000..bbcc069272b --- /dev/null +++ b/src/mongo/db/commands/map_reduce_out_options.cpp @@ -0,0 +1,109 @@ +/** + * Copyright (C) 2019-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include <string> +#include <utility> + +#include "mongo/db/commands/map_reduce_out_options.h" + +namespace mongo { + +using namespace std::string_literals; + +MapReduceOutOptions MapReduceOutOptions::parseFromBSON(const BSONElement& element) { + if (element.type() == BSONType::String) { + return MapReduceOutOptions("", element.str(), OutputType::Replace, false); + } else if (element.type() == BSONType::Object) { + const auto obj = element.embeddedObject(); + // The inline option is allowed alone. + if (const auto inMemory = obj["inline"]) { + uassert(ErrorCodes::BadValue, "'inline' must be specified alone", obj.nFields() == 1); + uassert( + ErrorCodes::BadValue, "'inline' takes only numeric '1'", inMemory.number() == 1.0); + + return MapReduceOutOptions("", "", OutputType::InMemory, false); + } + + int allowedNFields = 3; + + const auto sharded = [&]() { + if (const auto sharded = obj["sharded"]) { + uassert(ErrorCodes::BadValue, + "sharded field value must be boolean", + sharded.type() == Bool); + return sharded.boolean(); + } else { + --allowedNFields; + return false; + } + }(); + + const auto databaseName = [&]() { + if (const auto db = obj["db"]) { + uassert(ErrorCodes::BadValue, + "db field value must be string", + db.type() == BSONType::String); + return db.str(); + } else { + --allowedNFields; + return ""s; + } + }(); + + const auto [collectionName, outputType] = [&]() { + auto stringOrError = [](auto&& element) { + uassert(ErrorCodes::BadValue, + "'"s + element.fieldName() + + "' supports only string consisting of output collection name", + element.type() == BSONType::String); + return element.str(); + }; + if (const auto replace = obj["replace"]) + return std::pair{stringOrError(replace), OutputType::Replace}; + else if (const auto merge = obj["merge"]) + return std::pair{stringOrError(merge), OutputType::Merge}; + else if (const auto reduce = obj["reduce"]) + return std::pair{stringOrError(reduce), OutputType::Reduce}; + else + uasserted(ErrorCodes::BadValue, "'out' requires 'replace', 'merge' or 'reduce'"); + }(); + + uassert(ErrorCodes::BadValue, + "'out' supports only output type with collection name, optional 'sharded' and " + "optional 'db'", + obj.nFields() == allowedNFields); + + return MapReduceOutOptions( + std::move(databaseName), std::move(collectionName), outputType, sharded); + } else { + uasserted(ErrorCodes::BadValue, "'out' must be either a string or an object"); + } +} + +} // namespace mongo diff --git a/src/mongo/db/commands/map_reduce_out_options.h b/src/mongo/db/commands/map_reduce_out_options.h new file mode 100644 index 00000000000..155f48bea29 --- /dev/null +++ b/src/mongo/db/commands/map_reduce_out_options.h @@ -0,0 +1,92 @@ +/** + * Copyright (C) 2019-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <string> + +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonelement.h" +#include "mongo/bson/bsonobjbuilder.h" + +namespace mongo { + +enum class OutputType { + Replace, // Atomically replace the collection. + Merge, // Merge keys, override dups. + Reduce, // Merge keys, reduce dups. + InMemory // Only store in memory, limited in size. +}; + +/** + * Parsed MapReduce Out clause. + */ +class MapReduceOutOptions { +public: + static MapReduceOutOptions parseFromBSON(const BSONElement& element); + + MapReduceOutOptions() = default; + MapReduceOutOptions(std::string databaseName, + std::string collectionName, + const OutputType outputType, + bool sharded) + : _databaseName(std::move(databaseName)), + _collectionName(std::move(collectionName)), + _outputType(outputType), + _sharded(sharded) {} + + void serializeToBSON(StringData fieldName, BSONObjBuilder* builder) const { + BSONObjBuilder sub(builder->subobjStart(fieldName)); + switch (_outputType) { + case OutputType::InMemory: + sub.append("inline", 1); + break; + case OutputType::Replace: + sub.append("replace", _collectionName); + break; + case OutputType::Merge: + sub.append("merge", _collectionName); + break; + case OutputType::Reduce: + sub.append("reduce", _collectionName); + } + if (_databaseName != "") + sub.append("db", _databaseName); + if (_sharded) + sub.append("sharded", true); + } + +private: + std::string _databaseName; + std::string _collectionName; + OutputType _outputType; + bool _sharded; +}; + +} // namespace mongo diff --git a/src/mongo/db/commands/map_reduce_parse_test.cpp b/src/mongo/db/commands/map_reduce_parse_test.cpp new file mode 100644 index 00000000000..fa3533c19b3 --- /dev/null +++ b/src/mongo/db/commands/map_reduce_parse_test.cpp @@ -0,0 +1,159 @@ +/** + * Copyright (C) 2019-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonmisc.h" +#include "mongo/db/commands/map_reduce_gen.h" +#include "mongo/idl/idl_parser.h" +#include "mongo/util/assert_util.h" + +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { + +// The parser treats Javascript objects as black boxes so there's no need for realistic examples +// here. +constexpr auto initJavascript = "init!"_sd; +constexpr auto mapJavascript = "map!"_sd; +constexpr auto reduceJavascript = "reduce!"_sd; +constexpr auto finalizeJavascript = "finalize!"_sd; + +TEST(MapReduceParseTest, failedParse) { + auto ctx = IDLParserErrorContext("mapReduce"); + // Missing fields. + ASSERT_THROWS(MapReduce::parse(ctx, + BSON("" + << "" + << "$db" + << "db")), + DBException); + ASSERT_THROWS(MapReduce::parse(ctx, + BSON("mapReduce" + << "foo" + << "$db" + << "db")), + DBException); + ASSERT_THROWS(MapReduce::parse(ctx, + BSON("map" << mapJavascript << "reduce" << reduceJavascript + << "out" << BSON("inline" << 1) << "$db" + << "db")), + DBException); + + // Extra fields. + ASSERT_THROWS(MapReduce::parse(ctx, + BSON("mapReduce" + << "theSource" + << "map" << mapJavascript << "reduce" << reduceJavascript + << "out" << BSON("inline" << 1) << "alloy" + << "chromium steel" + << "$db" + << "db")), + DBException); + ASSERT_THROWS(MapReduce::parse(ctx, + BSON("mapReduce" + << "theSource" + << "map" << mapJavascript << "reduce" << reduceJavascript + << "out" << BSON("inline" << 1 << "notinline" << 0) << "$db" + << "db")), + DBException); +} + +TEST(MapReduceParseTest, parseOutputTypes) { + auto ctx = IDLParserErrorContext("mapReduce"); + + MapReduce::parse(ctx, + BSON("mapReduce" + << "theSource" + << "map" << mapJavascript << "reduce" << reduceJavascript << "out" + << BSON("inline" << 1) << "$db" + << "db")); + MapReduce::parse(ctx, + BSON("mapReduce" + << "theSource" + << "map" << mapJavascript << "reduce" << reduceJavascript << "out" + << "theSink" + << "$db" + << "db")); + MapReduce::parse(ctx, + BSON("mapReduce" + << "theSource" + << "map" << mapJavascript << "reduce" << reduceJavascript << "out" + << BSON("replace" + << "theSink" + << "db" + << "myDb") + << "$db" + << "db")); + MapReduce::parse(ctx, + BSON("mapReduce" + << "theSource" + << "map" << mapJavascript << "reduce" << reduceJavascript << "out" + << BSON("merge" + << "theSink") + << "$db" + << "db")); + MapReduce::parse(ctx, + BSON("mapReduce" + << "theSource" + << "map" << mapJavascript << "reduce" << reduceJavascript << "out" + << BSON("reduce" + << "theSink" + << "db" + << "myDb" + << "sharded" << true) + << "$db" + << "db")); + ASSERT(true); +} + +TEST(MapReduceParseTest, parseAllOptionalFields) { + auto ctx = IDLParserErrorContext("mapReduce"); + + MapReduce::parse(ctx, + BSON("mapReduce" + << "theSource" + << "map" << mapJavascript << "reduce" << reduceJavascript << "out" + << BSON("inline" << 1) << "query" + << BSON("author" + << "dave") + << "sort" << BSON("bottlecaps" << 1) << "collation" + << BSON("locale" + << "zh@collation=pinyin") + << "limit" << 86 << "finalize" << finalizeJavascript << "scope" + << BSON("global" << initJavascript) << "verbose" << false + << "bypassDocumentValidation" << true << "writeConcern" + << BSON("w" << 1 << "j" << false << "wtimeout" << 1498) << "$db" + << "db")); +} + +} // namespace +} // namespace mongo |