diff options
-rw-r--r-- | SConstruct | 1 | ||||
-rw-r--r-- | buildscripts/idl/idl/compiler.py | 93 | ||||
-rw-r--r-- | buildscripts/idl/idl/generator.py | 815 | ||||
-rw-r--r-- | buildscripts/idl/idlc.py | 73 | ||||
-rw-r--r-- | buildscripts/idl/sample/sample.idl | 79 | ||||
-rw-r--r-- | site_scons/site_tools/idl_tool.py | 58 | ||||
-rw-r--r-- | src/mongo/SConscript | 1 | ||||
-rw-r--r-- | src/mongo/bson/bsontypes.cpp | 21 | ||||
-rw-r--r-- | src/mongo/bson/bsontypes.h | 5 | ||||
-rw-r--r-- | src/mongo/idl/SConscript | 27 | ||||
-rw-r--r-- | src/mongo/idl/idl_parser.cpp | 187 | ||||
-rw-r--r-- | src/mongo/idl/idl_parser.h | 127 | ||||
-rw-r--r-- | src/mongo/idl/idl_test.cpp | 430 | ||||
-rw-r--r-- | src/mongo/idl/idl_test_types.h | 61 | ||||
-rw-r--r-- | src/mongo/idl/unittest.idl | 311 |
15 files changed, 2289 insertions, 0 deletions
diff --git a/SConstruct b/SConstruct index 1548f5b6fdf..62f2ca12da1 100644 --- a/SConstruct +++ b/SConstruct @@ -600,6 +600,7 @@ def variable_tools_converter(val): return tool_list + [ "distsrc", "gziptool", + 'idl_tool', "jsheader", "mergelib", "mongo_integrationtest", diff --git a/buildscripts/idl/idl/compiler.py b/buildscripts/idl/idl/compiler.py new file mode 100644 index 00000000000..f285f91e627 --- /dev/null +++ b/buildscripts/idl/idl/compiler.py @@ -0,0 +1,93 @@ +# Copyright (C) 2017 MongoDB Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3, +# as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +""" +IDL compiler driver. + +Orchestrates the 3 passes (parser, binder, and generator) together. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +import io +import logging +import os +# from typing import Any, List + +from . import binder +from . import errors +from . import generator +from . import parser + + +class CompilerArgs(object): + """Set of compiler arguments.""" + + def __init__(self): + # type: () -> None + """Create a container for compiler arguments.""" + self.import_directories = None # type: List[unicode] + self.input_file = None # type: unicode + + self.output_source = None # type: unicode + self.output_header = None # type: unicode + self.output_base_dir = None # type: unicode + self.output_suffix = None # type: unicode + + +def compile_idl(args): + # type: (CompilerArgs) -> bool + """Compile an IDL file into C++ code.""" + # Named compile_idl to avoid naming conflict with builtin + if not os.path.exists(args.input_file): + logging.error("File '%s' not found", args.input_file) + + # TODO: resolve the paths, and log if they do not exist under verbose when import supported is added + #for import_dir in args.import_directories: + # if not os.path.exists(args.input_file): + + error_file_name = os.path.basename(args.input_file) + + if args.output_source is None: + if not '.' in error_file_name: + logging.error("File name '%s' must be end with a filename extension, such as '%s.idl'", + error_file_name, error_file_name) + return False + + file_name_prefix = error_file_name.split('.')[0] + file_name_prefix += args.output_suffix + + source_file_name = file_name_prefix + ".cpp" + header_file_name = file_name_prefix + ".h" + else: + source_file_name = args.output_source + header_file_name = args.output_header + + # Compile the IDL through the 3 passes + with io.open(args.input_file) as file_stream: + parsed_doc = parser.parse(file_stream, error_file_name=error_file_name) + + if not parsed_doc.errors: + bound_doc = binder.bind(parsed_doc.spec) + if not bound_doc.errors: + generator.generate_code(bound_doc.spec, args.output_base_dir, header_file_name, + source_file_name) + + return True + else: + bound_doc.errors.dump_errors() + else: + parsed_doc.errors.dump_errors() + + return False diff --git a/buildscripts/idl/idl/generator.py b/buildscripts/idl/idl/generator.py new file mode 100644 index 00000000000..d8a34c1cc4c --- /dev/null +++ b/buildscripts/idl/idl/generator.py @@ -0,0 +1,815 @@ +# Copyright (C) 2017 MongoDB Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3, +# as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +"""IDL C++ Code Generator.""" + +from __future__ import absolute_import, print_function, unicode_literals + +import io +import os +import string +import sys +import textwrap +# from typing import List, Union + +from . import ast +from . import bson + +# Number of spaces to indent code +_INDENT_SPACE_COUNT = 4 + + +def _title_case(name): + # type: (unicode) -> unicode + """Return a CapitalCased version of a string.""" + return name[0:1].upper() + name[1:] + + +def _camel_case(name): + # type: (unicode) -> unicode + """Return a camelCased version of a string.""" + return name[0:1].lower() + name[1:] + + +def _get_method_name(name): + # type: (unicode) -> unicode + """Get a method name from a fully qualified method name.""" + pos = name.rfind('::') + if pos == -1: + return name + return name[pos + 2:] + + +def _get_method_name_from_qualified_method_name(name): + # type: (unicode) -> unicode + # pylint: disable=invalid-name + """Get a method name from a fully qualified method name.""" + # TODO: in the future, we may want to support full-qualified calls to static methods + prefix = 'mongo::' + pos = name.find(prefix) + if pos == -1: + return name + + return name[len(prefix):] + + +def _is_primitive_type(cpp_type): + # type: (unicode) -> bool + """Return True if a cpp_type is a primitive type and should not be returned as reference.""" + return cpp_type in [ + 'bool', 'double', 'std::int32_t', 'std::uint32_t', 'std::uint64_t', 'std::int64_t' + ] + + +def _is_view_type(cpp_type): + # type: (unicode) -> bool + """Return True if a cpp_type should be returned as a view type from an IDL class.""" + if cpp_type == 'std::string': + return True + + return False + + +def _get_view_type(cpp_type): + # type: (unicode) -> unicode + """Map a C++ type to its C++ view type if needed.""" + if cpp_type == 'std::string': + cpp_type = 'StringData' + + return cpp_type + + +def _get_view_type_to_base_method(cpp_type): + # type: (unicode) -> unicode + """Map a C++ View type to its C++ base type.""" + assert _is_view_type(cpp_type) + + return "toString" + + +def _get_field_cpp_type(field): + # type: (ast.Field) -> unicode + """Get the C++ type name for a field.""" + assert field.cpp_type is not None or field.struct_type is not None + + if field.struct_type: + cpp_type = _title_case(field.struct_type) + else: + cpp_type = field.cpp_type + + return cpp_type + + +def _qualify_optional_type(cpp_type, field): + # type: (unicode, ast.Field) -> unicode + """Qualify the type if the field is optional.""" + if field.optional: + return 'boost::optional<%s>' % (cpp_type) + + return cpp_type + + +def _get_field_parameter_type(field): + # type: (ast.Field) -> unicode + """Get the C++ type name for a parameter for a field.""" + assert field.cpp_type is not None or field.struct_type is not None + + cpp_type = _get_view_type(_get_field_cpp_type(field)) + + return _qualify_optional_type(cpp_type, field) + + +def _get_field_member_type(field): + # type: (ast.Field) -> unicode + """Get the C++ type name for a class member for a field.""" + cpp_type = _get_field_cpp_type(field) + + return _qualify_optional_type(cpp_type, field) + + +def _get_field_member_name(field): + # type: (ast.Field) -> unicode + """Get the C++ class member name for a field.""" + return '_%s' % (_camel_case(field.name)) + + +def _get_bson_type_check(field): + # type: (ast.Field) -> unicode + """Get the C++ bson type check for a field.""" + bson_types = field.bson_serialization_type + if len(bson_types) == 1: + if bson_types[0] == 'any': + # Skip BSON valiation when any + return None + + if not bson_types[0] == 'bindata': + return 'ctxt.checkAndAssertType(element, %s)' % bson.cpp_bson_type_name(bson_types[0]) + else: + return 'ctxt.checkAndAssertBinDataType(element, %s)' % bson.cpp_bindata_subtype_type_name( + field.bindata_subtype) + else: + type_list = '{%s}' % (', '.join([bson.cpp_bson_type_name(b) for b in bson_types])) + return 'ctxt.checkAndAssertTypes(element, %s)' % type_list + + +def _access_member(field): + # type: (ast.Field) -> unicode + """Get the declaration to access a member for a field.""" + member_name = _get_field_member_name(field) + + if not field.optional: + return '%s' % (member_name) + + # optional types need a method call to access their values + return '%s.get()' % (member_name) + + +def fill_spaces(count): + # type: (int) -> unicode + """Fill a string full of spaces.""" + fill = '' + for _ in range(count * _INDENT_SPACE_COUNT): + fill += ' ' + + return fill + + +def indent_text(count, unindented_text): + # type: (int, unicode) -> unicode + """Indent each line of a multi-line string.""" + lines = unindented_text.splitlines() + fill = fill_spaces(count) + return '\n'.join(fill + line for line in lines) + + +class _IndentedTextWriter(object): + """ + A simple class to manage writing indented lines of text. + + Supports both writing indented lines, and unindented lines. + Use write_empty_line() instead of write_line('') to avoid lines + full of blank spaces. + """ + + def __init__(self, stream): + # type: (io.StringIO) -> None + """Create an indented text writer.""" + self._stream = stream + self._indent = 0 + + def write_unindented_line(self, msg): + # type: (unicode) -> None + """Write an unindented line to the stream.""" + self._stream.write(msg) + self._stream.write("\n") + + def indent(self): + # type: () -> None + """Indent the text by one level.""" + self._indent += 1 + + def unindent(self): + # type: () -> None + """Unindent the text by one level.""" + assert self._indent > 0 + self._indent -= 1 + + def write_line(self, msg): + # type: (unicode) -> None + """Write a line to the stream.""" + self._stream.write(indent_text(self._indent, msg)) + self._stream.write("\n") + + def write_empty_line(self): + # type: () -> None + """Write a line to the stream.""" + self._stream.write("\n") + + +class _EmptyBlock(object): + """Do not generate an indented block.""" + + def __init__(self): + # type: () -> None + """Create an empty block.""" + pass + + def __enter__(self): + # type: () -> None + """Do nothing.""" + pass + + def __exit__(self, *args): + # type: (*str) -> None + """Do nothing.""" + pass + + +class _UnindentedScopedBlock(object): + """Generate an unindented block, and do not indent the contents.""" + + def __init__(self, writer, opening, closing): + # type: (_IndentedTextWriter, unicode, unicode) -> None + """Create a block.""" + self._writer = writer + self._opening = opening + self._closing = closing + + def __enter__(self): + # type: () -> None + """Write the beginning of the block and do not indent.""" + self._writer.write_unindented_line(self._opening) + + def __exit__(self, *args): + # type: (*str) -> None + """Write the end of the block and do not change indentation.""" + self._writer.write_unindented_line(self._closing) + + +class _IndentedScopedBlock(object): + """Generate a block, and indent the contents.""" + + def __init__(self, writer, opening, closing): + # type: (_IndentedTextWriter, unicode, unicode) -> None + """Create a block.""" + self._writer = writer + self._opening = opening + self._closing = closing + + def __enter__(self): + # type: () -> None + """Write the beginning of the block and then indent.""" + self._writer.write_line(self._opening) + self._writer.indent() + + def __exit__(self, *args): + # type: (*str) -> None + """Unindent the block and print the ending.""" + self._writer.unindent() + self._writer.write_line(self._closing) + + +class _FieldUsageChecker(object): + """Check for duplicate fields, and required fields as needed.""" + + def __init__(self, writer): + # type: (_IndentedTextWriter) -> None + """Create a field usage checker.""" + self._writer = writer # type: _IndentedTextWriter + self.fields = [] # type: List[ast.Field] + + # TODO: use a more optimal data type + self._writer.write_line('std::set<StringData> usedFields;') + + def add_store(self): + # type: () -> None + """Create the C++ field store initialization code.""" + self._writer.write_line('auto push_result = usedFields.insert(fieldName);') + with _IndentedScopedBlock(self._writer, 'if (push_result.second == false) {', '}'): + self._writer.write_line('ctxt.throwDuplicateField(element);') + + def add(self, field): + # type: (ast.Field) -> None + """Add a field to track.""" + self.fields.append(field) + + def add_final_checks(self): + # type: () -> None + """Output the code to check for missing fields.""" + for field in self.fields: + if (not field.optional) and (not field.ignore): + with _IndentedScopedBlock(self._writer, + 'if (usedFields.find("%s") == usedFields.end()) {' % + (field.name), '}'): + if field.default: + self._writer.write_line('object.%s = %s;' % + (_get_field_member_name(field), field.default)) + else: + self._writer.write_line('ctxt.throwMissingField("%s");' % (field.name)) + + +class _CppFileWriterBase(object): + """ + C++ File writer. + + Encapsulates low level knowledge of how to print a C++ file. + Relies on caller to orchestrate calls correctly though. + """ + + def __init__(self, writer): + # type: (_IndentedTextWriter) -> None + """Create a C++ code writer.""" + self._writer = writer # type: _IndentedTextWriter + + def write_unindented_line(self, msg): + # type: (unicode) -> None + """Write an unindented line to the stream.""" + self._writer.write_unindented_line(msg) + + def write_empty_line(self): + # type: () -> None + """Write an empty line to the stream.""" + self._writer.write_empty_line() + + def gen_file_header(self): + # type: () -> None + """Generate a file header saying the file is generated.""" + self._writer.write_unindented_line( + textwrap.dedent("""\ + /** + * WARNING: This is a generated file. Do not modify. + * + * Source: %s + */ + """ % (" ".join(sys.argv)))) + + def gen_system_include(self, include): + # type: (unicode) -> None + """Generate a system C++ include line.""" + self._writer.write_unindented_line('#include <%s>' % (include)) + + def gen_include(self, include): + # type: (unicode) -> None + """Generate a non-system C++ include line.""" + self._writer.write_unindented_line('#include "%s"' % (include)) + + def gen_namespace_block(self, namespace): + # type: (unicode) -> _UnindentedScopedBlock + """Generate a namespace block.""" + # TODO: support namespace strings which consist of '::' delimited namespaces + return _UnindentedScopedBlock(self._writer, 'namespace %s {' % (namespace), + '} // namespace %s' % (namespace)) + + def gen_description_comment(self, description): + # type: (unicode) -> None + """Generate a multiline comment with the description from the IDL.""" + self._writer.write_line( + textwrap.dedent("""\ + /** + * %s + */""" % (description))) + + def _block(self, opening, closing): + # type: (unicode, unicode) -> Union[_IndentedScopedBlock,_EmptyBlock] + """Generate an indented block if opening is not empty.""" + if not opening: + return _EmptyBlock() + + return _IndentedScopedBlock(self._writer, opening, closing) + + def _predicate(self, check_str, use_else_if=False): + # type: (unicode, bool) -> Union[_IndentedScopedBlock,_EmptyBlock] + """ + Generate an if block if the condition is not-empty. + + Generate 'else if' instead of use_else_if is True. + """ + if not check_str: + return _EmptyBlock() + + conditional = 'if' + if use_else_if: + conditional = 'else if' + + return _IndentedScopedBlock(self._writer, '%s (%s) {' % (conditional, check_str), '}') + + +class _CppHeaderFileWriter(_CppFileWriterBase): + """C++ .h File writer.""" + + def __init__(self, writer): + # type: (_IndentedTextWriter) -> None + """Create a C++ .cpp file code writer.""" + super(_CppHeaderFileWriter, self).__init__(writer) + + def gen_class_declaration_block(self, class_name): + # type: (unicode) -> _IndentedScopedBlock + """Generate a class declaration block.""" + return _IndentedScopedBlock(self._writer, 'class %s {' % _title_case(class_name), '};') + + def gen_serializer_methods(self, class_name): + # type: (unicode) -> None + """Generate a serializer method declarations.""" + self._writer.write_line( + 'static %s parse(const IDLParserErrorContext& ctxt, const BSONObj& object);' % + (_title_case(class_name))) + self._writer.write_line('void serialize(BSONObjBuilder* builder) const;') + self._writer.write_empty_line() + + def gen_getter(self, field): + # type: (ast.Field) -> None + """Generate the C++ getter definition for a field.""" + cpp_type = _get_field_cpp_type(field) + param_type = _get_field_parameter_type(field) + member_name = _get_field_member_name(field) + + optional_ampersand = "" + disable_xvalue = False + if not _is_view_type(cpp_type): + if not field.optional and not _is_primitive_type(cpp_type): + optional_ampersand = '&' + disable_xvalue = True + body = 'return %s;' % (member_name) + else: + body = 'return %s{%s};' % (param_type, member_name) + disable_xvalue = True + + # Generate a getter that disables xvalue for view types (i.e. StringData), constructed + # optional types, and non-primitive types. + if disable_xvalue: + self._writer.write_line('const %s%s get%s() const& { %s }' % + (param_type, optional_ampersand, _title_case(field.name), body)) + self._writer.write_line("const %s%s get%s() && = delete;" % + (param_type, optional_ampersand, _title_case(field.name))) + else: + self._writer.write_line('const %s%s get%s() const { %s }' % + (param_type, optional_ampersand, _title_case(field.name), body)) + + def gen_setter(self, field): + # type: (ast.Field) -> None + """Generate the C++ setter definition for a field.""" + cpp_type = _get_field_cpp_type(field) + param_type = _get_field_parameter_type(field) + member_name = _get_field_member_name(field) + + if _is_view_type(cpp_type): + if not field.optional: + self._writer.write_line('void set%s(%s value) & { %s = value.%s(); }' % + (_title_case(field.name), param_type, member_name, + _get_view_type_to_base_method(cpp_type))) + + else: + # We need to convert between two different types of optional<T> and yet retain the + # ability for the user to specific an uninitialized optional. This occurs for + # mongo::StringData and std::string paired together. + with self._block('void set%s(%s value) {' % (_title_case(field.name), param_type), + "}"): + self._writer.write_line( + textwrap.dedent("""\ + if (value.is_initialized()) { + %s = value.get().%s(); + } else { + %s = boost::none; + } + """ % (member_name, _get_view_type_to_base_method(cpp_type), member_name))) + + else: + self._writer.write_line('void set%s(%s value) { %s = std::move(value); }' % + (_title_case(field.name), param_type, member_name)) + self._writer.write_empty_line() + + def gen_member(self, field): + # type: (ast.Field) -> None + """Generate the C++ class member definition for a field.""" + member_type = _get_field_member_type(field) + member_name = _get_field_member_name(field) + + self._writer.write_line('%s %s;' % (member_type, member_name)) + + def generate(self, spec): + # type: (ast.IDLAST) -> None + """Generate the C++ header to a stream.""" + self.gen_file_header() + + self._writer.write_unindented_line('#pragma once') + self.write_empty_line() + + # Generate system includes first + header_list = [ + 'algorithm', + 'boost/optional.hpp', + 'cstdint', + 'string', + ] + + header_list.sort() + + for include in header_list: + self.gen_system_include(include) + + self.write_empty_line() + + # Generate user includes second + header_list = [ + 'mongo/base/string_data.h', + 'mongo/bson/bsonobj.h', + 'mongo/idl/idl_parser.h', + ] + spec.globals.cpp_includes + + header_list.sort() + + for include in header_list: + self.gen_include(include) + + self.write_empty_line() + + # Generate namesapce + with self.gen_namespace_block(spec.globals.cpp_namespace): + self.write_empty_line() + + for struct in spec.structs: + self.gen_description_comment(struct.description) + with self.gen_class_declaration_block(struct.name): + self.write_unindented_line('public:') + + # Write constructor + self.gen_serializer_methods(struct.name) + + # Write getters & setters + for field in struct.fields: + if not field.ignore: + if field.description: + self.gen_description_comment(field.description) + self.gen_getter(field) + self.gen_setter(field) + + self.write_unindented_line('private:') + + # Write member variables + for field in struct.fields: + if not field.ignore: + self.gen_member(field) + + self.write_empty_line() + + +class _CppSourceFileWriter(_CppFileWriterBase): + """C++ .cpp File writer.""" + + def __init__(self, writer): + # type: (_IndentedTextWriter) -> None + """Create a C++ .cpp file code writer.""" + super(_CppSourceFileWriter, self).__init__(writer) + + def gen_field_deserializer(self, field): + # type: (ast.Field) -> None + """Generate the C++ deserializer piece for a few field.""" + # May be an empty block if the type is any + type_predicate = _get_bson_type_check(field) + + with self._predicate(type_predicate): + + if field.struct_type: + self._writer.write_line('IDLParserErrorContext tempContext("%s", &ctxt);' % + (field.name)) + self._writer.write_line('const auto localObject = element.Obj();') + self._writer.write_line('object.%s = %s::parse(tempContext, localObject);' % ( + _get_field_member_name(field), _title_case(field.struct_type))) + elif 'BSONElement::' in field.deserializer: + method_name = _get_method_name(field.deserializer) + self._writer.write_line('object.%s = element.%s();' % + (_get_field_member_name(field), method_name)) + else: + # Custom method, call the method on object + # TODO: avoid this string hack in the future + if len(field.bson_serialization_type) == 1 and field.bson_serialization_type[ + 0] == 'string': + # Call a method like: Class::method(StringData value) + self._writer.write_line('auto tempValue = element.valueStringData();') + + method_name = _get_method_name(field.deserializer) + + self._writer.write_line('object.%s = %s(tempValue);' % + (_get_field_member_name(field), method_name)) + elif len(field.bson_serialization_type) == 1 and field.bson_serialization_type[ + 0] == 'object': + # Call a method like: Class::method(const BSONObj& value) + method_name = _get_method_name_from_qualified_method_name(field.deserializer) + self._writer.write_line('const BSONObj localObject = element.Obj();') + self._writer.write_line('object.%s = %s(localObject);' % + (_get_field_member_name(field), method_name)) + else: + # Call a method like: Class::method(const BSONElement& value) + method_name = _get_method_name_from_qualified_method_name(field.deserializer) + + self._writer.write_line('object.%s = %s(element);' % + (_get_field_member_name(field), method_name)) + + def gen_deserializer_method(self, struct): + # type: (ast.Struct) -> None + """Generate the C++ deserializer method definition.""" + + with self._block( + '%s %s::parse(const IDLParserErrorContext& ctxt, const BSONObj& bsonObject) {' % + (_title_case(struct.name), _title_case(struct.name)), '}'): + + self._writer.write_line('%s object;' % _title_case(struct.name)) + + field_usage_check = _FieldUsageChecker(self._writer) + self._writer.write_empty_line() + + with self._block('for (const auto& element : bsonObject) {', '}'): + + self._writer.write_line('const auto fieldName = element.fieldNameStringData();') + self._writer.write_empty_line() + + # TODO: generate command namespace string check + field_usage_check.add_store() + self._writer.write_empty_line() + + first_field = True + for field in struct.fields: + field_predicate = 'fieldName == "%s"' % field.name + field_usage_check.add(field) + + with self._predicate(field_predicate, not first_field): + if field.ignore: + self._writer.write_line('// ignore field') + else: + self.gen_field_deserializer(field) + + if first_field: + first_field = False + + # End of for fields + # Generate strict check for extranous fields + if struct.strict: + with self._block('else {', '}'): + self._writer.write_line('ctxt.throwUnknownField(fieldName);') + + self._writer.write_empty_line() + + # Check for required fields + field_usage_check.add_final_checks() + self._writer.write_empty_line() + + self._writer.write_line('return object;') + + def gen_serializer_method(self, struct): + # type: (ast.Struct) -> None + """Generate the serialize method definition.""" + + with self._block('void %s::serialize(BSONObjBuilder* builder) const {' % + _title_case(struct.name), '}'): + + for field in struct.fields: + # If fields are meant to be ignored during deserialization, there is not need to serialize them + if field.ignore: + continue + + member_name = _get_field_member_name(field) + + optional_block_start = None + if field.optional: + optional_block_start = 'if (%s) {' % (member_name) + elif field.struct_type: + # Introduce a new scope for required nested object serialization. + optional_block_start = '{' + + with self._block(optional_block_start, '}'): + + if not field.struct_type: + if field.serializer: + # Generate custom serialization + method_name = _get_method_name(field.serializer) + + if len(field.bson_serialization_type) == 1 and \ + field.bson_serialization_type[0] == 'string': + # TODO: expand this out to be less then a string only hack + self._writer.write_line('auto tempValue = %s.%s();' % + (_access_member(field), method_name)) + self._writer.write_line( + 'builder->append("%s", std::move(tempValue));' % (field.name)) + else: + self._writer.write_line('%s.%s(builder);' % + (_access_member(field), method_name)) + + else: + # Generate default serialization using BSONObjBuilder::append + self._writer.write_line('builder->append("%s", %s);' % + (field.name, _access_member(field))) + + else: + self._writer.write_line( + 'BSONObjBuilder subObjBuilder(builder->subobjStart("%s"));' % + (field.name)) + self._writer.write_line('%s.serialize(&subObjBuilder);' % + (_access_member(field))) + # Add a blank line after each block + self._writer.write_empty_line() + + def generate(self, spec, header_file_name): + # type: (ast.IDLAST, unicode) -> None + """Generate the C++ header to a stream.""" + self.gen_file_header() + + # Generate include for generated header first + self.gen_include(header_file_name) + self.write_empty_line() + + # Generate system includes second + self.gen_system_include('set') + self.write_empty_line() + + # Generate mongo includes third + self.gen_include('mongo/bson/bsonobjbuilder.h') + self.write_empty_line() + + # Generate namesapce + with self.gen_namespace_block(spec.globals.cpp_namespace): + self.write_empty_line() + + for struct in spec.structs: + # Write deserializer + self.gen_deserializer_method(struct) + self.write_empty_line() + + # Write serializer + self.gen_serializer_method(struct) + self.write_empty_line() + + +def _generate_header(spec, file_name): + # type: (ast.IDLAST, unicode) -> None + """Generate a C++ header.""" + stream = io.StringIO() + text_writer = _IndentedTextWriter(stream) + + header = _CppHeaderFileWriter(text_writer) + + header.generate(spec) + + # Generate structs + with io.open(file_name, mode='wb') as file_handle: + file_handle.write(stream.getvalue().encode()) + + +def _generate_source(spec, file_name, header_file_name): + # type: (ast.IDLAST, unicode, unicode) -> None + """Generate a C++ source file.""" + stream = io.StringIO() + text_writer = _IndentedTextWriter(stream) + + source = _CppSourceFileWriter(text_writer) + + source.generate(spec, header_file_name) + + # Generate structs + with io.open(file_name, mode='wb') as file_handle: + file_handle.write(stream.getvalue().encode()) + + +def generate_code(spec, output_base_dir, header_file_name, source_file_name): + # type: (ast.IDLAST, unicode, unicode, unicode) -> None + """Generate a C++ header and source file from an idl.ast tree.""" + + _generate_header(spec, header_file_name) + + include_h_file_name = os.path.relpath( + os.path.normpath(header_file_name), os.path.normpath(output_base_dir)) + + # Normalize to POSIX style for consistency across Windows and POSIX. + include_h_file_name = include_h_file_name.replace("\\", "/") + + _generate_source(spec, source_file_name, include_h_file_name) diff --git a/buildscripts/idl/idlc.py b/buildscripts/idl/idlc.py new file mode 100644 index 00000000000..04225b13d7b --- /dev/null +++ b/buildscripts/idl/idlc.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python2 +# Copyright (C) 2017 MongoDB Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3, +# as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +"""IDL Compiler Driver Main Entry point.""" + +from __future__ import absolute_import, print_function + +import argparse +import sys + +import idl.compiler + + +def main(): + # type: () -> None + """Main Entry point.""" + parser = argparse.ArgumentParser(description='MongoDB IDL Compiler.') + + parser.add_argument('file', type=str, help="IDL input file") + + parser.add_argument('-o', '--output', type=str, help="IDL output source file") + + parser.add_argument('--header', type=str, help="IDL output header file") + + parser.add_argument( + '-i', + '--include', + type=str, + action="append", + help="Directory to search for IDL import files") + + parser.add_argument('-v', '--verbose', action='count', help="Enable verbose tracing") + + parser.add_argument('--base_dir', type=str, help="IDL output relative base directory") + + args = parser.parse_args() + + compiler_args = idl.compiler.CompilerArgs() + + compiler_args.input_file = args.file + compiler_args.import_directories = args.include + + compiler_args.output_source = args.output + compiler_args.output_header = args.header + compiler_args.output_base_dir = args.base_dir + compiler_args.output_suffix = "_gen" + + if (args.output is not None and args.header is None) or \ + (args.output is None and args.header is not None): + print("ERROR: Either both --header and --output must be specified or neither.") + sys.exit(1) + + # Compile the IDL document the user specified + success = idl.compiler.compile_idl(compiler_args) + + if not success: + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/buildscripts/idl/sample/sample.idl b/buildscripts/idl/sample/sample.idl new file mode 100644 index 00000000000..bb836cfe2a4 --- /dev/null +++ b/buildscripts/idl/sample/sample.idl @@ -0,0 +1,79 @@ +# Copyright (C) 2017 MongoDB Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3, +# as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# Sample idl +# Demonstrates a subset of features +# 1. string types +# 2. int types +# 3. a custom type, NamespaceString +# 4. Nested structs +# 5. Optional fields +# 6. Default values +global: + cpp_namespace: "mongo" + cpp_includes: + - "mongo/db/namespace_string.h" + +types: + string: + bson_serialization_type: string + description: "A BSON UTF-8 string" + cpp_type: "std::string" + deserializer: "mongo::BSONElement::str" + + int: + bson_serialization_type: int + description: "A BSON 32-bit integer" + cpp_type: "std::int32_t" + deserializer: "mongo::BSONElement::_numberInt" + + namespacestring: + bson_serialization_type: string + description: "A MongoDB NamespaceString" + cpp_type: "mongo::NamespaceString" + serializer: mongo::NamespaceString::toString + deserializer: mongo::NamespaceString + + safeInt32: + description: Accepts any numerical type within int32 range + cpp_type: std::int64_t + bson_serialization_type: + - long + - int + - decimal + - double + deserializer: "mongo::BSONElement::numberInt" + +structs: + default_values: + description: UnitTest for a single safeInt32 + fields: + stringfield: + type: string + default: '"a default"' + description: "An example string field with default value" + intfield: + type: int + default: 42 + description: "An example int field with default value" + numericfield: + type: safeInt32 + description: "A numeric type that supports multiple types" + nsfield: + type: namespacestring + description: "A namespace string type" + optionalField: + type: string + optional: true + description: "An optional string" diff --git a/site_scons/site_tools/idl_tool.py b/site_scons/site_tools/idl_tool.py new file mode 100644 index 00000000000..22e75c4ac42 --- /dev/null +++ b/site_scons/site_tools/idl_tool.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python2 +# Copyright (C) 2017 MongoDB Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3, +# as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +"""IDL Compiler Scons Tool.""" + +import os.path +import sys + +import SCons + +def idlc_emitter(target, source, env): + """For each input IDL file, the tool produces a .cpp and .h file.""" + first_source = str(source[0]) + + if not first_source.endswith(".idl"): + raise ValueError("Bad idl file name '%s', it must end with '.idl' " % (first_source)) + + base_file_name, _ = SCons.Util.splitext(str(target[0])) + target_source = base_file_name + "_gen.cpp" + target_header = base_file_name + "_gen.h" + + return [target_source, target_header], source + + +IDLCAction = SCons.Action.Action('$IDLCCOM', '$IDLCCOMSTR') + +# TODO: create a scanner for imports when imports are implemented +IDLCBuilder = SCons.Builder.Builder( + action=IDLCAction, + emitter=idlc_emitter, + srcsuffx=".idl", + suffix=".cpp" + ) + +def generate(env): + bld = IDLCBuilder + env['BUILDERS']['Idlc'] = bld + + env['IDLC'] = sys.executable + " buildscripts/idl/idlc.py" + env['IDLCFLAGS'] = '' + base_dir = env.subst('$BUILD_ROOT/$VARIANT_DIR').replace("#", "") + env['IDLCCOM'] = '$IDLC --base_dir %s --header ${TARGETS[1]} --output ${TARGETS[0]} $SOURCES ' % (base_dir) + env['IDLCSUFFIX'] = '.idl' + +def exists(env): + return True
\ No newline at end of file diff --git a/src/mongo/SConscript b/src/mongo/SConscript index fc24c847bc7..39de568e960 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -27,6 +27,7 @@ env.SConscript( 'db', 'dbtests', 'executor', + 'idl', 'installer', 'logger', 'platform', diff --git a/src/mongo/bson/bsontypes.cpp b/src/mongo/bson/bsontypes.cpp index b1f545ee180..a2dbcaaf334 100644 --- a/src/mongo/bson/bsontypes.cpp +++ b/src/mongo/bson/bsontypes.cpp @@ -127,4 +127,25 @@ bool isValidBSONType(int type) { } } +const char* typeName(BinDataType type) { + switch (type) { + case BinDataGeneral: + return "general"; + case Function: + return "function"; + case ByteArrayDeprecated: + return "byte(deprecated)"; + case bdtUUID: + return "UUID(deprecated)"; + case newUUID: + return "UUID"; + case MD5Type: + return "MD5"; + case bdtCustom: + return "Custom"; + default: + return "invalid"; + } +} + } // namespace mongo diff --git a/src/mongo/bson/bsontypes.h b/src/mongo/bson/bsontypes.h index ce3283ebb59..0bd9caeff9a 100644 --- a/src/mongo/bson/bsontypes.h +++ b/src/mongo/bson/bsontypes.h @@ -137,6 +137,11 @@ enum BinDataType { bdtCustom = 128 }; +/** + * Return the name of the BinData Type. + */ +const char* typeName(BinDataType type); + /** Returns a number for where a given type falls in the sort order. * Elements with the same return value should be compared for value equality. * The return value is not a BSONType and should not be treated as one. diff --git a/src/mongo/idl/SConscript b/src/mongo/idl/SConscript new file mode 100644 index 00000000000..93a57d332e0 --- /dev/null +++ b/src/mongo/idl/SConscript @@ -0,0 +1,27 @@ +# -*- mode: python -*- +Import("env") + +env = env.Clone() + +env.Library( + target="idl_parser", + source=[ + 'idl_parser.cpp' + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + ] +) + +env.CppUnitTest( + target='idl_test', + source=[ + 'idl_test.cpp', + env.Idlc('unittest.idl')[0] + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/namespace_string', + '$BUILD_DIR/mongo/idl/idl_parser', + ], +) diff --git a/src/mongo/idl/idl_parser.cpp b/src/mongo/idl/idl_parser.cpp new file mode 100644 index 00000000000..5e1edd7af4c --- /dev/null +++ b/src/mongo/idl/idl_parser.cpp @@ -0,0 +1,187 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include <stack> +#include <string> + +#include "mongo/idl/idl_parser.h" + +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + +namespace { +/** + * For a vector of BSONType, return a string of comma separated names. + * + * Example: "string, bool, numberDouble" + */ +std::string toCommaDelimitedList(const std::vector<BSONType>& types) { + str::stream builder; + + for (std::size_t i = 0; i < types.size(); ++i) { + if (i > 0) { + builder << ", "; + } + + builder << typeName(types[i]); + } + + return builder; +} + +} // namespace + +bool IDLParserErrorContext::checkAndAssertType(const BSONElement& element, BSONType type) const { + auto elementType = element.type(); + + if (elementType != type) { + // If the type is wrong, ignore Null and Undefined values + if (elementType == jstNULL || elementType == Undefined) { + return false; + } + + std::string path = getElementPath(element); + uasserted(40410, + str::stream() << "BSON field '" << path << "' is the wrong type '" + << typeName(element.type()) + << "', expected type '" + << typeName(type) + << "'"); + } + + return true; +} + +bool IDLParserErrorContext::checkAndAssertBinDataType(const BSONElement& element, + BinDataType type) const { + bool isBinDataType = checkAndAssertType(element, BinData); + if (!isBinDataType) { + return false; + } + + if (element.binDataType() != type) { + std::string path = getElementPath(element); + uasserted(40411, + str::stream() << "BSON field '" << path << "' is the wrong bindData type '" + << typeName(element.binDataType()) + << "', expected type '" + << typeName(type) + << "'"); + } + + return true; +} + +bool IDLParserErrorContext::checkAndAssertTypes(const BSONElement& element, + const std::vector<BSONType>& types) const { + auto elementType = element.type(); + + auto pos = std::find(types.begin(), types.end(), elementType); + if (pos == types.end()) { + // If the type is wrong, ignore Null and Undefined values + if (elementType == jstNULL || elementType == Undefined) { + return false; + } + + std::string path = getElementPath(element); + std::string type_str = toCommaDelimitedList(types); + uasserted(40412, + str::stream() << "BSON field '" << path << "' is the wrong type '" + << typeName(element.type()) + << "', expected types '[" + << type_str + << "']"); + } + + return true; +} + + +std::string IDLParserErrorContext::getElementPath(const BSONElement& element) const { + return getElementPath(element.fieldNameStringData()); +} + +std::string IDLParserErrorContext::getElementPath(StringData fieldName) const { + if (_predecessor == nullptr) { + str::stream builder; + + builder << _currentField; + + if (!fieldName.empty()) { + builder << "." << fieldName; + } + + return builder; + } else { + std::stack<StringData> pieces; + + if (!fieldName.empty()) { + pieces.push(fieldName); + } + + pieces.push(_currentField); + + const IDLParserErrorContext* head = _predecessor; + while (head) { + pieces.push(head->_currentField); + head = head->_predecessor; + } + + str::stream builder; + + while (!pieces.empty()) { + builder << pieces.top(); + pieces.pop(); + + if (!pieces.empty()) { + builder << "."; + } + } + + return builder; + } +} + +void IDLParserErrorContext::throwDuplicateField(const BSONElement& element) const { + std::string path = getElementPath(element); + uasserted(40413, str::stream() << "BSON field '" << path << "' is a duplicate field"); +} + +void IDLParserErrorContext::throwMissingField(StringData fieldName) const { + std::string path = getElementPath(fieldName); + uasserted(40414, + str::stream() << "BSON field '" << path << "' is missing but a required field"); +} + +void IDLParserErrorContext::throwUnknownField(StringData fieldName) const { + std::string path = getElementPath(fieldName); + uasserted(40415, str::stream() << "BSON field '" << path << "' is an unknown field."); +} +} // namespace mongo diff --git a/src/mongo/idl/idl_parser.h b/src/mongo/idl/idl_parser.h new file mode 100644 index 00000000000..a7aa8255eb6 --- /dev/null +++ b/src/mongo/idl/idl_parser.h @@ -0,0 +1,127 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include <string> + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonelement.h" +#include "mongo/bson/bsontypes.h" + +namespace mongo { + +/** + * IDLParserErrorContext manages the current parser context for parsing BSON documents. + * + * The class stores the path to the current document to enable it provide more useful error + * messages. The path is a dot delimited list of field names which is useful for nested struct + * parsing. + * + * This class is responsible for throwing all error messages the IDL generated parsers throw, + * and provide utility methods like checking a BSON type or set of BSON types. + */ +class IDLParserErrorContext { + MONGO_DISALLOW_COPYING(IDLParserErrorContext); + +public: + IDLParserErrorContext(StringData fieldName) : _currentField(fieldName), _predecessor(nullptr) {} + + IDLParserErrorContext(StringData fieldName, const IDLParserErrorContext* predecessor) + : _currentField(fieldName), _predecessor(predecessor) {} + + /** + * Check that BSON element is a given type or whether the field should be skipped. + * + * Returns true if the BSON element is the correct type. + * Return false if the BSON element is Null or Undefined and the field's value should not be + * processed. + * Throws an exception if the BSON element's type is wrong. + */ + bool checkAndAssertType(const BSONElement& element, BSONType type) const; + + /** + * Check that BSON element is a bin data type, and has the specified bin data subtype, or + * whether the field should be skipped. + * + * Returns true if the BSON element is the correct type. + * Return false if the BSON element is Null or Undefined and the field's value should not be + * processed. + * Throws an exception if the BSON element's type is wrong. + */ + bool checkAndAssertBinDataType(const BSONElement& element, BinDataType type) const; + + /** + * Check that BSON element is one of a given type or whether the field should be skipped. + * + * Returns true if the BSON element is one of the types. + * Return false if the BSON element is Null or Undefined and the field's value should not be + * processed. + * Throws an exception if the BSON element's type is wrong. + */ + bool checkAndAssertTypes(const BSONElement& element, const std::vector<BSONType>& types) const; + + /** + * Throw an error message about the BSONElement being a duplicate field. + */ + void throwDuplicateField(const BSONElement& element) const; + + /** + * Throw an error message about the required field missing form the document. + */ + void throwMissingField(StringData fieldName) const; + + /** + * Throw an error message about the required field missing form the document. + */ + void throwUnknownField(StringData fieldName) const; + +private: + /** + * See comment on getElementPath below. + */ + std::string getElementPath(const BSONElement& element) const; + + /** + * Return a dot seperated path to the specified field. For instance, if the code is parsing a + * grandchild field that has an error, this will return "grandparent.parent.child". + */ + std::string getElementPath(StringData fieldName) const; + +private: + // Name of the current field that is being parsed. + const StringData _currentField; + + // Pointer to a parent parser context. + // This provides a singly linked list of parent pointers, and use to produce a full path to a + // field with an error. + const IDLParserErrorContext* _predecessor; +}; + +} // namespace mongo diff --git a/src/mongo/idl/idl_test.cpp b/src/mongo/idl/idl_test.cpp new file mode 100644 index 00000000000..5fa8e153614 --- /dev/null +++ b/src/mongo/idl/idl_test.cpp @@ -0,0 +1,430 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/bson/bsonmisc.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/idl/unittest_gen.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { + +// Use a seperate function to get better error messages when types do not match. +template <typename T1, typename T2> +void assert_same_types() { + static_assert(std::is_same<T1, T2>::value, "expected correct type"); +} + +template <typename ParserT, typename TestT, BSONType Test_bson_type> +void TestLoopback(TestT test_value) { + IDLParserErrorContext ctxt("root"); + + auto testDoc = BSON("value" << test_value); + + auto element = testDoc.firstElement(); + ASSERT_EQUALS(element.type(), Test_bson_type); + + auto testStruct = ParserT::parse(ctxt, testDoc); + assert_same_types<decltype(testStruct.getValue()), TestT>(); + + ASSERT_EQUALS(testStruct.getValue(), test_value); + + // Positive: Test we can roundtrip from the just parsed document + { + BSONObjBuilder builder; + testStruct.serialize(&builder); + auto loopbackDoc = builder.obj(); + + ASSERT_BSONOBJ_EQ(testDoc, loopbackDoc); + } + + // Positive: Test we can serialize from nothing the same document + { + BSONObjBuilder builder; + ParserT one_new; + one_new.setValue(test_value); + testStruct.serialize(&builder); + + auto serializedDoc = builder.obj(); + ASSERT_BSONOBJ_EQ(testDoc, serializedDoc); + } +} + +/// Type tests: +// Positive: Test we can serialize the type out and back again +TEST(IDLOneTypeTests, TestLoopbackTest) { + TestLoopback<One_string, const StringData, String>("test_value"); + TestLoopback<One_int, std::int32_t, NumberInt>(123); + TestLoopback<One_long, std::int64_t, NumberLong>(456); + TestLoopback<One_double, double, NumberDouble>(3.14159); + TestLoopback<One_bool, bool, Bool>(true); + TestLoopback<One_objectid, const OID&, jstOID>(OID::max()); + TestLoopback<One_date, const Date_t&, Date>(Date_t::now()); + TestLoopback<One_timestamp, const Timestamp&, bsonTimestamp>(Timestamp::max()); +} + +// Test if a given value for a given bson document parses successfully or fails if the bson types +// mismatch. +template <typename ParserT, BSONType Parser_bson_type, typename TestT, BSONType Test_bson_type> +void TestParse(TestT test_value) { + IDLParserErrorContext ctxt("root"); + + auto testDoc = BSON("value" << test_value); + + auto element = testDoc.firstElement(); + ASSERT_EQUALS(element.type(), Test_bson_type); + + if (Parser_bson_type != Test_bson_type) { + ASSERT_THROWS(ParserT::parse(ctxt, testDoc), UserException); + } else { + (void)ParserT::parse(ctxt, testDoc); + } +} + +// Test each of types either fail or succeeded based on the parser's bson type +template <typename ParserT, BSONType Parser_bson_type> +void TestParsers() { + TestParse<ParserT, Parser_bson_type, StringData, String>("test_value"); + TestParse<ParserT, Parser_bson_type, std::int32_t, NumberInt>(123); + TestParse<ParserT, Parser_bson_type, std::int64_t, NumberLong>(456); + TestParse<ParserT, Parser_bson_type, double, NumberDouble>(3.14159); + TestParse<ParserT, Parser_bson_type, bool, Bool>(true); + TestParse<ParserT, Parser_bson_type, OID, jstOID>(OID::max()); + TestParse<ParserT, Parser_bson_type, Date_t, Date>(Date_t::now()); + TestParse<ParserT, Parser_bson_type, Timestamp, bsonTimestamp>(Timestamp::max()); +} + +// Negative: document with wrong types for required field +TEST(IDLOneTypeTests, TestNegativeWrongTypes) { + TestParsers<One_string, String>(); + TestParsers<One_int, NumberInt>(); + TestParsers<One_long, NumberLong>(); + TestParsers<One_double, NumberDouble>(); + TestParsers<One_bool, Bool>(); + TestParsers<One_objectid, jstOID>(); + TestParsers<One_date, Date>(); + TestParsers<One_timestamp, bsonTimestamp>(); +} + +// Mixed: test a type that accepts multiple bson types +TEST(IDLOneTypeTests, TestSafeInt32) { + TestParse<One_safeint32, NumberInt, StringData, String>("test_value"); + TestParse<One_safeint32, NumberInt, std::int32_t, NumberInt>(123); + TestParse<One_safeint32, NumberLong, std::int64_t, NumberLong>(456); + TestParse<One_safeint32, NumberDouble, double, NumberDouble>(3.14159); + TestParse<One_safeint32, NumberInt, bool, Bool>(true); + TestParse<One_safeint32, NumberInt, OID, jstOID>(OID::max()); + TestParse<One_safeint32, NumberInt, Date_t, Date>(Date_t::now()); + TestParse<One_safeint32, NumberInt, Timestamp, bsonTimestamp>(Timestamp::max()); +} + +// Mixed: test a type that accepts NamespaceString +TEST(IDLOneTypeTests, TestNamespaceString) { + IDLParserErrorContext ctxt("root"); + + auto testDoc = BSON("value" + << "foo.bar"); + + auto element = testDoc.firstElement(); + ASSERT_EQUALS(element.type(), String); + + auto testStruct = One_namespacestring::parse(ctxt, testDoc); + ASSERT_EQUALS(testStruct.getValue(), NamespaceString("foo.bar")); + + // Positive: Test we can roundtrip from the just parsed document + { + BSONObjBuilder builder; + testStruct.serialize(&builder); + auto loopbackDoc = builder.obj(); + + ASSERT_BSONOBJ_EQ(testDoc, loopbackDoc); + } + + // Positive: Test we can serialize from nothing the same document + { + BSONObjBuilder builder; + One_namespacestring one_new; + one_new.setValue(NamespaceString("foo.bar")); + testStruct.serialize(&builder); + + auto serializedDoc = builder.obj(); + ASSERT_BSONOBJ_EQ(testDoc, serializedDoc); + } + + // Negative: invalid namespace + { + auto testBadDoc = BSON("value" << StringData("foo\0bar", 7)); + + ASSERT_THROWS(One_namespacestring::parse(ctxt, testBadDoc), UserException); + } +} + +// Postive: Test any type +TEST(IDLOneTypeTests, TestAnyType) { + IDLParserErrorContext ctxt("root"); + + // Positive: string field + { + auto testDoc = BSON("value" + << "Foo"); + One_any_basic_type::parse(ctxt, testDoc); + } + + // Positive: int field + { + auto testDoc = BSON("value" << 12); + One_any_basic_type::parse(ctxt, testDoc); + } +} + +// Postive: Test object type +TEST(IDLOneTypeTests, TestObjectType) { + IDLParserErrorContext ctxt("root"); + + // Positive: object + { + auto testDoc = BSON("value" << BSON("value" + << "foo")); + One_any_basic_type::parse(ctxt, testDoc); + } +} + + +// Negative: Test object type +TEST(IDLOneTypeTests, TestObjectTypeNegative) { + IDLParserErrorContext ctxt("root"); + + // Negative: string field + { + auto testDoc = BSON("value" + << "Foo"); + One_any_basic_type::parse(ctxt, testDoc); + } + + // Negative: int field + { + auto testDoc = BSON("value" << 12); + One_any_basic_type::parse(ctxt, testDoc); + } +} + +/// Struct tests: +// Positive: strict, 3 required fields +// Negative: strict, ensure extra fields fail +// Negative: strict, duplicate fields +TEST(IDLStructTests, TestStrictStruct) { + IDLParserErrorContext ctxt("root"); + + // Positive: Just 3 required fields + { + auto testDoc = BSON("field1" << 12 << "field2" << 123 << "field3" << 1234); + RequiredStrictField3::parse(ctxt, testDoc); + } + + // Negative: Missing 1 required field + { + auto testDoc = BSON("field2" << 123 << "field3" << 1234); + ASSERT_THROWS(RequiredStrictField3::parse(ctxt, testDoc), UserException); + } + { + auto testDoc = BSON("field1" << 12 << "field3" << 1234); + ASSERT_THROWS(RequiredStrictField3::parse(ctxt, testDoc), UserException); + } + { + auto testDoc = BSON("field1" << 12 << "field2" << 123); + ASSERT_THROWS(RequiredStrictField3::parse(ctxt, testDoc), UserException); + } + + // Negative: Extra field + { + auto testDoc = + BSON("field1" << 12 << "field2" << 123 << "field3" << 1234 << "field4" << 1234); + ASSERT_THROWS(RequiredStrictField3::parse(ctxt, testDoc), UserException); + } + + // Negative: Duplicate field + { + auto testDoc = + BSON("field1" << 12 << "field2" << 123 << "field3" << 1234 << "field2" << 12345); + ASSERT_THROWS(RequiredStrictField3::parse(ctxt, testDoc), UserException); + } +} +// Positive: non-strict, ensure extra fields work +// Negative: non-strict, duplicate fields +TEST(IDLStructTests, TestNonStrictStruct) { + IDLParserErrorContext ctxt("root"); + + // Positive: Just 3 required fields + { + auto testDoc = BSON("field1" << 12 << "field2" << 123 << "field3" << 1234); + RequiredNonStrictField3::parse(ctxt, testDoc); + } + + // Negative: Missing 1 required field + { + auto testDoc = BSON("field2" << 123 << "field3" << 1234); + ASSERT_THROWS(RequiredNonStrictField3::parse(ctxt, testDoc), UserException); + } + { + auto testDoc = BSON("field1" << 12 << "field3" << 1234); + ASSERT_THROWS(RequiredNonStrictField3::parse(ctxt, testDoc), UserException); + } + { + auto testDoc = BSON("field1" << 12 << "field2" << 123); + ASSERT_THROWS(RequiredNonStrictField3::parse(ctxt, testDoc), UserException); + } + + // Positive: Extra field + { + auto testDoc = + BSON("field1" << 12 << "field2" << 123 << "field3" << 1234 << "field4" << 1234); + RequiredNonStrictField3::parse(ctxt, testDoc); + } + + // Negative: Duplicate field + { + auto testDoc = + BSON("field1" << 12 << "field2" << 123 << "field3" << 1234 << "field2" << 12345); + ASSERT_THROWS(RequiredNonStrictField3::parse(ctxt, testDoc), UserException); + } + + // Negative: Duplicate extra field + { + auto testDoc = BSON( + "field4" << 1234 << "field1" << 12 << "field2" << 123 << "field3" << 1234 << "field4" + << 1234); + ASSERT_THROWS(RequiredNonStrictField3::parse(ctxt, testDoc), UserException); + } +} + +/// Field tests +// Positive: check ignored field is ignored +TEST(IDLFieldTests, TestStrictStructIgnoredField) { + IDLParserErrorContext ctxt("root"); + + // Positive: Test ignored field is ignored + { + auto testDoc = BSON("required_field" << 12 << "ignored_field" << 123); + IgnoredField::parse(ctxt, testDoc); + } + + // Positive: Test ignored field is not required + { + auto testDoc = BSON("required_field" << 12); + IgnoredField::parse(ctxt, testDoc); + } +} + +// First test: test an empty document and the default value +// Second test: test a non-empty document and that we do not get the default value +#define TEST_DEFAULT_VALUES(field_name, default_value, new_value) \ + { \ + auto testDoc = BSONObj(); \ + auto testStruct = Default_values::parse(ctxt, testDoc); \ + ASSERT_EQUALS(testStruct.get##field_name(), default_value); \ + } \ + { \ + auto testDoc = BSON(#field_name << new_value); \ + auto testStruct = Default_values::parse(ctxt, testDoc); \ + ASSERT_EQUALS(testStruct.get##field_name(), new_value); \ + } + +// Mixed: struct strict, and ignored field works +TEST(IDLFieldTests, TestDefaultFields) { + IDLParserErrorContext ctxt("root"); + + TEST_DEFAULT_VALUES(V_string, "a default", "foo"); + TEST_DEFAULT_VALUES(V_int, 42, 3); + TEST_DEFAULT_VALUES(V_long, 423, 4LL); + TEST_DEFAULT_VALUES(V_double, 3.14159, 2.8); + TEST_DEFAULT_VALUES(V_bool, true, false); +} + +// Positive: struct strict, and optional field works +TEST(IDLFieldTests, TestOptionalFields) { + IDLParserErrorContext ctxt("root"); + + + // Positive: Test document with only string field + { + auto testDoc = BSON("field1" + << "Foo"); + auto testStruct = Optional_field::parse(ctxt, testDoc); + + static_assert(std::is_same<decltype(testStruct.getField2()), + const boost::optional<std::int32_t>>::value, + "expected int32"); + static_assert(std::is_same<decltype(testStruct.getField1()), + const boost::optional<mongo::StringData>>::value, + "expected StringData"); + + ASSERT_EQUALS("Foo", testStruct.getField1().get()); + ASSERT_FALSE(testStruct.getField2().is_initialized()); + } + + // Positive: Serialize struct with only string field + { + BSONObjBuilder builder; + Optional_field testStruct; + auto field1 = boost::optional<StringData>("Foo"); + testStruct.setField1(field1); + testStruct.serialize(&builder); + auto loopbackDoc = builder.obj(); + + auto testDoc = BSON("field1" + << "Foo"); + ASSERT_BSONOBJ_EQ(testDoc, loopbackDoc); + } + + // Positive: Test document with only int field + { + auto testDoc = BSON("field2" << 123); + auto testStruct = Optional_field::parse(ctxt, testDoc); + ASSERT_FALSE(testStruct.getField1().is_initialized()); + ASSERT_EQUALS(123, testStruct.getField2().get()); + } + + // Positive: Serialize struct with only int field + { + BSONObjBuilder builder; + Optional_field testStruct; + testStruct.setField2(123); + testStruct.serialize(&builder); + auto loopbackDoc = builder.obj(); + + auto testDoc = BSON("field2" << 123); + ASSERT_BSONOBJ_EQ(testDoc, loopbackDoc); + } +} + +/// TODO: Array tests +// Validate array parsing +// Check array vs non-array + +} // namespace mongo diff --git a/src/mongo/idl/idl_test_types.h b/src/mongo/idl/idl_test_types.h new file mode 100644 index 00000000000..ca1e7e2853c --- /dev/null +++ b/src/mongo/idl/idl_test_types.h @@ -0,0 +1,61 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjbuilder.h" + +namespace mongo { + +/** + * Simple class that demonstrates the contract a class must implement to parse an IDL "any" type. + */ +class AnyBasicType { +public: + static AnyBasicType parse(BSONElement element) { + return AnyBasicType(); + } + + void serialize(BSONObjBuilder* builder) const {} +}; + +/** + * Simple class that demonstrates the contract a class must implement to parse a BSON "object" type + * from the IDL parser. + */ +class ObjectBasicType { +public: + static ObjectBasicType parse(const BSONObj& obj) { + return ObjectBasicType(); + } + + void serialize(BSONObjBuilder* builder) const {} +}; + +} // namespace mongo diff --git a/src/mongo/idl/unittest.idl b/src/mongo/idl/unittest.idl new file mode 100644 index 00000000000..c08a31cb6ce --- /dev/null +++ b/src/mongo/idl/unittest.idl @@ -0,0 +1,311 @@ +# Copyright (C) 2017 MongoDB Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3, +# as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +# IDL Unit Tests IDL file +global: + cpp_namespace: "mongo" + cpp_includes: + - "mongo/db/namespace_string.h" + - "mongo/idl/idl_test_types.h" + +types: + string: + bson_serialization_type: string + description: "A BSON UTF-8 string" + cpp_type: "std::string" + deserializer: "mongo::BSONElement::str" + + int: + bson_serialization_type: int + description: "A BSON 32-bit integer" + cpp_type: "std::int32_t" + deserializer: "mongo::BSONElement::_numberInt" + + long: + bson_serialization_type: long + description: "A BSON 64-bit integer" + cpp_type: "std::int64_t" + deserializer: "mongo::BSONElement::_numberLong" + + double: + bson_serialization_type: double + description: "A BSON 64-bit floating point number" + cpp_type: "double" + deserializer: "mongo::BSONElement::_numberDouble" + + decimal: + bson_serialization_type: double + description: "A BSON 128-bit floating point decimal number" + cpp_type: "mongo::Decimal128" + deserializer: "mongo::BSONElement::_numberDecimal" + + bool: + bson_serialization_type: bool + description: "A BSON bool" + cpp_type: "bool" + deserializer: "mongo::BSONElement::boolean" + + # TODO: support bindata + # bindata: + # bson_serialization_type: bindata + # cpp_type: "std:" + # deserializer: "mongo::BSONElement::str" + + objectid: + bson_serialization_type: objectid + description: "A BSON ObjectID" + cpp_type: "mongo::OID" + deserializer: "mongo::BSONElement::OID" + + date: + bson_serialization_type: date + description: "A BSON UTC DateTime" + cpp_type: "mongo::Date_t" + deserializer: "mongo::BSONElement::date" + + timestamp: + bson_serialization_type: timestamp + description: "A BSON TimeStamp" + cpp_type: "mongo::Timestamp" + deserializer: "mongo::BSONElement::timestamp" + +################################################################################################## +# +# Test a custom non-BSONElement deserialization and serialization methods for a string type +# +################################################################################################## + + namespacestring: + bson_serialization_type: string + description: "A MongoDB NamespaceString" + cpp_type: "mongo::NamespaceString" + serializer: mongo::NamespaceString::toString + deserializer: mongo::NamespaceString + +################################################################################################## +# +# Test a custom non-BSONElement deserialization and serialization methods for an any type +# +################################################################################################## + + any_basic_type: + bson_serialization_type: any + description: "An Any Type" + cpp_type: "mongo::AnyBasicType" + serializer: mongo::AnyBasicType::serialize + deserializer: mongo::AnyBasicType::parse + +################################################################################################## +# +# Test a custom non-BSONElement deserialization and serialization methods for an object type +# +################################################################################################## + + object_basic_type: + bson_serialization_type: object + description: "An object Type" + cpp_type: "mongo::ObjectBasicType" + serializer: mongo::ObjectBasicType::serialize + deserializer: mongo::ObjectBasicType::parse + +################################################################################################## +# +# Test types that accept multiple serialization types +# +################################################################################################## + + safeInt32: + description: Accepts any numerical type within int32 range + cpp_type: std::int64_t + bson_serialization_type: + - long + - int + - decimal + - double + deserializer: "mongo::BSONElement::numberInt" + + +################################################################################################## +# +# Unit test structs for a single value to ensure type validation works correctly +# +################################################################################################## + +structs: + one_string: + description: UnitTest for a single string + fields: + value: string + + one_int: + description: UnitTest for a single int + fields: + value: int + + one_long: + description: UnitTest for a single long + fields: + value: long + + one_double: + description: UnitTest for a single double + fields: + value: double + + one_decimal: + description: UnitTest for a single decimal + fields: + value: decimal + + one_bool: + description: UnitTest for a single bool + fields: + value: bool + + one_objectid: + description: UnitTest for a single objectid + fields: + value: objectid + + one_date: + description: UnitTest for a single date + fields: + value: date + + one_timestamp: + description: UnitTest for a single timestamp + fields: + value: timestamp + +################################################################################################## +# +# Structs to test various options for structs/fields +# +################################################################################################## + RequiredStrictField3: + description: UnitTest for a strict struct with 3 required fields + fields: + field1: int + field2: int + field3: int + + RequiredNonStrictField3: + description: UnitTest for a non-strict struct with 3 required fields + strict: false + fields: + field1: int + field2: int + field3: int + +################################################################################################## +# +# Structs to test various options for fields +# +################################################################################################## + ignoredField: + description: UnitTest for a struct with an ignored_field + fields: + required_field: int + ignored_field: + type: int + ignore: true + + +################################################################################################## +# +# Test a custom non-BSONElement deserialization and serialization methods for a string type +# +################################################################################################## + + one_namespacestring: + description: UnitTest for a single namespacestring + fields: + value: namespacestring + +################################################################################################## +# +# Test a custom non-BSONElement deserialization and serialization methods for an any type +# +################################################################################################## + + one_any_basic_type: + description: UnitTest for a single any type + fields: + value: any_basic_type + + +################################################################################################## +# +# Test a custom non-BSONElement deserialization and serialization methods for an object type +# +################################################################################################## + + one_object_basic_type: + description: UnitTest for a single object type + fields: + value: object_basic_type + + +################################################################################################## +# +# Test types that accept multiple serialization types +# +################################################################################################## + + one_safeint32: + description: UnitTest for a single safeInt32 + fields: + value: safeInt32 + +################################################################################################## +# +# Test fields with default values +# +################################################################################################## + + default_values: + description: UnitTest for a single safeInt32 + fields: + V_string: + type: string + default: '"a default"' + V_int: + type: int + default: 42 + V_long: + type: long + default: 423 + V_double: + type: double + default: 3.14159 + V_bool: + type: bool + default: true + +################################################################################################## +# +# Test fields with optional values +# +################################################################################################## + + optional_field: + description: UnitTest for a optional field + fields: + field1: + type: string + optional: true + field2: + type: int + optional: true |