diff options
author | Mark Benvenuto <mark.benvenuto@mongodb.com> | 2017-12-29 12:15:35 -0500 |
---|---|---|
committer | Mark Benvenuto <mark.benvenuto@mongodb.com> | 2017-12-29 12:15:35 -0500 |
commit | 449905c81af3803ebb16eb8f7ca779ae783517d9 (patch) | |
tree | 4336c1e1efcd1646b7e2dc5b9968cd0d2c8f11e8 | |
parent | 9fc948694d27b5ed78f713b026cb6e5661c95551 (diff) | |
download | mongo-449905c81af3803ebb16eb8f7ca779ae783517d9.tar.gz |
SERVER-32386 IDL should allow custom types for command's first element
-rw-r--r-- | buildscripts/idl/idl/ast.py | 4 | ||||
-rw-r--r-- | buildscripts/idl/idl/binder.py | 83 | ||||
-rw-r--r-- | buildscripts/idl/idl/common.py | 1 | ||||
-rw-r--r-- | buildscripts/idl/idl/errors.py | 10 | ||||
-rw-r--r-- | buildscripts/idl/idl/generator.py | 168 | ||||
-rw-r--r-- | buildscripts/idl/idl/parser.py | 25 | ||||
-rw-r--r-- | buildscripts/idl/idl/struct_types.py | 97 | ||||
-rw-r--r-- | buildscripts/idl/idl/syntax.py | 4 | ||||
-rw-r--r-- | buildscripts/idl/tests/test_binder.py | 63 | ||||
-rw-r--r-- | buildscripts/idl/tests/test_parser.py | 99 | ||||
-rw-r--r-- | src/mongo/idl/idl_test.cpp | 215 | ||||
-rw-r--r-- | src/mongo/idl/unittest.idl | 30 |
12 files changed, 700 insertions, 99 deletions
diff --git a/buildscripts/idl/idl/ast.py b/buildscripts/idl/idl/ast.py index c1bbba4f266..ef5a0aba44d 100644 --- a/buildscripts/idl/idl/ast.py +++ b/buildscripts/idl/idl/ast.py @@ -76,10 +76,13 @@ class Struct(common.SourceLocation): All fields are either required or have a non-None default. """ + # pylint: disable=too-many-instance-attributes + def __init__(self, file_name, line, column): # type: (unicode, int, int) -> None """Construct a struct.""" self.name = None # type: unicode + self.cpp_name = None # type: unicode self.description = None # type: unicode self.strict = True # type: bool self.immutable = False # type: bool @@ -150,6 +153,7 @@ class Command(Struct): # type: (unicode, int, int) -> None """Construct a command.""" self.namespace = None # type: unicode + self.command_field = None # type: Field super(Command, self).__init__(file_name, line, column) diff --git a/buildscripts/idl/idl/binder.py b/buildscripts/idl/idl/binder.py index d0888f7efc0..f8809ea503d 100644 --- a/buildscripts/idl/idl/binder.py +++ b/buildscripts/idl/idl/binder.py @@ -245,6 +245,9 @@ def _bind_struct_common(ctxt, parsed_spec, struct, ast_struct): ast_struct.immutable = struct.immutable ast_struct.inline_chained_structs = struct.inline_chained_structs ast_struct.generate_comparison_operators = struct.generate_comparison_operators + ast_struct.cpp_name = struct.name + if struct.cpp_name: + ast_struct.cpp_name = struct.cpp_name # Validate naming restrictions if ast_struct.name.startswith("array<"): @@ -334,6 +337,71 @@ def _inject_hidden_command_fields(command): command.fields.append(db_field) +def _bind_command_type(ctxt, parsed_spec, command): + # type: (errors.ParserContext, syntax.IDLSpec, syntax.Command) -> ast.Field + """Bind the type field in a command as the first field.""" + # pylint: disable=too-many-branches,too-many-statements + ast_field = ast.Field(command.file_name, command.line, command.column) + ast_field.name = command.name + ast_field.description = command.description + ast_field.optional = False + ast_field.supports_doc_sequence = False + ast_field.serialize_op_msg_request_only = False + ast_field.constructed = False + + ast_field.cpp_name = "CommandParameter" + + # Validate naming restrictions + if ast_field.name.startswith("array<"): + ctxt.add_array_not_valid_error(ast_field, "field", ast_field.name) + + # Resolve the command type as a field + syntax_symbol = parsed_spec.symbols.resolve_field_type(ctxt, command, command.name, + command.type) + if syntax_symbol is None: + return None + + if isinstance(syntax_symbol, syntax.Command): + ctxt.add_bad_command_as_field_error(ast_field, command.type) + return None + + assert not isinstance(syntax_symbol, syntax.Enum) + + # If the field type is an array, mark the AST version as such. + if syntax.parse_array_type(command.type): + ast_field.array = True + + # Copy over only the needed information if this a struct or a type + if isinstance(syntax_symbol, syntax.Struct): + struct = cast(syntax.Struct, syntax_symbol) + cpp_name = struct.name + if struct.cpp_name: + cpp_name = struct.cpp_name + ast_field.struct_type = common.qualify_cpp_name(struct.cpp_namespace, cpp_name) + ast_field.bson_serialization_type = ["object"] + + _validate_field_of_type_struct(ctxt, ast_field) + else: + # Produce the union of type information for the type and this field. + idltype = cast(syntax.Type, syntax_symbol) + + # Copy over the type fields first + ast_field.cpp_type = idltype.cpp_type + ast_field.bson_serialization_type = idltype.bson_serialization_type + ast_field.bindata_subtype = idltype.bindata_subtype + ast_field.serializer = _normalize_method_name(idltype.cpp_type, idltype.serializer) + ast_field.deserializer = _normalize_method_name(idltype.cpp_type, idltype.deserializer) + ast_field.default = idltype.default + + # Validate merged type + _validate_type_properties(ctxt, ast_field, "command.type") + + # Validate merged type + _validate_field_properties(ctxt, ast_field) + + return ast_field + + def _bind_command(ctxt, parsed_spec, command): # type: (errors.ParserContext, syntax.IDLSpec, syntax.Command) -> ast.Command """ @@ -352,6 +420,9 @@ def _bind_command(ctxt, parsed_spec, command): ast_command.namespace = command.namespace + if command.type: + ast_command.command_field = _bind_command_type(ctxt, parsed_spec, command) + if [field for field in ast_command.fields if field.name == ast_command.name]: ctxt.add_bad_command_name_duplicates_field(ast_command, ast_command.name) @@ -368,7 +439,7 @@ def _validate_ignored_field(ctxt, field): def _validate_field_of_type_struct(ctxt, field): - # type: (errors.ParserContext, syntax.Field) -> None + # type: (errors.ParserContext, Union[syntax.Field, ast.Field]) -> None """Validate that for fields with a type of struct, no other properties are set.""" if field.default is not None: ctxt.add_struct_field_must_be_empty_error(field, field.name, "default") @@ -494,7 +565,10 @@ def _bind_field(ctxt, parsed_spec, field): # Copy over only the needed information if this a struct or a type if isinstance(syntax_symbol, syntax.Struct): struct = cast(syntax.Struct, syntax_symbol) - ast_field.struct_type = common.qualify_cpp_name(struct.cpp_namespace, struct.name) + cpp_name = struct.name + if struct.cpp_name: + cpp_name = struct.cpp_name + ast_field.struct_type = common.qualify_cpp_name(struct.cpp_namespace, cpp_name) ast_field.bson_serialization_type = ["object"] _validate_field_of_type_struct(ctxt, field) @@ -597,7 +671,10 @@ def _bind_chained_struct(ctxt, parsed_spec, ast_struct, chained_struct): ast_chained_field.name = struct.name ast_chained_field.cpp_name = chained_struct.cpp_name ast_chained_field.description = struct.description - ast_chained_field.struct_type = struct.name + cpp_name = struct.name + if struct.cpp_name: + cpp_name = struct.cpp_name + ast_chained_field.struct_type = cpp_name ast_chained_field.bson_serialization_type = ["object"] ast_chained_field.chained = True diff --git a/buildscripts/idl/idl/common.py b/buildscripts/idl/idl/common.py index 1a52477998d..eee647e0219 100644 --- a/buildscripts/idl/idl/common.py +++ b/buildscripts/idl/idl/common.py @@ -26,6 +26,7 @@ from typing import Mapping COMMAND_NAMESPACE_CONCATENATE_WITH_DB = "concatenate_with_db" COMMAND_NAMESPACE_IGNORED = "ignored" +COMMAND_NAMESPACE_TYPE = "type" def title_case(name): diff --git a/buildscripts/idl/idl/errors.py b/buildscripts/idl/idl/errors.py index a883d09d40a..ed2963f1251 100644 --- a/buildscripts/idl/idl/errors.py +++ b/buildscripts/idl/idl/errors.py @@ -85,6 +85,7 @@ ERROR_ID_COMMAND_DUPLICATES_FIELD = "ID0048" ERROR_ID_IS_NODE_VALID_INT = "ID0049" ERROR_ID_IS_NODE_VALID_NON_NEGATIVE_INT = "ID0050" ERROR_ID_IS_DUPLICATE_COMPARISON_ORDER = "ID0051" +ERROR_ID_IS_COMMAND_TYPE_EXTRANEOUS = "ID0052" class IDLError(Exception): @@ -658,6 +659,15 @@ class ParserContext(object): ("Struct '%s' cannot have two fields with the same comparison_order value '%d'.") % (struct_name, comparison_order)) + def add_extranous_command_type(self, location, command_name): + # type: (common.SourceLocation, unicode) -> None + """Add an error about commands having type when not needed.""" + # pylint: disable=invalid-name + self._add_error( + location, ERROR_ID_IS_COMMAND_TYPE_EXTRANEOUS, + ("Command '%s' cannot have a 'type' property unless namespace equals 'type'.") % + (command_name)) + def _assert_unique_error_messages(): # type: () -> None diff --git a/buildscripts/idl/idl/generator.py b/buildscripts/idl/idl/generator.py index 7ea82ecf16e..f1b9cb97bdb 100644 --- a/buildscripts/idl/idl/generator.py +++ b/buildscripts/idl/idl/generator.py @@ -105,6 +105,18 @@ def _get_bson_type_check(bson_element, ctxt_name, field): return '%s.checkAndAssertTypes(%s, %s)' % (ctxt_name, bson_element, type_list) +def _get_all_fields(struct): + # type: (ast.Struct) -> List[ast.Field] + """Get a list of all the fields, including the command field.""" + all_fields = [] + if isinstance(struct, ast.Command) and struct.command_field: + all_fields.append(struct.command_field) + + all_fields += struct.fields + + return sorted([field for field in all_fields], key=lambda f: f.cpp_name) + + class _FieldUsageCheckerBase(object): """Check for duplicate fields, and required fields as needed.""" @@ -494,9 +506,8 @@ class _CppHeaderFileWriter(_CppFileWriterBase): # type: (ast.Struct) -> None # pylint: disable=invalid-name """Generate a StringData constant for field name.""" - sorted_fields = sorted([field for field in struct.fields], key=lambda f: f.cpp_name) - for field in sorted_fields: + for field in _get_all_fields(struct): self._writer.write_line( common.template_args( 'static constexpr auto ${constant_name} = "${field_name}"_sd;', @@ -535,16 +546,22 @@ class _CppHeaderFileWriter(_CppFileWriterBase): def gen_op_msg_request_methods(self, command): # type: (ast.Command) -> None """Generate the methods for a command.""" - struct_type_info = struct_types.get_struct_info(command) - struct_type_info.gen_getter_method(self._writer) + if command.command_field: + self.gen_getter(command.command_field) + else: + struct_type_info = struct_types.get_struct_info(command) + struct_type_info.gen_getter_method(self._writer) self._writer.write_empty_line() def gen_op_msg_request_member(self, command): # type: (ast.Command) -> None """Generate the class members for a command.""" - struct_type_info = struct_types.get_struct_info(command) - struct_type_info.gen_member(self._writer) + if command.command_field: + self.gen_member(command.command_field) + else: + struct_type_info = struct_types.get_struct_info(command) + struct_type_info.gen_member(self._writer) self._writer.write_empty_line() @@ -658,7 +675,7 @@ class _CppHeaderFileWriter(_CppFileWriterBase): for struct in spec_and_structs: self.gen_description_comment(struct.description) - with self.gen_class_declaration_block(struct.name): + with self.gen_class_declaration_block(struct.cpp_name): self.write_unindented_line('public:') # Generate a sorted list of string constants @@ -782,8 +799,8 @@ class _CppSourceFileWriter(_CppFileWriterBase): return '%s(%s)' % (method_name, element_name) - def _gen_array_deserializer(self, field): - # type: (ast.Field) -> None + def _gen_array_deserializer(self, field, bson_element): + # type: (ast.Field, unicode) -> None """Generate the C++ deserializer piece for an array field.""" cpp_type_info = cpp_types.get_cpp_type(field) cpp_type = cpp_type_info.get_type_name() @@ -794,7 +811,7 @@ class _CppSourceFileWriter(_CppFileWriterBase): self._writer.write_line('std::vector<%s> values;' % (cpp_type)) self._writer.write_empty_line() - self._writer.write_line('const BSONObj arrayObject = element.Obj();') + self._writer.write_line('const BSONObj arrayObject = %s.Obj();' % (bson_element)) with self._block('for (const auto& arrayElement : arrayObject) {', '}'): @@ -837,11 +854,11 @@ class _CppSourceFileWriter(_CppFileWriterBase): else: self._writer.write_line('%s = std::move(values);' % (_get_field_member_name(field))) - def gen_field_deserializer(self, field, bson_object): - # type: (ast.Field, unicode) -> None + def gen_field_deserializer(self, field, bson_object, bson_element): + # type: (ast.Field, unicode, unicode) -> None """Generate the C++ deserializer piece for a field.""" if field.array: - self._gen_array_deserializer(field) + self._gen_array_deserializer(field, bson_element) return if field.chained: @@ -859,11 +876,11 @@ class _CppSourceFileWriter(_CppFileWriterBase): self._writer.write_line('%s = %s;' % (_get_field_member_name(field), expression)) else: # May be an empty block if the type is 'any' - predicate = _get_bson_type_check('element', 'ctxt', field) + predicate = _get_bson_type_check(bson_element, 'ctxt', field) if predicate: predicate = "MONGO_likely(%s)" % (predicate) with self._predicate(predicate): - object_value = self._gen_field_deserializer_expression('element', field) + object_value = self._gen_field_deserializer_expression(bson_element, field) if field.chained_struct_field: self._writer.write_line('%s.%s(%s);' % (_get_field_member_name(field.chained_struct_field), @@ -962,6 +979,19 @@ class _CppSourceFileWriter(_CppFileWriterBase): with self._block('%s %s {' % (constructor.get_definition(), initializers_str), '}'): self._writer.write_line('// Used for initialization only') + def _gen_command_deserializer(self, struct, bson_object): + # type: (ast.Struct, unicode) -> None + """Generate the command field deserializer.""" + + if isinstance(struct, ast.Command) and struct.command_field: + with self._block('{', '}'): + self.gen_field_deserializer(struct.command_field, bson_object, "commandElement") + else: + struct_type_info = struct_types.get_struct_info(struct) + + # Generate namespace check now that "$db" has been read or defaulted + struct_type_info.gen_namespace_check(self._writer, "_dbName", "commandElement") + def _gen_fields_deserializer_common(self, struct, bson_object): # type: (ast.Struct, unicode) -> _FieldUsageCheckerBase """Generate the C++ code to deserialize list of fields.""" @@ -1009,7 +1039,7 @@ class _CppSourceFileWriter(_CppFileWriterBase): self._writer.write_line('%s = true;' % (_get_has_field_member_name(field))) - self.gen_field_deserializer(field, bson_object) + self.gen_field_deserializer(field, bson_object, "element") if first_field: first_field = False @@ -1035,7 +1065,7 @@ class _CppSourceFileWriter(_CppFileWriterBase): continue # Simply generate deserializers since these are all 'any' types - self.gen_field_deserializer(field, bson_object) + self.gen_field_deserializer(field, bson_object, "element") self._writer.write_empty_line() self._writer.write_empty_line() @@ -1051,10 +1081,17 @@ class _CppSourceFileWriter(_CppFileWriterBase): with self._block('%s {' % (func_def), '}'): if isinstance(struct, ast.Command) and struct.namespace != common.COMMAND_NAMESPACE_IGNORED: - self._writer.write_line('NamespaceString localNS;') - self._writer.write_line('%s object(localNS);' % (common.title_case(struct.name))) + if struct.namespace == common.COMMAND_NAMESPACE_TYPE: + cpp_type_info = cpp_types.get_cpp_type(struct.command_field) + self._writer.write_line('%s localCmdType;' % (cpp_type_info.get_storage_type())) + self._writer.write_line('%s object(localCmdType);' % + (common.title_case(struct.cpp_name))) + elif struct.namespace == common.COMMAND_NAMESPACE_CONCATENATE_WITH_DB: + self._writer.write_line('NamespaceString localNS;') + self._writer.write_line('%s object(localNS);' % + (common.title_case(struct.cpp_name))) else: - self._writer.write_line('%s object;' % common.title_case(struct.name)) + self._writer.write_line('%s object;' % common.title_case(struct.cpp_name)) self._writer.write_line(method_info.get_call('object')) self._writer.write_line('return object;') @@ -1078,8 +1115,7 @@ class _CppSourceFileWriter(_CppFileWriterBase): field_usage_check.add_final_checks() self._writer.write_empty_line() - # Generate namespace check now that "$db" has been read or defaulted - struct_type_info.gen_namespace_check(self._writer, "_dbName", "commandElement") + self._gen_command_deserializer(struct, "bsonObject") def gen_op_msg_request_deserializer_methods(self, struct): # type: (ast.Struct) -> None @@ -1141,8 +1177,7 @@ class _CppSourceFileWriter(_CppFileWriterBase): field_usage_check.add_final_checks() self._writer.write_empty_line() - # Generate namespace check now that "$db" has been read or defaulted - struct_type_info.gen_namespace_check(self._writer, "_dbName", "commandElement") + self._gen_command_deserializer(struct, "request.body") def _gen_serializer_method_custom(self, field): # type: (ast.Field) -> None @@ -1237,10 +1272,41 @@ class _CppSourceFileWriter(_CppFileWriterBase): 'BSONObjBuilder subObjBuilder(builder->subobjStart(${field_name}));') self._writer.write_template('${access_member}.serialize(&subObjBuilder);') + def _gen_serializer_method_common(self, field): + # type: (ast.Field) -> None + """Generate the serialize method definition.""" + member_name = _get_field_member_name(field) + + # Is this a scalar bson C++ type? + bson_cpp_type = cpp_types.get_bson_cpp_type(field) + + needs_custom_serializer = field.serializer or (bson_cpp_type and + bson_cpp_type.has_serializer()) + + optional_block_start = None + if field.optional: + optional_block_start = 'if (%s.is_initialized()) {' % (member_name) + elif field.struct_type or needs_custom_serializer or field.array: + # Introduce a new scope for required nested object serialization. + optional_block_start = '{' + + with self._block(optional_block_start, '}'): + + if not field.struct_type: + if needs_custom_serializer: + self._gen_serializer_method_custom(field) + else: + # Generate default serialization using BSONObjBuilder::append + # Note: BSONObjBuilder::append has overrides for std::vector also + self._writer.write_line( + 'builder->append(%s, %s);' % + (_get_field_constant_name(field), _access_member(field))) + else: + self._gen_serializer_method_struct(field) + def _gen_serializer_methods_common(self, struct, is_op_msg_request): # type: (ast.Struct, bool) -> None """Generate the serialize method definition.""" - # pylint: disable=too-many-branches struct_type_info = struct_types.get_struct_info(struct) @@ -1257,8 +1323,11 @@ class _CppSourceFileWriter(_CppFileWriterBase): # Serialize the namespace as the first field if isinstance(struct, ast.Command): - struct_type_info = struct_types.get_struct_info(struct) - struct_type_info.gen_serializer(self._writer) + if struct.command_field: + self._gen_serializer_method_common(struct.command_field) + else: + struct_type_info = struct_types.get_struct_info(struct) + struct_type_info.gen_serializer(self._writer) for field in struct.fields: # If fields are meant to be ignored during deserialization, there is no need to @@ -1279,34 +1348,7 @@ class _CppSourceFileWriter(_CppFileWriterBase): if field.supports_doc_sequence and is_op_msg_request: continue - member_name = _get_field_member_name(field) - - # Is this a scalar bson C++ type? - bson_cpp_type = cpp_types.get_bson_cpp_type(field) - - needs_custom_serializer = field.serializer or (bson_cpp_type and - bson_cpp_type.has_serializer()) - - optional_block_start = None - if field.optional: - optional_block_start = 'if (%s.is_initialized()) {' % (member_name) - elif field.struct_type or needs_custom_serializer or field.array: - # Introduce a new scope for required nested object serialization. - optional_block_start = '{' - - with self._block(optional_block_start, '}'): - - if not field.struct_type: - if needs_custom_serializer: - self._gen_serializer_method_custom(field) - else: - # Generate default serialization using BSONObjBuilder::append - # Note: BSONObjBuilder::append has overrides for std::vector also - self._writer.write_line( - 'builder->append(%s, %s);' % - (_get_field_constant_name(field), _access_member(field))) - else: - self._gen_serializer_method_struct(field) + self._gen_serializer_method_common(field) # Add a blank line after each block self._writer.write_empty_line() @@ -1406,22 +1448,18 @@ class _CppSourceFileWriter(_CppFileWriterBase): # pylint: disable=invalid-name """Generate a StringData constant for field name in the cpp file.""" - # Generate a sorted list of string constants - - sorted_fields = sorted([field for field in struct.fields], key=lambda f: f.cpp_name) - - for field in sorted_fields: + for field in _get_all_fields(struct): self._writer.write_line( common.template_args( 'constexpr StringData ${class_name}::${constant_name};', - class_name=common.title_case(struct.name), + class_name=common.title_case(struct.cpp_name), constant_name=_get_field_constant_name(field))) if isinstance(struct, ast.Command): self._writer.write_line( common.template_args( 'constexpr StringData ${class_name}::kCommandName;', - class_name=common.title_case(struct.name))) + class_name=common.title_case(struct.cpp_name))) def gen_enum_definition(self, idl_enum): # type: (ast.Enum) -> None @@ -1442,7 +1480,7 @@ class _CppSourceFileWriter(_CppFileWriterBase): block_name = common.template_args( 'const std::vector<StringData> ${class_name}::_knownFields {', - class_name=common.title_case(struct.name)) + class_name=common.title_case(struct.cpp_name)) with self._block(block_name, "};"): sorted_fields = sorted([field for field in struct.fields], key=lambda f: f.cpp_name) @@ -1450,12 +1488,12 @@ class _CppSourceFileWriter(_CppFileWriterBase): self._writer.write_line( common.template_args( '${class_name}::${constant_name},', - class_name=common.title_case(struct.name), + class_name=common.title_case(struct.cpp_name), constant_name=_get_field_constant_name(field))) self._writer.write_line( common.template_args( - '${class_name}::kCommandName,', class_name=common.title_case(struct.name))) + '${class_name}::kCommandName,', class_name=common.title_case(struct.cpp_name))) def generate(self, spec, header_file_name): # type: (ast.IDLAST, unicode) -> None diff --git a/buildscripts/idl/idl/parser.py b/buildscripts/idl/idl/parser.py index 49821707c94..2fdd1034b5b 100644 --- a/buildscripts/idl/idl/parser.py +++ b/buildscripts/idl/idl/parser.py @@ -419,6 +419,8 @@ def _parse_command(ctxt, spec, name, node): "chained_structs": _RuleDesc('mapping', mapping_parser_func=_parse_chained_structs), "fields": _RuleDesc('mapping', mapping_parser_func=_parse_fields), "namespace": _RuleDesc('scalar', _RuleDesc.REQUIRED), + "cpp_name": _RuleDesc('scalar'), + "type": _RuleDesc('scalar'), "strict": _RuleDesc("bool_scalar"), "inline_chained_structs": _RuleDesc("bool_scalar"), "immutable": _RuleDesc('bool_scalar'), @@ -427,14 +429,25 @@ def _parse_command(ctxt, spec, name, node): # TODO: support the first argument as UUID depending on outcome of Catalog Versioning changes. valid_commands = [ - common.COMMAND_NAMESPACE_CONCATENATE_WITH_DB, common.COMMAND_NAMESPACE_IGNORED + common.COMMAND_NAMESPACE_CONCATENATE_WITH_DB, common.COMMAND_NAMESPACE_IGNORED, + common.COMMAND_NAMESPACE_TYPE ] - if command.namespace and command.namespace not in valid_commands: - ctxt.add_bad_command_namespace_error(command, command.name, command.namespace, - valid_commands) - if command.fields is None: - ctxt.add_empty_struct_error(node, command.name) + if command.namespace: + if command.namespace not in valid_commands: + ctxt.add_bad_command_namespace_error(command, command.name, command.namespace, + valid_commands) + + # type property must be specified for a namespace = type + if command.namespace == common.COMMAND_NAMESPACE_TYPE and not command.type: + ctxt.add_missing_required_field_error(node, "command", "type") + + if command.namespace != common.COMMAND_NAMESPACE_TYPE and command.type: + ctxt.add_extranous_command_type(command, command.name) + + # Commands may only have the first parameter, ensure the fields property is an empty array. + if not command.fields: + command.fields = [] spec.symbols.add_command(ctxt, command) diff --git a/buildscripts/idl/idl/struct_types.py b/buildscripts/idl/idl/struct_types.py index 9e2a9504acd..81a845daf2c 100644 --- a/buildscripts/idl/idl/struct_types.py +++ b/buildscripts/idl/idl/struct_types.py @@ -21,6 +21,7 @@ from typing import Optional, List from . import ast from . import common +from . import cpp_types from . import writer @@ -216,12 +217,12 @@ class _StructTypeInfo(StructTypeInfoBase): def get_constructor_method(self): # type: () -> MethodInfo - class_name = common.title_case(self._struct.name) + class_name = common.title_case(self._struct.cpp_name) return MethodInfo(class_name, class_name, []) def get_deserializer_static_method(self): # type: () -> MethodInfo - class_name = common.title_case(self._struct.name) + class_name = common.title_case(self._struct.cpp_name) return MethodInfo( class_name, 'parse', ['const IDLParserErrorContext& ctxt', 'const BSONObj& bsonObject'], @@ -231,20 +232,21 @@ class _StructTypeInfo(StructTypeInfoBase): def get_deserializer_method(self): # type: () -> MethodInfo return MethodInfo( - common.title_case(self._struct.name), 'parseProtected', + common.title_case(self._struct.cpp_name), 'parseProtected', ['const IDLParserErrorContext& ctxt', 'const BSONObj& bsonObject'], 'void') def get_serializer_method(self): # type: () -> MethodInfo return MethodInfo( - common.title_case(self._struct.name), + common.title_case(self._struct.cpp_name), 'serialize', ['BSONObjBuilder* builder'], 'void', const=True) def get_to_bson_method(self): # type: () -> MethodInfo - return MethodInfo(common.title_case(self._struct.name), 'toBSON', [], 'BSONObj', const=True) + return MethodInfo( + common.title_case(self._struct.cpp_name), 'toBSON', [], 'BSONObj', const=True) def get_op_msg_request_serializer_method(self): # type: () -> Optional[MethodInfo] @@ -288,14 +290,14 @@ class _CommandBaseTypeInfo(_StructTypeInfo): def get_op_msg_request_serializer_method(self): # type: () -> Optional[MethodInfo] return MethodInfo( - common.title_case(self._struct.name), + common.title_case(self._struct.cpp_name), 'serialize', ['const BSONObj& commandPassthroughFields'], 'OpMsgRequest', const=True) def get_op_msg_request_deserializer_static_method(self): # type: () -> Optional[MethodInfo] - class_name = common.title_case(self._struct.name) + class_name = common.title_case(self._struct.cpp_name) return MethodInfo( class_name, 'parse', ['const IDLParserErrorContext& ctxt', 'const OpMsgRequest& request'], @@ -305,7 +307,7 @@ class _CommandBaseTypeInfo(_StructTypeInfo): def get_op_msg_request_deserializer_method(self): # type: () -> Optional[MethodInfo] return MethodInfo( - common.title_case(self._struct.name), 'parseProtected', + common.title_case(self._struct.cpp_name), 'parseProtected', ['const IDLParserErrorContext& ctxt', 'const OpMsgRequest& request'], 'void') @@ -322,7 +324,7 @@ class _IgnoredCommandTypeInfo(_CommandBaseTypeInfo): def get_serializer_method(self): # type: () -> MethodInfo return MethodInfo( - common.title_case(self._struct.name), + common.title_case(self._struct.cpp_name), 'serialize', ['const BSONObj& commandPassthroughFields', 'BSONObjBuilder* builder'], 'void', const=True) @@ -331,7 +333,7 @@ class _IgnoredCommandTypeInfo(_CommandBaseTypeInfo): # type: () -> MethodInfo # Commands that require namespaces require it as a parameter to serialize() return MethodInfo( - common.title_case(self._struct.name), + common.title_case(self._struct.cpp_name), 'toBSON', ['const BSONObj& commandPassthroughFields'], 'BSONObj', const=True) @@ -361,6 +363,68 @@ class _IgnoredCommandTypeInfo(_CommandBaseTypeInfo): pass +class _CommandFromType(_CommandBaseTypeInfo): + """Class for command code generation for custom type.""" + + def __init__(self, command): + # type: (ast.Command) -> None + """Create a _CommandFromType instance.""" + assert command.command_field + self._command = command + super(_CommandFromType, self).__init__(command) + + def get_constructor_method(self): + # type: () -> MethodInfo + cpp_type_info = cpp_types.get_cpp_type(self._command.command_field) + # Use the storage type for the constructor argument since the generated code will use + # std::move. + member_type = cpp_type_info.get_storage_type() + + class_name = common.title_case(self._struct.cpp_name) + + arg = "const %s %s" % (member_type, common.camel_case(self._command.command_field.cpp_name)) + return MethodInfo(class_name, class_name, [arg], explicit=True) + + def get_serializer_method(self): + # type: () -> MethodInfo + return MethodInfo( + common.title_case(self._struct.cpp_name), + 'serialize', ['const BSONObj& commandPassthroughFields', 'BSONObjBuilder* builder'], + 'void', + const=True) + + def get_to_bson_method(self): + # type: () -> MethodInfo + return MethodInfo( + common.title_case(self._struct.cpp_name), + 'toBSON', ['const BSONObj& commandPassthroughFields'], + 'BSONObj', + const=True) + + def get_deserializer_method(self): + # type: () -> MethodInfo + return MethodInfo( + common.title_case(self._struct.cpp_name), 'parseProtected', + ['const IDLParserErrorContext& ctxt', 'const BSONObj& bsonObject'], 'void') + + def gen_getter_method(self, indented_writer): + # type: (writer.IndentedTextWriter) -> None + raise NotImplementedError + + def gen_member(self, indented_writer): + # type: (writer.IndentedTextWriter) -> None + raise NotImplementedError + + def gen_serializer(self, indented_writer): + # type: (writer.IndentedTextWriter) -> None + raise NotImplementedError + + def gen_namespace_check(self, indented_writer, db_name, element): + # type: (writer.IndentedTextWriter, unicode, unicode) -> None + # TODO: should the name of the first element be validated?? + raise NotImplementedError + + class _CommandWithNamespaceTypeInfo(_CommandBaseTypeInfo): """Class for command code generation.""" @@ -373,13 +437,13 @@ class _CommandWithNamespaceTypeInfo(_CommandBaseTypeInfo): def get_constructor_method(self): # type: () -> MethodInfo - class_name = common.title_case(self._struct.name) + class_name = common.title_case(self._struct.cpp_name) return MethodInfo(class_name, class_name, ['const NamespaceString nss'], explicit=True) def get_serializer_method(self): # type: () -> MethodInfo return MethodInfo( - common.title_case(self._struct.name), + common.title_case(self._struct.cpp_name), 'serialize', ['const BSONObj& commandPassthroughFields', 'BSONObjBuilder* builder'], 'void', const=True) @@ -387,7 +451,7 @@ class _CommandWithNamespaceTypeInfo(_CommandBaseTypeInfo): def get_to_bson_method(self): # type: () -> MethodInfo return MethodInfo( - common.title_case(self._struct.name), + common.title_case(self._struct.cpp_name), 'toBSON', ['const BSONObj& commandPassthroughFields'], 'BSONObj', const=True) @@ -395,7 +459,7 @@ class _CommandWithNamespaceTypeInfo(_CommandBaseTypeInfo): def get_deserializer_method(self): # type: () -> MethodInfo return MethodInfo( - common.title_case(self._struct.name), 'parseProtected', + common.title_case(self._struct.cpp_name), 'parseProtected', ['const IDLParserErrorContext& ctxt', 'const BSONObj& bsonObject'], 'void') def gen_getter_method(self, indented_writer): @@ -427,6 +491,9 @@ def get_struct_info(struct): if isinstance(struct, ast.Command): if struct.namespace == common.COMMAND_NAMESPACE_IGNORED: return _IgnoredCommandTypeInfo(struct) - return _CommandWithNamespaceTypeInfo(struct) + elif struct.namespace == common.COMMAND_NAMESPACE_CONCATENATE_WITH_DB: + return _CommandWithNamespaceTypeInfo(struct) + else: + return _CommandFromType(struct) return _StructTypeInfo(struct) diff --git a/buildscripts/idl/idl/syntax.py b/buildscripts/idl/idl/syntax.py index dabcad44800..4aca9e333fe 100644 --- a/buildscripts/idl/idl/syntax.py +++ b/buildscripts/idl/idl/syntax.py @@ -348,6 +348,9 @@ class Struct(common.SourceLocation): self.chained_structs = None # type: List[ChainedStruct] self.fields = None # type: List[Field] + # Command only property + self.cpp_name = None # type: unicode + # Internal property that is not represented as syntax. An imported struct is read from an # imported file, and no code is generated for it. self.imported = False # type: bool @@ -369,6 +372,7 @@ class Command(Struct): # type: (unicode, int, int) -> None """Construct a Command.""" self.namespace = None # type: unicode + self.type = None # type: unicode super(Command, self).__init__(file_name, line, column) diff --git a/buildscripts/idl/tests/test_binder.py b/buildscripts/idl/tests/test_binder.py index b11d2d2984d..f0aae94372e 100644 --- a/buildscripts/idl/tests/test_binder.py +++ b/buildscripts/idl/tests/test_binder.py @@ -57,6 +57,8 @@ def indent_text(count, unindented_text): class TestBinder(testcase.IDLTestcase): """Test cases for the IDL binder.""" + # pylint: disable=too-many-public-methods + def test_empty(self): # type: () -> None """Test an empty document works.""" @@ -1529,6 +1531,67 @@ class TestBinder(testcase.IDLTestcase): supports_doc_sequence: true """), idl.errors.ERROR_ID_NO_DOC_SEQUENCE_FOR_NON_OBJECT) + def test_command_type_positive(self): + # type: () -> None + """Positive command custom type test cases.""" + test_preamble = textwrap.dedent(""" + types: + string: + description: foo + cpp_type: foo + bson_serialization_type: string + serializer: foo + deserializer: foo + """) + + # string + self.assert_bind(test_preamble + textwrap.dedent(""" + commands: + foo: + description: foo + strict: true + namespace: type + type: string + fields: + field1: string + """)) + + # array of string + self.assert_bind(test_preamble + textwrap.dedent(""" + commands: + foo: + description: foo + strict: true + namespace: type + type: array<string> + fields: + field1: string + """)) + + def test_command_type_negative(self): + # type: () -> None + """Negative command type test cases.""" + test_preamble = textwrap.dedent(""" + types: + string: + description: foo + cpp_type: foo + bson_serialization_type: string + serializer: foo + deserializer: foo + """) + + # supports_doc_sequence must be a bool + self.assert_bind_fail(test_preamble + textwrap.dedent(""" + commands: + foo: + description: foo + namespace: type + type: int + fields: + field1: string + """), idl.errors.ERROR_ID_UNKNOWN_TYPE) + if __name__ == '__main__': diff --git a/buildscripts/idl/tests/test_parser.py b/buildscripts/idl/tests/test_parser.py index 32b0fa4b627..5a94514e88a 100644 --- a/buildscripts/idl/tests/test_parser.py +++ b/buildscripts/idl/tests/test_parser.py @@ -373,6 +373,17 @@ class TestParser(testcase.IDLTestcase): foo: bar """), idl.errors.ERROR_ID_IS_NODE_VALID_BOOL) + # cpp_name is not allowed + self.assert_parse_fail( + textwrap.dedent(""" + structs: + foo: + description: foo + cpp_name: bar + fields: + foo: bar + """), idl.errors.ERROR_ID_UNKNOWN_NODE) + def test_field_positive(self): # type: () -> None """Positive field test cases.""" @@ -828,6 +839,7 @@ class TestParser(testcase.IDLTestcase): immutable: true inline_chained_structs: true generate_comparison_operators: true + cpp_name: foo fields: foo: bar """)) @@ -869,6 +881,16 @@ class TestParser(testcase.IDLTestcase): foo: bar """)) + # No fields + self.assert_parse( + textwrap.dedent(""" + commands: + foo: + description: foo + namespace: ignored + strict: true + """)) + def test_command_negative(self): # type: () -> None """Negative command test cases.""" @@ -880,16 +902,6 @@ class TestParser(testcase.IDLTestcase): foo: foo """), idl.errors.ERROR_ID_IS_NODE_TYPE) - # Missing fields - self.assert_parse_fail( - textwrap.dedent(""" - commands: - foo: - description: foo - namespace: ignored - strict: true - """), idl.errors.ERROR_ID_EMPTY_FIELDS) - # unknown field self.assert_parse_fail( textwrap.dedent(""" @@ -974,6 +986,18 @@ class TestParser(testcase.IDLTestcase): foo: string """), idl.errors.ERROR_ID_DUPLICATE_SYMBOL) + # Namespace concatenate_with_db + self.assert_parse_fail( + textwrap.dedent(""" + commands: + foo: + description: foo + namespace: concatenate_with_db + type: foobar + fields: + foo: bar + """), idl.errors.ERROR_ID_IS_COMMAND_TYPE_EXTRANEOUS) + def test_command_doc_sequence_positive(self): # type: () -> None """Positive supports_doc_sequence test cases.""" @@ -1023,6 +1047,61 @@ class TestParser(testcase.IDLTestcase): supports_doc_sequence: foo """), idl.errors.ERROR_ID_IS_NODE_VALID_BOOL) + def test_command_type_positive(self): + # type: () -> None + """Positive command custom type test cases.""" + # string + self.assert_parse( + textwrap.dedent(""" + commands: + foo: + description: foo + strict: true + namespace: type + type: string + fields: + foo: bar + """)) + + # array of string + self.assert_parse( + textwrap.dedent(""" + commands: + foo: + description: foo + strict: true + namespace: type + type: array<string> + fields: + foo: bar + """)) + + # no fields + self.assert_parse( + textwrap.dedent(""" + commands: + foo: + description: foo + strict: true + namespace: type + type: string + """)) + + def test_command_type_negative(self): + # type: () -> None + """Negative command type test cases.""" + + # supports_doc_sequence must be a bool + self.assert_parse_fail( + textwrap.dedent(""" + commands: + foo: + description: foo + namespace: type + fields: + foo: bar + """), idl.errors.ERROR_ID_MISSING_REQUIRED_FIELD) + if __name__ == '__main__': diff --git a/src/mongo/idl/idl_test.cpp b/src/mongo/idl/idl_test.cpp index 4c871112b1b..ab2e9328c3a 100644 --- a/src/mongo/idl/idl_test.cpp +++ b/src/mongo/idl/idl_test.cpp @@ -2329,5 +2329,220 @@ TEST(IDLChainedStruct, TestInline) { } } +// Positive: verify a command a string arg +TEST(IDLTypeCommand, TestString) { + IDLParserErrorContext ctxt("root"); + + auto testDoc = BSON(CommandTypeStringCommand::kCommandName << "foo" + << "field1" + << 3 + << "$db" + << "db"); + + auto testStruct = CommandTypeStringCommand::parse(ctxt, makeOMR(testDoc)); + ASSERT_EQUALS(testStruct.getField1(), 3); + ASSERT_EQUALS(testStruct.getCommandParameter(), "foo"); + + assert_same_types<decltype(testStruct.getCommandParameter()), const StringData>(); + + // Positive: Test we can roundtrip from the just parsed document + { + BSONObjBuilder builder; + OpMsgRequest reply = testStruct.serialize(BSONObj()); + + ASSERT_BSONOBJ_EQ(testDoc, reply.body); + } + + // Positive: Test we can serialize from nothing the same document except for $db + { + auto testDocWithoutDb = BSON(CommandTypeStringCommand::kCommandName << "foo" + << "field1" + << 3); + + BSONObjBuilder builder; + CommandTypeStringCommand one_new("foo"); + one_new.setField1(3); + one_new.setDbName("db"); + one_new.serialize(BSONObj(), &builder); + + auto serializedDoc = builder.obj(); + ASSERT_BSONOBJ_EQ(testDocWithoutDb, serializedDoc); + } + + // Positive: Test we can serialize from nothing the same document + { + BSONObjBuilder builder; + CommandTypeStringCommand one_new("foo"); + one_new.setField1(3); + one_new.setDbName("db"); + OpMsgRequest reply = one_new.serialize(BSONObj()); + + ASSERT_BSONOBJ_EQ(testDoc, reply.body); + } +} + +// Positive: verify a command can take an array of object +TEST(IDLTypeCommand, TestArrayObject) { + IDLParserErrorContext ctxt("root"); + + auto testDoc = BSON(CommandTypeArrayObjectCommand::kCommandName << BSON_ARRAY(BSON("sample" + << "doc")) + << "$db" + << "db"); + + auto testStruct = CommandTypeArrayObjectCommand::parse(ctxt, makeOMR(testDoc)); + ASSERT_EQUALS(testStruct.getCommandParameter().size(), 1UL); + + assert_same_types<decltype(testStruct.getCommandParameter()), + const std::vector<mongo::BSONObj>&>(); + + // Positive: Test we can roundtrip from the just parsed document + { + BSONObjBuilder builder; + OpMsgRequest reply = testStruct.serialize(BSONObj()); + + ASSERT_BSONOBJ_EQ(testDoc, reply.body); + } + + // Positive: Test we can serialize from nothing the same document + { + BSONObjBuilder builder; + std::vector<BSONObj> vec; + vec.emplace_back(BSON("sample" + << "doc")); + CommandTypeArrayObjectCommand one_new(vec); + one_new.setDbName("db"); + OpMsgRequest reply = one_new.serialize(BSONObj()); + + ASSERT_BSONOBJ_EQ(testDoc, reply.body); + } +} + +// Positive: verify a command can take a struct +TEST(IDLTypeCommand, TestStruct) { + IDLParserErrorContext ctxt("root"); + + auto testDoc = BSON(CommandTypeStructCommand::kCommandName << BSON("value" + << "sample") + << "$db" + << "db"); + + auto testStruct = CommandTypeStructCommand::parse(ctxt, makeOMR(testDoc)); + ASSERT_EQUALS(testStruct.getCommandParameter().getValue(), "sample"); + + assert_same_types<decltype(testStruct.getCommandParameter()), + mongo::idl::import::One_string&>(); + + // Positive: Test we can roundtrip from the just parsed document + { + BSONObjBuilder builder; + OpMsgRequest reply = testStruct.serialize(BSONObj()); + + ASSERT_BSONOBJ_EQ(testDoc, reply.body); + } + + // Positive: Test we can serialize from nothing the same document + { + BSONObjBuilder builder; + One_string os; + os.setValue("sample"); + CommandTypeStructCommand one_new(os); + one_new.setDbName("db"); + OpMsgRequest reply = one_new.serialize(BSONObj()); + + ASSERT_BSONOBJ_EQ(testDoc, reply.body); + } +} + +// Positive: verify a command can take an array of structs +TEST(IDLTypeCommand, TestStructArray) { + IDLParserErrorContext ctxt("root"); + + auto testDoc = BSON(CommandTypeArrayStructCommand::kCommandName << BSON_ARRAY(BSON("value" + << "sample")) + << "$db" + << "db"); + + auto testStruct = CommandTypeArrayStructCommand::parse(ctxt, makeOMR(testDoc)); + ASSERT_EQUALS(testStruct.getCommandParameter().size(), 1UL); + + assert_same_types<decltype(testStruct.getCommandParameter()), + const std::vector<mongo::idl::import::One_string>&>(); + + // Positive: Test we can roundtrip from the just parsed document + { + BSONObjBuilder builder; + OpMsgRequest reply = testStruct.serialize(BSONObj()); + + ASSERT_BSONOBJ_EQ(testDoc, reply.body); + } + + // Positive: Test we can serialize from nothing the same document + { + BSONObjBuilder builder; + std::vector<One_string> vec; + One_string os; + os.setValue("sample"); + vec.push_back(os); + CommandTypeArrayStructCommand one_new(vec); + one_new.setDbName("db"); + OpMsgRequest reply = one_new.serialize(BSONObj()); + + ASSERT_BSONOBJ_EQ(testDoc, reply.body); + } +} + +// Positive: verify a command a string arg and alternate C++ name +TEST(IDLTypeCommand, TestUnderscoreCommand) { + IDLParserErrorContext ctxt("root"); + + auto testDoc = BSON(WellNamedCommand::kCommandName << "foo" + << "field1" + << 3 + << "$db" + << "db"); + + auto testStruct = WellNamedCommand::parse(ctxt, makeOMR(testDoc)); + ASSERT_EQUALS(testStruct.getField1(), 3); + ASSERT_EQUALS(testStruct.getCommandParameter(), "foo"); + + assert_same_types<decltype(testStruct.getCommandParameter()), const StringData>(); + + // Positive: Test we can roundtrip from the just parsed document + { + BSONObjBuilder builder; + OpMsgRequest reply = testStruct.serialize(BSONObj()); + + ASSERT_BSONOBJ_EQ(testDoc, reply.body); + } + + // Positive: Test we can serialize from nothing the same document except for $db + { + auto testDocWithoutDb = BSON(WellNamedCommand::kCommandName << "foo" + << "field1" + << 3); + + BSONObjBuilder builder; + WellNamedCommand one_new("foo"); + one_new.setField1(3); + one_new.setDbName("db"); + one_new.serialize(BSONObj(), &builder); + + auto serializedDoc = builder.obj(); + ASSERT_BSONOBJ_EQ(testDocWithoutDb, serializedDoc); + } + + // Positive: Test we can serialize from nothing the same document + { + BSONObjBuilder builder; + WellNamedCommand one_new("foo"); + one_new.setField1(3); + one_new.setDbName("db"); + OpMsgRequest reply = one_new.serialize(BSONObj()); + + ASSERT_BSONOBJ_EQ(testDoc, reply.body); + } +} + } // namespace } // namespace mongo diff --git a/src/mongo/idl/unittest.idl b/src/mongo/idl/unittest.idl index 8c621daa1d0..01475aeccdc 100644 --- a/src/mongo/idl/unittest.idl +++ b/src/mongo/idl/unittest.idl @@ -586,3 +586,33 @@ commands: fields: field3: type: int + + CommandTypeStringCommand: + description: Command with custom type string + namespace: type + type: string + fields: + field1: int + + CommandTypeArrayObjectCommand: + description: Command with just an array of object parameter + namespace: type + type: array<object> + + CommandTypeStructCommand: + description: Command with just a struct parameter + namespace: type + type: one_string + + CommandTypeArrayStructCommand: + description: Command with just an array of struct parameter + namespace: type + type: array<one_string> + + _underscore_command: + description: Command with custom type string + namespace: type + type: string + cpp_name: WellNamedCommand + fields: + field1: int
\ No newline at end of file |