diff options
| -rw-r--r-- | buildscripts/idl/idl/binder.py | 3 | ||||
| -rw-r--r-- | buildscripts/idl/idl/compiler.py | 130 | ||||
| -rw-r--r-- | buildscripts/idl/idl/errors.py | 12 | ||||
| -rw-r--r-- | buildscripts/idl/idl/generator.py | 2 | ||||
| -rw-r--r-- | buildscripts/idl/idl/parser.py | 128 | ||||
| -rw-r--r-- | buildscripts/idl/idl/syntax.py | 36 | ||||
| -rw-r--r-- | buildscripts/idl/idlc.py | 10 | ||||
| -rw-r--r-- | buildscripts/idl/tests/test_import.py | 419 | ||||
| -rw-r--r-- | buildscripts/idl/tests/testcase.py | 51 | ||||
| -rw-r--r-- | site_scons/site_tools/idl_tool.py | 20 | ||||
| -rw-r--r-- | src/mongo/idl/SConscript | 3 | ||||
| -rw-r--r-- | src/mongo/idl/basic_types.idl | 79 | ||||
| -rw-r--r-- | src/mongo/idl/unittest.idl | 114 | ||||
| -rw-r--r-- | src/mongo/idl/unittest_import.idl | 74 |
14 files changed, 934 insertions, 147 deletions
diff --git a/buildscripts/idl/idl/binder.py b/buildscripts/idl/idl/binder.py index b46eec84c33..b4e7dc8bcad 100644 --- a/buildscripts/idl/idl/binder.py +++ b/buildscripts/idl/idl/binder.py @@ -311,7 +311,8 @@ def bind(parsed_spec): _validate_types(ctxt, parsed_spec) for struct in parsed_spec.symbols.structs: - bound_spec.structs.append(_bind_struct(ctxt, parsed_spec, struct)) + if not struct.imported: + bound_spec.structs.append(_bind_struct(ctxt, parsed_spec, struct)) if ctxt.errors.has_errors(): return ast.IDLBoundSpec(None, ctxt.errors) diff --git a/buildscripts/idl/idl/compiler.py b/buildscripts/idl/idl/compiler.py index 5dea3166f74..92d2abebc43 100644 --- a/buildscripts/idl/idl/compiler.py +++ b/buildscripts/idl/idl/compiler.py @@ -29,6 +29,7 @@ from . import binder from . import errors from . import generator from . import parser +from . import syntax class CompilerArgs(object): @@ -45,6 +46,113 @@ class CompilerArgs(object): self.output_base_dir = None # type: unicode self.output_suffix = None # type: unicode + self.write_dependencies = False # type: bool + + +class CompilerImportResolver(parser.ImportResolverBase): + """Class for the IDL compiler to resolve imported files.""" + + def __init__(self, import_directories): + # type: (List[unicode]) -> None + """Construct a ImportResolver.""" + self._import_directories = import_directories + + super(CompilerImportResolver, self).__init__() + + def resolve(self, base_file, imported_file_name): + # type: (unicode, unicode) -> unicode + """Return the complete path to an imported file name.""" + + logging.debug("Resolving imported file '%s' for file '%s'", imported_file_name, base_file) + + # Check for fully-qualified paths + logging.debug("Checking for imported file '%s' for file '%s' at '%s'", imported_file_name, + base_file, imported_file_name) + if os.path.isabs(imported_file_name) and os.path.exists(imported_file_name): + logging.debug("Found imported file '%s' for file '%s' at '%s'", imported_file_name, + base_file, imported_file_name) + return imported_file_name + + for candidate_dir in self._import_directories or []: + base_dir = os.path.abspath(candidate_dir) + resolved_file_name = os.path.normpath(os.path.join(base_dir, imported_file_name)) + + logging.debug("Checking for imported file '%s' for file '%s' at '%s'", + imported_file_name, base_file, resolved_file_name) + + if os.path.exists(resolved_file_name): + logging.debug("Found imported file '%s' for file '%s' at '%s'", imported_file_name, + base_file, resolved_file_name) + return resolved_file_name + + msg = ("Cannot find imported file '%s' for file '%s'" % (imported_file_name, base_file)) + logging.error(msg) + + raise errors.IDLError(msg) + + def open(self, resolved_file_name): + # type: (unicode) -> Any + """Return an io.Stream for the requested file.""" + return io.open(resolved_file_name) + + +def _write_dependencies(spec): + # type: (syntax.IDLSpec) -> None + """Write a list of dependencies to standard out.""" + if not spec.imports: + return + + dependencies = sorted(spec.imports.dependencies) + for resolved_file_name in dependencies: + print(resolved_file_name) + + +def _update_import_includes(args, spec, header_file_name): + # type: (CompilerArgs, syntax.IDLSpec, unicode) -> None + """Update the list of imports with a list of include files for each import with structs.""" + # This function is fragile: + # In order to try to generate headers with an "include what you use" set of headers, the IDL + # compiler needs to generate include statements to headers for imported files with structs. The + # problem is that the IDL compiler needs to make the following assumptions: + # 1. The layout of build vs source directory. + # 2. The file naming suffix rules for all IDL invocations are consistent. + if not spec.imports: + return + + if args.output_base_dir: + base_include_h_file_name = os.path.relpath( + os.path.normpath(header_file_name), os.path.normpath(args.output_base_dir)) + else: + base_include_h_file_name = os.path.abspath(header_file_name) + + # Normalize to POSIX style for consistency across Windows and POSIX. + base_include_h_file_name = base_include_h_file_name.replace("\\", "/") + + # Modify the includes list of the root_doc to include all of its direct imports + if not spec.globals: + spec.globals = syntax.Global(args.input_file, -1, -1) + + first_dir = base_include_h_file_name.split('/')[0] + + for resolved_file_name in spec.imports.resolved_imports: + # Guess: the file naming rules are consistent across IDL invocations + include_h_file_name = resolved_file_name.split('.')[0] + args.output_suffix + ".h" + + if args.output_base_dir: + include_h_file_name = os.path.relpath( + os.path.normpath(include_h_file_name), os.path.normpath(args.output_base_dir)) + else: + include_h_file_name = os.path.abspath(include_h_file_name) + + # Normalize to POSIX style for consistency across Windows and POSIX. + include_h_file_name = include_h_file_name.replace("\\", "/") + + if args.output_base_dir: + # Guess: The layout of build vs source directory + include_h_file_name = include_h_file_name[include_h_file_name.find(first_dir):] + + spec.globals.cpp_includes.append(include_h_file_name) + def compile_idl(args): # type: (CompilerArgs) -> bool @@ -53,19 +161,13 @@ def compile_idl(args): if not os.path.exists(args.input_file): logging.error("File '%s' not found", args.input_file) - # TODO: resolve the paths, and log if they do not exist under verbose when import supported is added - #for import_dir in args.import_directories: - # if not os.path.exists(args.input_file): - - error_file_name = os.path.basename(args.input_file) - if args.output_source is None: - if not '.' in error_file_name: + if not '.' in args.input_file: logging.error("File name '%s' must be end with a filename extension, such as '%s.idl'", - error_file_name, error_file_name) + args.input_file, args.input_file) return False - file_name_prefix = error_file_name.split('.')[0] + file_name_prefix = args.input_file.split('.')[0] file_name_prefix += args.output_suffix source_file_name = file_name_prefix + ".cpp" @@ -76,9 +178,17 @@ def compile_idl(args): # Compile the IDL through the 3 passes with io.open(args.input_file) as file_stream: - parsed_doc = parser.parse(file_stream, error_file_name=error_file_name) + parsed_doc = parser.parse(file_stream, args.input_file, + CompilerImportResolver(args.import_directories)) + + # Stop compiling if we only need to scan import dependencies + if args.write_dependencies: + _write_dependencies(parsed_doc.spec) + return True if not parsed_doc.errors: + _update_import_includes(args, parsed_doc.spec, header_file_name) + bound_doc = binder.bind(parsed_doc.spec) if not bound_doc.errors: generator.generate_code(bound_doc.spec, args.output_base_dir, header_file_name, diff --git a/buildscripts/idl/idl/errors.py b/buildscripts/idl/idl/errors.py index 47986c47435..efedd6cd354 100644 --- a/buildscripts/idl/idl/errors.py +++ b/buildscripts/idl/idl/errors.py @@ -22,6 +22,7 @@ Common error handling code for IDL compiler. from __future__ import absolute_import, print_function, unicode_literals import inspect +import os import sys from typing import List, Union, Any from yaml import nodes @@ -57,6 +58,7 @@ ERROR_ID_BAD_ANY_TYPE_USE = "ID0021" ERROR_ID_BAD_NUMERIC_CPP_TYPE = "ID0022" ERROR_ID_BAD_ARRAY_TYPE_NAME = "ID0023" ERROR_ID_ARRAY_NO_DEFAULT = "ID0024" +ERROR_ID_BAD_IMPORT = "ID0025" class IDLError(Exception): @@ -93,8 +95,8 @@ class ParserError(common.SourceLocation): Example error message: test.idl: (17, 4): ID0008: Unknown IDL node 'cpp_namespac' for YAML entity 'global'. """ - msg = "%s: (%d, %d): %s: %s" % (self.file_name, self.line, self.column, self.error_id, - self.msg) + msg = "%s: (%d, %d): %s: %s" % (os.path.basename(self.file_name), self.line, self.column, + self.error_id, self.msg) return msg # type: ignore @@ -405,6 +407,12 @@ class ParserContext(object): "Field '%s' is not allowed to have both a default value and be an array type" % (field_name)) + def add_cannot_find_import(self, location, imported_file_name): + # type: (common.SourceLocation, unicode) -> None + """Add an error about not being able to find an import.""" + self._add_error(location, ERROR_ID_BAD_IMPORT, + "Could not resolve import '%s', file not found" % (imported_file_name)) + def _assert_unique_error_messages(): # type: () -> None diff --git a/buildscripts/idl/idl/generator.py b/buildscripts/idl/idl/generator.py index 90f18f02129..c2efad8feef 100644 --- a/buildscripts/idl/idl/generator.py +++ b/buildscripts/idl/idl/generator.py @@ -1070,7 +1070,7 @@ def generate_code(spec, output_base_dir, header_file_name, source_file_name): include_h_file_name = os.path.relpath( os.path.normpath(header_file_name), os.path.normpath(output_base_dir)) else: - include_h_file_name = header_file_name + include_h_file_name = os.path.abspath(os.path.normpath(header_file_name)) # Normalize to POSIX style for consistency across Windows and POSIX. include_h_file_name = include_h_file_name.replace("\\", "/") diff --git a/buildscripts/idl/idl/parser.py b/buildscripts/idl/idl/parser.py index ee37a0f600b..d9382065dfe 100644 --- a/buildscripts/idl/idl/parser.py +++ b/buildscripts/idl/idl/parser.py @@ -20,10 +20,13 @@ Only validates the document is syntatically correct, not semantically. """ from __future__ import absolute_import, print_function, unicode_literals -from typing import Any, Callable, Dict, List, Set, Union -from yaml import nodes +from abc import ABCMeta, abstractmethod +import io import yaml +from yaml import nodes +from typing import Any, Callable, Dict, List, Set, Tuple, Union +from . import common from . import errors from . import syntax @@ -132,6 +135,21 @@ def _parse_global(ctxt, spec, node): spec.globals = idlglobal +def _parse_imports(ctxt, spec, node): + # type: (errors.ParserContext, syntax.IDLSpec, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None + """Parse an imports section in the IDL file.""" + if not ctxt.is_sequence_node(node, "imports"): + return + + if spec.imports: + ctxt.add_duplicate_error(node, "imports") + return + + imports = syntax.Import(ctxt.file_name, node.start_mark.line, node.start_mark.column) + imports.imports = ctxt.get_list(node) + spec.imports = imports + + def _parse_type(ctxt, spec, name, node): # type: (errors.ParserContext, syntax.IDLSpec, unicode, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None """Parse a type section in the IDL file.""" @@ -255,7 +273,7 @@ def _parse_structs(ctxt, spec, node): _parse_struct(ctxt, spec, first_name, second_node) -def parse(stream, error_file_name="unknown"): +def _parse(stream, error_file_name): # type: (Any, unicode) -> syntax.IDLParsedSpec """ Parse a YAML document into an idl.syntax tree. @@ -294,6 +312,8 @@ def parse(stream, error_file_name="unknown"): if first_name == "global": _parse_global(ctxt, spec, second_node) + elif first_name == "imports": + _parse_imports(ctxt, spec, second_node) elif first_name == "types": _parse_types(ctxt, spec, second_node) elif first_name == "structs": @@ -307,3 +327,105 @@ def parse(stream, error_file_name="unknown"): return syntax.IDLParsedSpec(None, ctxt.errors) else: return syntax.IDLParsedSpec(spec, None) + + +class ImportResolverBase(object): + """Base class for resolving imported files.""" + + __metaclass__ = ABCMeta + + def __init__(self): + # type: () -> None + """Construct a ImportResolver.""" + pass + + @abstractmethod + def resolve(self, base_file, imported_file_name): + # type: (unicode, unicode) -> unicode + """Return the complete path to an imported file name.""" + pass + + @abstractmethod + def open(self, resolved_file_name): + # type: (unicode) -> Any + """Return an io.Stream for the requested file.""" + pass + + +def parse(stream, input_file_name, resolver): + # type: (Any, unicode, ImportResolverBase) -> syntax.IDLParsedSpec + """ + Parse a YAML document into an idl.syntax tree. + + stream: is a io.Stream. + input_file_name: a file name for error messages to use, and to help resolve imported files. + """ + # pylint: disable=too-many-locals + + root_doc = _parse(stream, input_file_name) + + if root_doc.errors: + return root_doc + + imports = [] # type: List[Tuple[common.SourceLocation, unicode, unicode]] + needs_include = [] # type: List[unicode] + if root_doc.spec.imports: + imports = [(root_doc.spec.imports, input_file_name, import_file_name) + for import_file_name in root_doc.spec.imports.imports] + + resolved_file_names = [] # type: List[unicode] + + ctxt = errors.ParserContext(input_file_name, errors.ParserErrorCollection()) + + # Process imports in a breadth-first search + while imports: + file_import_tuple = imports[0] + imports = imports[1:] + + import_location = file_import_tuple[0] + base_file_name = file_import_tuple[1] + imported_file_name = file_import_tuple[2] + + # Check for already resolved file + resolved_file_name = resolver.resolve(base_file_name, imported_file_name) + if not resolved_file_name: + ctxt.add_cannot_find_import(import_location, imported_file_name) + return syntax.IDLParsedSpec(None, ctxt.errors) + + if resolved_file_name in resolved_file_names: + continue + + resolved_file_names.append(resolved_file_name) + + # Parse imported file + with resolver.open(resolved_file_name) as file_stream: + parsed_doc = _parse(file_stream, resolved_file_name) + + # Check for errors + if parsed_doc.errors: + return parsed_doc + + # We need to generate includes for imported IDL files which have structs + if base_file_name == input_file_name and len(parsed_doc.spec.symbols.structs): + needs_include.append(imported_file_name) + + # Add other imported files to the list of files to parse + if parsed_doc.spec.imports: + imports += [(parsed_doc.spec.imports, resolved_file_name, import_file_name) + for import_file_name in parsed_doc.spec.imports.imports] + + # Merge symbol tables together + root_doc.spec.symbols.add_imported_symbol_table(ctxt, parsed_doc.spec.symbols) + if ctxt.errors.has_errors(): + return syntax.IDLParsedSpec(None, ctxt.errors) + + # Resolve the direct imports which contain structs for root document so they can be translated + # into include file paths in generated code. + for needs_include_name in needs_include: + resolved_file_name = resolver.resolve(base_file_name, needs_include_name) + root_doc.spec.imports.resolved_imports.append(resolved_file_name) + + if root_doc.spec.imports: + root_doc.spec.imports.dependencies = resolved_file_names + + return root_doc diff --git a/buildscripts/idl/idl/syntax.py b/buildscripts/idl/idl/syntax.py index 9ec520c0235..27ca889fc73 100644 --- a/buildscripts/idl/idl/syntax.py +++ b/buildscripts/idl/idl/syntax.py @@ -52,7 +52,7 @@ class IDLSpec(object): """Construct an IDL spec.""" self.symbols = SymbolTable() # type: SymbolTable self.globals = None # type: Optional[Global] - #TODO self.imports = None # type: Optional[Imports] + self.imports = None # type: Optional[Import] def parse_array_type(name): @@ -119,6 +119,21 @@ class SymbolTable(object): # TODO: add commands pass + def add_imported_symbol_table(self, ctxt, imported_symbols): + # type: (errors.ParserContext, SymbolTable) -> None + """ + Merge all the symbols in the imported_symbols symbol table into the symbol table. + + Marks imported structs as imported, and errors on duplicate symbols. + """ + for struct in imported_symbols.structs: + if not self._is_duplicate(ctxt, struct, struct.name, "struct"): + struct.imported = True + self.structs.append(struct) + + for idltype in imported_symbols.types: + self.add_type(ctxt, idltype) + def resolve_field_type(self, ctxt, field): # type: (errors.ParserContext, Field) -> Tuple[Optional[Struct], Optional[Type]] """Find the type or struct a field refers to or log an error.""" @@ -164,11 +179,21 @@ class Global(common.SourceLocation): super(Global, self).__init__(file_name, line, column) -# TODO: add support for imports class Import(common.SourceLocation): """IDL imports object.""" - pass + def __init__(self, file_name, line, column): + # type: (unicode, int, int) -> None + """Construct an Imports section.""" + self.imports = [] # type: List[unicode] + + # These are not part of the IDL syntax but are produced by the parser. + # List of imports with structs. + self.resolved_imports = [] # type: List[unicode] + # All imports directly or indirectly included + self.dependencies = [] # type: List[unicode] + + super(Import, self).__init__(file_name, line, column) class Type(common.SourceLocation): @@ -235,6 +260,11 @@ class Struct(common.SourceLocation): self.description = None # type: unicode self.strict = True # type: bool self.fields = None # type: List[Field] + + # Internal property that is not represented as syntax. An imported struct is read from an + # imported file, and no code is generated for it. + self.imported = False # type: bool + super(Struct, self).__init__(file_name, line, column) diff --git a/buildscripts/idl/idlc.py b/buildscripts/idl/idlc.py index 04225b13d7b..a9abc0f19cf 100644 --- a/buildscripts/idl/idlc.py +++ b/buildscripts/idl/idlc.py @@ -18,6 +18,7 @@ from __future__ import absolute_import, print_function import argparse +import logging import sys import idl.compiler @@ -45,8 +46,16 @@ def main(): parser.add_argument('--base_dir', type=str, help="IDL output relative base directory") + parser.add_argument( + '--write-dependencies', + action='store_true', + help='only print out a list of dependent imports') + args = parser.parse_args() + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + compiler_args = idl.compiler.CompilerArgs() compiler_args.input_file = args.file @@ -56,6 +65,7 @@ def main(): compiler_args.output_header = args.header compiler_args.output_base_dir = args.base_dir compiler_args.output_suffix = "_gen" + compiler_args.write_dependencies = args.write_dependencies if (args.output is not None and args.header is None) or \ (args.output is None and args.header is not None): diff --git a/buildscripts/idl/tests/test_import.py b/buildscripts/idl/tests/test_import.py new file mode 100644 index 00000000000..9306c441dfe --- /dev/null +++ b/buildscripts/idl/tests/test_import.py @@ -0,0 +1,419 @@ +#!/usr/bin/env python2 +# Copyright (C) 2017 MongoDB Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3, +# as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +"""Test cases for IDL binder.""" + +from __future__ import absolute_import, print_function, unicode_literals + +import io +import textwrap +import unittest +from typing import Any, Dict + +# import package so that it works regardless of whether we run as a module or file +if __package__ is None: + import sys + from os import path + sys.path.append(path.dirname(path.abspath(__file__))) + from context import idl + import testcase +else: + from .context import idl + from . import testcase + + +class DictionaryImportResolver(idl.parser.ImportResolverBase): + """An import resolver resolves files from a dictionary.""" + + def __init__(self, import_dict): + # type: (Dict[unicode, unicode]) -> None + """Construct a DictionaryImportResolver.""" + self._import_dict = import_dict + super(DictionaryImportResolver, self).__init__() + + def resolve(self, base_file, imported_file_name): + # type: (unicode, unicode) -> unicode + """Return the complete path to an imported file name.""" + # pylint: disable=unused-argument + if not imported_file_name in self._import_dict: + return None + + return "imported_%s" % (imported_file_name) + + def open(self, resolved_file_name): + # type: (unicode) -> Any + """Return an io.Stream for the requested file.""" + assert resolved_file_name.startswith("imported_") + imported_file_name = resolved_file_name.replace("imported_", "") + + return io.StringIO(self._import_dict[imported_file_name]) + + +class TestImport(testcase.IDLTestcase): + """Test cases for the IDL binder.""" + + # Test: import wrong types + def test_import_negative_parser(self): + # type: () -> None + """Negative import parser tests.""" + + self.assert_parse_fail( + textwrap.dedent(""" + imports: + - "a.idl" + + imports: + - "b.idl" + """), idl.errors.ERROR_ID_DUPLICATE_NODE) + + self.assert_parse_fail( + textwrap.dedent(""" + imports: "basetypes.idl" + """), idl.errors.ERROR_ID_IS_NODE_TYPE) + + self.assert_parse_fail( + textwrap.dedent(""" + imports: + a: "a.idl" + b: "b.idl" + """), idl.errors.ERROR_ID_IS_NODE_TYPE) + + def test_import_positive(self): + # type: () -> None + """Postive import tests.""" + + import_dict = { + "basetypes.idl": + textwrap.dedent(""" + global: + cpp_namespace: 'something' + + types: + string: + description: foo + cpp_type: foo + bson_serialization_type: string + serializer: foo + deserializer: foo + default: foo + + structs: + bar: + description: foo + strict: false + fields: + foo: string + """), + "recurse1.idl": + textwrap.dedent(""" + imports: + - "basetypes.idl" + + types: + int: + description: foo + cpp_type: foo + bson_serialization_type: int + """), + "recurse2.idl": + textwrap.dedent(""" + imports: + - "recurse1.idl" + + types: + double: + description: foo + cpp_type: foo + bson_serialization_type: double + """), + "recurse1b.idl": + textwrap.dedent(""" + imports: + - "basetypes.idl" + + types: + bool: + description: foo + cpp_type: foo + bson_serialization_type: bool + """), + "cycle1a.idl": + textwrap.dedent(""" + global: + cpp_namespace: 'something' + + imports: + - "cycle1b.idl" + + types: + string: + description: foo + cpp_type: foo + bson_serialization_type: string + serializer: foo + deserializer: foo + default: foo + + structs: + bar: + description: foo + strict: false + fields: + foo: string + foo1: bool + """), + "cycle1b.idl": + textwrap.dedent(""" + global: + cpp_namespace: 'something' + + imports: + - "cycle1a.idl" + + types: + bool: + description: foo + cpp_type: foo + bson_serialization_type: bool + + structs: + bar2: + description: foo + strict: false + fields: + foo: string + foo1: bool + """), + "cycle2.idl": + textwrap.dedent(""" + global: + cpp_namespace: 'something' + + imports: + - "cycle2.idl" + + types: + string: + description: foo + cpp_type: foo + bson_serialization_type: string + serializer: foo + deserializer: foo + default: foo + """), + } + + resolver = DictionaryImportResolver(import_dict) + + # Test simple import + self.assert_bind( + textwrap.dedent(""" + global: + cpp_namespace: 'something' + + imports: + - "basetypes.idl" + + structs: + foobar: + description: foo + strict: false + fields: + foo: string + """), + resolver=resolver) + + # Test nested import + self.assert_bind( + textwrap.dedent(""" + global: + cpp_namespace: 'something' + + imports: + - "recurse2.idl" + + structs: + foobar: + description: foo + strict: false + fields: + foo: string + foo1: int + foo2: double + """), + resolver=resolver) + + # Test diamond import + self.assert_bind( + textwrap.dedent(""" + global: + cpp_namespace: 'something' + + imports: + - "recurse2.idl" + - "recurse1b.idl" + + structs: + foobar: + description: foo + strict: false + fields: + foo: string + foo1: int + foo2: double + foo3: bool + """), + resolver=resolver) + + # Test cycle import + self.assert_bind( + textwrap.dedent(""" + global: + cpp_namespace: 'something' + + imports: + - "cycle1a.idl" + + structs: + foobar: + description: foo + strict: false + fields: + foo: string + foo1: bool + """), + resolver=resolver) + + # Test self cycle import + self.assert_bind( + textwrap.dedent(""" + global: + cpp_namespace: 'something' + + imports: + - "cycle2.idl" + + structs: + foobar: + description: foo + strict: false + fields: + foo: string + """), + resolver=resolver) + + def test_import_negative(self): + # type: () -> None + """Negative import tests.""" + + import_dict = { + "basetypes.idl": + textwrap.dedent(""" + global: + cpp_namespace: 'something' + + types: + string: + description: foo + cpp_type: foo + bson_serialization_type: string + serializer: foo + deserializer: foo + default: foo + + structs: + bar: + description: foo + strict: false + fields: + foo: string + """) + } + + resolver = DictionaryImportResolver(import_dict) + + # Bad import + self.assert_parse_fail( + textwrap.dedent(""" + imports: + - "notfound.idl" + """), + idl.errors.ERROR_ID_BAD_IMPORT, + resolver=resolver) + + # Duplicate types + self.assert_parse_fail( + textwrap.dedent(""" + imports: + - "basetypes.idl" + + types: + string: + description: foo + cpp_type: foo + bson_serialization_type: string + """), + idl.errors.ERROR_ID_DUPLICATE_SYMBOL, + resolver=resolver) + + # Duplicate structs + self.assert_parse_fail( + textwrap.dedent(""" + imports: + - "basetypes.idl" + + structs: + bar: + description: foo + fields: + foo1: string + """), + idl.errors.ERROR_ID_DUPLICATE_SYMBOL, + resolver=resolver) + + # Duplicate struct and type + self.assert_parse_fail( + textwrap.dedent(""" + imports: + - "basetypes.idl" + + structs: + string: + description: foo + fields: + foo1: string + """), + idl.errors.ERROR_ID_DUPLICATE_SYMBOL, + resolver=resolver) + + # Duplicate type and struct + self.assert_parse_fail( + textwrap.dedent(""" + imports: + - "basetypes.idl" + + types: + bar: + description: foo + cpp_type: foo + bson_serialization_type: string + """), + idl.errors.ERROR_ID_DUPLICATE_SYMBOL, + resolver=resolver) + + +if __name__ == '__main__': + + unittest.main() diff --git a/buildscripts/idl/tests/testcase.py b/buildscripts/idl/tests/testcase.py index d3def880db0..f279279c8da 100644 --- a/buildscripts/idl/tests/testcase.py +++ b/buildscripts/idl/tests/testcase.py @@ -17,6 +17,7 @@ from __future__ import absolute_import, print_function, unicode_literals import unittest +from typing import Any if __name__ == 'testcase': import sys @@ -35,15 +36,34 @@ def errors_to_str(errors): return "<empty>" +class NothingImportResolver(idl.parser.ImportResolverBase): + """An import resolver that does nothing.""" + + def __init__(self): + # type: () -> None + """Construct a NothingImportResolver.""" + super(NothingImportResolver, self).__init__() + + def resolve(self, base_file, imported_file_name): + # type: (unicode, unicode) -> unicode + """Return the complete path to an imported file name.""" + raise NotImplementedError() + + def open(self, imported_file_name): + # type: (unicode) -> Any + """Return an io.Stream for the requested file.""" + raise NotImplementedError() + + class IDLTestcase(unittest.TestCase): """IDL Test case base class.""" - def _parse(self, doc_str): - # type: (unicode) -> idl.syntax.IDLParsedSpec + def _parse(self, doc_str, resolver): + # type: (unicode, idl.parser.ImportResolverBase) -> idl.syntax.IDLParsedSpec """Parse a document and throw a unittest failure if it fails to parse as a valid YAML document.""" try: - return idl.parser.parse(doc_str) + return idl.parser.parse(doc_str, "unknown", resolver) except: # pylint: disable=bare-except self.fail("Failed to parse document:\n%s" % (doc_str)) @@ -55,20 +75,21 @@ class IDLTestcase(unittest.TestCase): (doc_str, errors_to_str(parsed_doc.errors))) self.assertIsNotNone(parsed_doc.spec, "Expected a parsed doc") - def assert_parse(self, doc_str): - # type: (unicode) -> None + def assert_parse(self, doc_str, resolver=NothingImportResolver()): + # type: (unicode, idl.parser.ImportResolverBase) -> None """Assert a document parsed correctly by the IDL compiler and returned no errors.""" - parsed_doc = self._parse(doc_str) + parsed_doc = self._parse(doc_str, resolver) self._assert_parse(doc_str, parsed_doc) - def assert_parse_fail(self, doc_str, error_id, multiple=False): - # type: (unicode, unicode, bool) -> None + def assert_parse_fail(self, doc_str, error_id, multiple=False, + resolver=NothingImportResolver()): + # type: (unicode, unicode, bool, idl.parser.ImportResolverBase) -> None """ Assert a document parsed correctly by the YAML parser, but not the by the IDL compiler. Asserts only one error is found in the document to make future IDL changes easier. """ - parsed_doc = self._parse(doc_str) + parsed_doc = self._parse(doc_str, resolver) self.assertIsNone(parsed_doc.spec, "Expected no parsed doc") self.assertIsNotNone(parsed_doc.errors, "Expected parser errors") @@ -85,10 +106,10 @@ class IDLTestcase(unittest.TestCase): "For document:\n%s\nExpected error message '%s' but received only errors:\n %s" % (doc_str, error_id, errors_to_str(parsed_doc.errors))) - def assert_bind(self, doc_str): - # type: (unicode) -> idl.ast.IDLBoundSpec + def assert_bind(self, doc_str, resolver=NothingImportResolver()): + # type: (unicode, idl.parser.ImportResolverBase) -> idl.ast.IDLBoundSpec """Assert a document parsed and bound correctly by the IDL compiler and returned no errors.""" - parsed_doc = self._parse(doc_str) + parsed_doc = self._parse(doc_str, resolver) self._assert_parse(doc_str, parsed_doc) bound_doc = idl.binder.bind(parsed_doc.spec) @@ -100,14 +121,14 @@ class IDLTestcase(unittest.TestCase): return bound_doc.spec - def assert_bind_fail(self, doc_str, error_id): - # type: (unicode, unicode) -> None + def assert_bind_fail(self, doc_str, error_id, resolver=NothingImportResolver()): + # type: (unicode, unicode, idl.parser.ImportResolverBase) -> None """ Assert a document parsed correctly by the YAML parser and IDL parser, but not bound by the IDL binder. Asserts only one error is found in the document to make future IDL changes easier. """ - parsed_doc = self._parse(doc_str) + parsed_doc = self._parse(doc_str, resolver) self._assert_parse(doc_str, parsed_doc) bound_doc = idl.binder.bind(parsed_doc.spec) diff --git a/site_scons/site_tools/idl_tool.py b/site_scons/site_tools/idl_tool.py index 22e75c4ac42..d44aa5a40d4 100644 --- a/site_scons/site_tools/idl_tool.py +++ b/site_scons/site_tools/idl_tool.py @@ -16,6 +16,7 @@ """IDL Compiler Scons Tool.""" import os.path +import subprocess import sys import SCons @@ -44,15 +45,32 @@ IDLCBuilder = SCons.Builder.Builder( suffix=".cpp" ) +def idl_scanner(node, env, path): + # Use the import scanner mode of the IDL compiler to file imported files + cmd = [sys.executable, "buildscripts/idl/idlc.py", '--include','src', str(node), '--write-dependencies'] + deps_str = subprocess.check_output(cmd) + + deps_list = deps_str.splitlines() + + nodes_deps_list = [ env.File(d) for d in deps_list] + + return nodes_deps_list + +idl_scanner = SCons.Scanner.Scanner(function=idl_scanner, skeys=['.idl']) + def generate(env): bld = IDLCBuilder + + env.Append(SCANNERS = idl_scanner) + env['BUILDERS']['Idlc'] = bld env['IDLC'] = sys.executable + " buildscripts/idl/idlc.py" env['IDLCFLAGS'] = '' base_dir = env.subst('$BUILD_ROOT/$VARIANT_DIR').replace("#", "") - env['IDLCCOM'] = '$IDLC --base_dir %s --header ${TARGETS[1]} --output ${TARGETS[0]} $SOURCES ' % (base_dir) + env['IDLCCOM'] = '$IDLC --include src --base_dir %s --header ${TARGETS[1]} --output ${TARGETS[0]} $SOURCES ' % (base_dir) env['IDLCSUFFIX'] = '.idl' + def exists(env): return True
\ No newline at end of file diff --git a/src/mongo/idl/SConscript b/src/mongo/idl/SConscript index 93a57d332e0..87c5d3c2158 100644 --- a/src/mongo/idl/SConscript +++ b/src/mongo/idl/SConscript @@ -17,7 +17,8 @@ env.CppUnitTest( target='idl_test', source=[ 'idl_test.cpp', - env.Idlc('unittest.idl')[0] + env.Idlc('unittest_import.idl')[0], + env.Idlc('unittest.idl')[0], ], LIBDEPS=[ '$BUILD_DIR/mongo/base', diff --git a/src/mongo/idl/basic_types.idl b/src/mongo/idl/basic_types.idl new file mode 100644 index 00000000000..198d8f20980 --- /dev/null +++ b/src/mongo/idl/basic_types.idl @@ -0,0 +1,79 @@ +# Copyright (C) 2017 MongoDB Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3, +# as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +# IDL Basic Types +global: + cpp_namespace: "mongo" + +types: + string: + bson_serialization_type: string + description: "A BSON UTF-8 string" + cpp_type: "std::string" + deserializer: "mongo::BSONElement::str" + + int: + bson_serialization_type: int + description: "A BSON 32-bit integer" + cpp_type: "std::int32_t" + deserializer: "mongo::BSONElement::_numberInt" + + long: + bson_serialization_type: long + description: "A BSON 64-bit integer" + cpp_type: "std::int64_t" + deserializer: "mongo::BSONElement::_numberLong" + + double: + bson_serialization_type: double + description: "A BSON 64-bit floating point number" + cpp_type: "double" + deserializer: "mongo::BSONElement::_numberDouble" + + decimal: + bson_serialization_type: double + description: "A BSON 128-bit floating point decimal number" + cpp_type: "mongo::Decimal128" + deserializer: "mongo::BSONElement::_numberDecimal" + + bool: + bson_serialization_type: bool + description: "A BSON bool" + cpp_type: "bool" + deserializer: "mongo::BSONElement::boolean" + + # TODO: support bindata + # bindata: + # bson_serialization_type: bindata + # cpp_type: "std:" + # deserializer: "mongo::BSONElement::str" + + objectid: + bson_serialization_type: objectid + description: "A BSON ObjectID" + cpp_type: "mongo::OID" + deserializer: "mongo::BSONElement::OID" + + date: + bson_serialization_type: date + description: "A BSON UTC DateTime" + cpp_type: "mongo::Date_t" + deserializer: "mongo::BSONElement::date" + + timestamp: + bson_serialization_type: timestamp + description: "A BSON TimeStamp" + cpp_type: "mongo::Timestamp" + deserializer: "mongo::BSONElement::timestamp"
\ No newline at end of file diff --git a/src/mongo/idl/unittest.idl b/src/mongo/idl/unittest.idl index d0705b7d260..252d1ead736 100644 --- a/src/mongo/idl/unittest.idl +++ b/src/mongo/idl/unittest.idl @@ -20,67 +20,11 @@ global: - "mongo/db/namespace_string.h" - "mongo/idl/idl_test_types.h" -types: - string: - bson_serialization_type: string - description: "A BSON UTF-8 string" - cpp_type: "std::string" - deserializer: "mongo::BSONElement::str" - - int: - bson_serialization_type: int - description: "A BSON 32-bit integer" - cpp_type: "std::int32_t" - deserializer: "mongo::BSONElement::_numberInt" - - long: - bson_serialization_type: long - description: "A BSON 64-bit integer" - cpp_type: "std::int64_t" - deserializer: "mongo::BSONElement::_numberLong" - - double: - bson_serialization_type: double - description: "A BSON 64-bit floating point number" - cpp_type: "double" - deserializer: "mongo::BSONElement::_numberDouble" - - decimal: - bson_serialization_type: double - description: "A BSON 128-bit floating point decimal number" - cpp_type: "mongo::Decimal128" - deserializer: "mongo::BSONElement::_numberDecimal" - - bool: - bson_serialization_type: bool - description: "A BSON bool" - cpp_type: "bool" - deserializer: "mongo::BSONElement::boolean" - - # TODO: support bindata - # bindata: - # bson_serialization_type: bindata - # cpp_type: "std:" - # deserializer: "mongo::BSONElement::str" - - objectid: - bson_serialization_type: objectid - description: "A BSON ObjectID" - cpp_type: "mongo::OID" - deserializer: "mongo::BSONElement::OID" - - date: - bson_serialization_type: date - description: "A BSON UTC DateTime" - cpp_type: "mongo::Date_t" - deserializer: "mongo::BSONElement::date" - - timestamp: - bson_serialization_type: timestamp - description: "A BSON TimeStamp" - cpp_type: "mongo::Timestamp" - deserializer: "mongo::BSONElement::timestamp" +imports: + - "mongo/idl/basic_types.idl" + - "mongo/idl/unittest_import.idl" +types: ################################################################################################## # # Test a custom non-BSONElement deserialization and serialization methods for a string type @@ -137,57 +81,7 @@ types: deserializer: "mongo::BSONElement::numberInt" -################################################################################################## -# -# Unit test structs for a single value to ensure type validation works correctly -# -################################################################################################## - structs: - one_string: - description: UnitTest for a single string - fields: - value: string - - one_int: - description: UnitTest for a single int - fields: - value: int - - one_long: - description: UnitTest for a single long - fields: - value: long - - one_double: - description: UnitTest for a single double - fields: - value: double - - one_decimal: - description: UnitTest for a single decimal - fields: - value: decimal - - one_bool: - description: UnitTest for a single bool - fields: - value: bool - - one_objectid: - description: UnitTest for a single objectid - fields: - value: objectid - - one_date: - description: UnitTest for a single date - fields: - value: date - - one_timestamp: - description: UnitTest for a single timestamp - fields: - value: timestamp ################################################################################################## # diff --git a/src/mongo/idl/unittest_import.idl b/src/mongo/idl/unittest_import.idl new file mode 100644 index 00000000000..a2064766133 --- /dev/null +++ b/src/mongo/idl/unittest_import.idl @@ -0,0 +1,74 @@ +# Copyright (C) 2017 MongoDB Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3, +# as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +# IDL Unit Tests IDL file for testing imported structs +global: + cpp_namespace: "mongo" + +imports: + - "mongo/idl/basic_types.idl" + + +################################################################################################## +# +# Unit test structs for a single value to ensure type validation works correctly +# +################################################################################################## + +structs: + one_string: + description: UnitTest for a single string + fields: + value: string + + one_int: + description: UnitTest for a single int + fields: + value: int + + one_long: + description: UnitTest for a single long + fields: + value: long + + one_double: + description: UnitTest for a single double + fields: + value: double + + one_decimal: + description: UnitTest for a single decimal + fields: + value: decimal + + one_bool: + description: UnitTest for a single bool + fields: + value: bool + + one_objectid: + description: UnitTest for a single objectid + fields: + value: objectid + + one_date: + description: UnitTest for a single date + fields: + value: date + + one_timestamp: + description: UnitTest for a single timestamp + fields: + value: timestamp
\ No newline at end of file |