diff options
author | scoder <stefan_ml@behnel.de> | 2022-07-30 18:43:10 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-07-30 18:43:10 +0200 |
commit | 84afe5550e619cc1b40e9909e4a35234841ad366 (patch) | |
tree | 5282a2fb1528b4d1b829df2b923f193818f69db4 | |
parent | 9b69292ee4f5340994776cac4ba18c27c9578ea1 (diff) | |
download | cython-84afe5550e619cc1b40e9909e4a35234841ad366.tar.gz |
Allow C code assertions in tests by defining regular expressions in module directives. (GH-4938)
-rw-r--r-- | Cython/Compiler/Options.py | 11 | ||||
-rw-r--r-- | Cython/Compiler/Parsing.py | 3 | ||||
-rw-r--r-- | Cython/Compiler/Pipeline.py | 15 | ||||
-rw-r--r-- | Cython/TestUtils.py | 70 | ||||
-rw-r--r-- | tests/compile/c_directives.pyx | 2 | ||||
-rw-r--r-- | tests/run/c_file_validation.srctree | 72 |
6 files changed, 165 insertions, 8 deletions
diff --git a/Cython/Compiler/Options.py b/Cython/Compiler/Options.py index f400e4ba2..bb547e978 100644 --- a/Cython/Compiler/Options.py +++ b/Cython/Compiler/Options.py @@ -171,7 +171,7 @@ def copy_inherited_directives(outer_directives, **new_directives): # For example, test_assert_path_exists and test_fail_if_path_exists should not be inherited # otherwise they can produce very misleading test failures new_directives_out = dict(outer_directives) - for name in ('test_assert_path_exists', 'test_fail_if_path_exists'): + for name in ('test_assert_path_exists', 'test_fail_if_path_exists', 'test_assert_c_code_has', 'test_fail_if_c_code_has'): new_directives_out.pop(name, None) new_directives_out.update(new_directives) return new_directives_out @@ -247,6 +247,8 @@ _directive_defaults = { # test support 'test_assert_path_exists' : [], 'test_fail_if_path_exists' : [], + 'test_assert_c_code_has' : [], + 'test_fail_if_c_code_has' : [], # experimental, subject to change 'formal_grammar': False, @@ -364,6 +366,8 @@ directive_scopes = { # defaults to available everywhere 'set_initial_path' : ('module',), 'test_assert_path_exists' : ('function', 'class', 'cclass'), 'test_fail_if_path_exists' : ('function', 'class', 'cclass'), + 'test_assert_c_code_has' : ('module',), + 'test_fail_if_c_code_has' : ('module',), 'freelist': ('cclass',), 'emit_code_comments': ('module',), # Avoid scope-specific to/from_py_functions for c_string. @@ -509,6 +513,11 @@ def parse_directive_list(s, relaxed_bool=False, ignore_unknown=False, result[directive] = parsed_value if not found and not ignore_unknown: raise ValueError('Unknown option: "%s"' % name) + elif directive_types.get(name) is list: + if name in result: + result[name].append(value) + else: + result[name] = [value] else: parsed_value = parse_directive_value(name, value, relaxed_bool=relaxed_bool) result[name] = parsed_value diff --git a/Cython/Compiler/Parsing.py b/Cython/Compiler/Parsing.py index d7c844849..8160149af 100644 --- a/Cython/Compiler/Parsing.py +++ b/Cython/Compiler/Parsing.py @@ -3853,6 +3853,9 @@ def p_compiler_directive_comments(s): for name in new_directives: if name not in result: pass + elif Options.directive_types.get(name) is list: + result[name] += new_directives[name] + new_directives[name] = result[name] elif new_directives[name] == result[name]: warning(pos, "Duplicate directive found: %s" % (name,)) else: diff --git a/Cython/Compiler/Pipeline.py b/Cython/Compiler/Pipeline.py index 3a5c42352..2fd3a1d3f 100644 --- a/Cython/Compiler/Pipeline.py +++ b/Cython/Compiler/Pipeline.py @@ -231,14 +231,15 @@ def create_pipeline(context, mode, exclude_classes=()): return stages def create_pyx_pipeline(context, options, result, py=False, exclude_classes=()): - if py: - mode = 'py' - else: - mode = 'pyx' + mode = 'py' if py else 'pyx' + test_support = [] + ctest_support = [] if options.evaluate_tree_assertions: from ..TestUtils import TreeAssertVisitor - test_support.append(TreeAssertVisitor()) + test_validator = TreeAssertVisitor() + test_support.append(test_validator) + ctest_support.append(test_validator.create_c_file_validator()) if options.gdb_debug: from ..Debugger import DebugWriter # requires Py2.5+ @@ -257,7 +258,9 @@ def create_pyx_pipeline(context, options, result, py=False, exclude_classes=()): inject_utility_code_stage_factory(context), abort_on_errors], debug_transform, - [generate_pyx_code_stage_factory(options, result)])) + [generate_pyx_code_stage_factory(options, result)], + ctest_support, + )) def create_pxd_pipeline(context, scope, module_name): from .CodeGeneration import ExtractPxdCode diff --git a/Cython/TestUtils.py b/Cython/TestUtils.py index bb2070d39..8328a3d6f 100644 --- a/Cython/TestUtils.py +++ b/Cython/TestUtils.py @@ -1,12 +1,14 @@ from __future__ import absolute_import import os +import re import unittest import shlex import sys import tempfile import textwrap from io import open +from functools import partial from .Compiler import Errors from .CodeWriter import CodeWriter @@ -161,11 +163,64 @@ class TransformTest(CythonTest): return tree +# For the test C code validation, we have to take care that the test directives (and thus +# the match strings) do not just appear in (multiline) C code comments containing the original +# Cython source code. Thus, we discard the comments before matching. +# This seems a prime case for re.VERBOSE, but it seems to match some of the whitespace. +_strip_c_comments = partial(re.compile( + re.sub('\s+', '', r''' + /[*] ( + (?: [^*\n] | [*][^/] )* + [\n] + (?: [^*] | [*][^/] )* + ) [*]/ + ''') +).sub, '') + + class TreeAssertVisitor(VisitorTransform): # actually, a TreeVisitor would be enough, but this needs to run # as part of the compiler pipeline - def visit_CompilerDirectivesNode(self, node): + def __init__(self): + super(TreeAssertVisitor, self).__init__() + self._module_pos = None + self._c_patterns = [] + self._c_antipatterns = [] + + def create_c_file_validator(self): + patterns, antipatterns = self._c_patterns, self._c_antipatterns + + def fail(pos, pattern, found, file_path): + Errors.error(pos, "Pattern '%s' %s found in %s" %( + pattern, + 'was' if found else 'was not', + file_path, + )) + + def validate_c_file(result): + c_file = result.c_file + if not (patterns or antipatterns): + #print("No patterns defined for %s" % c_file) + return result + + with open(c_file, encoding='utf8') as f: + c_content = f.read() + c_content = _strip_c_comments(c_content) + + for pattern in patterns: + #print("Searching pattern '%s'" % pattern) + if not re.search(pattern, c_content): + fail(self._module_pos, pattern, found=False, file_path=c_file) + + for antipattern in antipatterns: + #print("Searching antipattern '%s'" % antipattern) + if re.search(antipattern, c_content): + fail(self._module_pos, antipattern, found=True, file_path=c_file) + + return validate_c_file + + def _check_directives(self, node): directives = node.directives if 'test_assert_path_exists' in directives: for path in directives['test_assert_path_exists']: @@ -179,6 +234,19 @@ class TreeAssertVisitor(VisitorTransform): Errors.error( node.pos, "Unexpected path '%s' found in result tree" % path) + if 'test_assert_c_code_has' in directives: + self._c_patterns.extend(directives['test_assert_c_code_has']) + if 'test_fail_if_c_code_has' in directives: + self._c_antipatterns.extend(directives['test_fail_if_c_code_has']) + + def visit_ModuleNode(self, node): + self._module_pos = node.pos + self._check_directives(node) + self.visitchildren(node) + return node + + def visit_CompilerDirectivesNode(self, node): + self._check_directives(node) self.visitchildren(node) return node diff --git a/tests/compile/c_directives.pyx b/tests/compile/c_directives.pyx index 0ede90ba8..ee19e652f 100644 --- a/tests/compile/c_directives.pyx +++ b/tests/compile/c_directives.pyx @@ -2,6 +2,8 @@ # cython: boundscheck = False # cython: ignoreme = OK # cython: warn.undeclared = False +# cython: test_assert_c_code_has = Generated by Cython +# cython: test_fail_if_c_code_has = Generated by Python # This testcase is most useful if you inspect the generated C file diff --git a/tests/run/c_file_validation.srctree b/tests/run/c_file_validation.srctree new file mode 100644 index 000000000..cceb014ac --- /dev/null +++ b/tests/run/c_file_validation.srctree @@ -0,0 +1,72 @@ +""" +PYTHON run_test.py +""" + +######## run_test.py ######## + +import os +from collections import defaultdict +from os.path import basename, splitext + +from Cython.Compiler.Options import CompilationOptions +from Cython.Compiler.Main import compile as cython_compile +from Cython.Compiler.Options import default_options + + +def validate_file(filename): + module_name = basename(filename) + c_file = splitext(filename)[0] + '.c' + + options = CompilationOptions( + default_options, + language_level="3", + evaluate_tree_assertions=True, + ) + result = cython_compile(filename, options=options) + return result.num_errors + + +counts = defaultdict(int) +failed = False + +for filename in sorted(os.listdir(".")): + if "run_test" in filename: + continue + + print("Testing '%s'" % filename) + num_errors = validate_file(filename) + print(num_errors, filename) + counts[num_errors] += 1 + + if '_ok' in filename: + if num_errors > 0: + failed = True + print("ERROR: Compilation failed: %s (%s errors)" % (filename, num_errors)) + else: + if num_errors == 0: + failed = True + print("ERROR: Expected failure, but compilation succeeded: %s" % filename) + +assert counts == {0: 2, 1: 2}, counts +assert not failed + + +######## assert_ok.py ######## + +# cython: test_assert_c_code_has = Generated by Cython +# cython: test_assert_c_code_has = CYTHON_HEX_VERSION + + +######## assert_missing.py ######## + +# cython: test_assert_c_code_has = Generated by Python + + +######## fail_if_ok.py ######## + +# cython: test_fail_if_c_code_has = Generated by Python + + +######## fail_if_found.py ######## + +# cython: test_fail_if_c_code_has = Generated by Cython |