Allow C code assertions in tests by defining regular expressions in module directives. (GH-4938)

6 files changed, 165 insertions, 8 deletions

diff --git a/Cython/Compiler/Options.py b/Cython/Compiler/Options.py
index f400e4ba2..bb547e978 100644
--- a/Cython/Compiler/Options.py
+++ b/Cython/Compiler/Options.py
@@ -171,7 +171,7 @@ def copy_inherited_directives(outer_directives, **new_directives):
     # For example, test_assert_path_exists and test_fail_if_path_exists should not be inherited
     #  otherwise they can produce very misleading test failures
     new_directives_out = dict(outer_directives)
-    for name in ('test_assert_path_exists', 'test_fail_if_path_exists'):
+    for name in ('test_assert_path_exists', 'test_fail_if_path_exists', 'test_assert_c_code_has', 'test_fail_if_c_code_has'):
         new_directives_out.pop(name, None)
     new_directives_out.update(new_directives)
     return new_directives_out
@@ -247,6 +247,8 @@ _directive_defaults = {
 # test support
     'test_assert_path_exists' : [],
     'test_fail_if_path_exists' : [],
+    'test_assert_c_code_has' : [],
+    'test_fail_if_c_code_has' : [],
 
 # experimental, subject to change
     'formal_grammar': False,
@@ -364,6 +366,8 @@ directive_scopes = {  # defaults to available everywhere
     'set_initial_path' : ('module',),
     'test_assert_path_exists' : ('function', 'class', 'cclass'),
     'test_fail_if_path_exists' : ('function', 'class', 'cclass'),
+    'test_assert_c_code_has' : ('module',),
+    'test_fail_if_c_code_has' : ('module',),
     'freelist': ('cclass',),
     'emit_code_comments': ('module',),
     # Avoid scope-specific to/from_py_functions for c_string.
@@ -509,6 +513,11 @@ def parse_directive_list(s, relaxed_bool=False, ignore_unknown=False,
                         result[directive] = parsed_value
             if not found and not ignore_unknown:
                 raise ValueError('Unknown option: "%s"' % name)
+        elif directive_types.get(name) is list:
+            if name in result:
+                result[name].append(value)
+            else:
+                result[name] = [value]
         else:
             parsed_value = parse_directive_value(name, value, relaxed_bool=relaxed_bool)
             result[name] = parsed_value
diff --git a/Cython/Compiler/Parsing.py b/Cython/Compiler/Parsing.py
index d7c844849..8160149af 100644
--- a/Cython/Compiler/Parsing.py
+++ b/Cython/Compiler/Parsing.py
@@ -3853,6 +3853,9 @@ def p_compiler_directive_comments(s):
             for name in new_directives:
                 if name not in result:
                     pass
+                elif Options.directive_types.get(name) is list:
+                    result[name] += new_directives[name]
+                    new_directives[name] = result[name]
                 elif new_directives[name] == result[name]:
                     warning(pos, "Duplicate directive found: %s" % (name,))
                 else:
diff --git a/Cython/Compiler/Pipeline.py b/Cython/Compiler/Pipeline.py
index 3a5c42352..2fd3a1d3f 100644
--- a/Cython/Compiler/Pipeline.py
+++ b/Cython/Compiler/Pipeline.py
@@ -231,14 +231,15 @@ def create_pipeline(context, mode, exclude_classes=()):
     return stages
 
 def create_pyx_pipeline(context, options, result, py=False, exclude_classes=()):
-    if py:
-        mode = 'py'
-    else:
-        mode = 'pyx'
+    mode = 'py' if py else 'pyx'
+
     test_support = []
+    ctest_support = []
     if options.evaluate_tree_assertions:
         from ..TestUtils import TreeAssertVisitor
-        test_support.append(TreeAssertVisitor())
+        test_validator = TreeAssertVisitor()
+        test_support.append(test_validator)
+        ctest_support.append(test_validator.create_c_file_validator())
 
     if options.gdb_debug:
         from ..Debugger import DebugWriter  # requires Py2.5+
@@ -257,7 +258,9 @@ def create_pyx_pipeline(context, options, result, py=False, exclude_classes=()):
          inject_utility_code_stage_factory(context),
          abort_on_errors],
         debug_transform,
-        [generate_pyx_code_stage_factory(options, result)]))
+        [generate_pyx_code_stage_factory(options, result)],
+        ctest_support,
+    ))
 
 def create_pxd_pipeline(context, scope, module_name):
     from .CodeGeneration import ExtractPxdCode
diff --git a/Cython/TestUtils.py b/Cython/TestUtils.py
index bb2070d39..8328a3d6f 100644
--- a/Cython/TestUtils.py
+++ b/Cython/TestUtils.py
@@ -1,12 +1,14 @@
 from __future__ import absolute_import
 
 import os
+import re
 import unittest
 import shlex
 import sys
 import tempfile
 import textwrap
 from io import open
+from functools import partial
 
 from .Compiler import Errors
 from .CodeWriter import CodeWriter
@@ -161,11 +163,64 @@ class TransformTest(CythonTest):
         return tree
 
 
+# For the test C code validation, we have to take care that the test directives (and thus
+# the match strings) do not just appear in (multiline) C code comments containing the original
+# Cython source code.  Thus, we discard the comments before matching.
+# This seems a prime case for re.VERBOSE, but it seems to match some of the whitespace.
+_strip_c_comments = partial(re.compile(
+    re.sub('\s+', '', r'''
+        /[*] (
+            (?: [^*\n] | [*][^/] )*
+            [\n]
+            (?: [^*] | [*][^/] )*
+        ) [*]/
+    ''')
+).sub, '')
+
+
 class TreeAssertVisitor(VisitorTransform):
     # actually, a TreeVisitor would be enough, but this needs to run
     # as part of the compiler pipeline
 
-    def visit_CompilerDirectivesNode(self, node):
+    def __init__(self):
+        super(TreeAssertVisitor, self).__init__()
+        self._module_pos = None
+        self._c_patterns = []
+        self._c_antipatterns = []
+
+    def create_c_file_validator(self):
+        patterns, antipatterns = self._c_patterns, self._c_antipatterns
+
+        def fail(pos, pattern, found, file_path):
+            Errors.error(pos, "Pattern '%s' %s found in %s" %(
+                pattern,
+                'was' if found else 'was not',
+                file_path,
+            ))
+
+        def validate_c_file(result):
+            c_file = result.c_file
+            if not (patterns or antipatterns):
+                #print("No patterns defined for %s" % c_file)
+                return result
+
+            with open(c_file, encoding='utf8') as f:
+                c_content = f.read()
+            c_content = _strip_c_comments(c_content)
+
+            for pattern in patterns:
+                #print("Searching pattern '%s'" % pattern)
+                if not re.search(pattern, c_content):
+                    fail(self._module_pos, pattern, found=False, file_path=c_file)
+
+            for antipattern in antipatterns:
+                #print("Searching antipattern '%s'" % antipattern)
+                if re.search(antipattern, c_content):
+                    fail(self._module_pos, antipattern, found=True, file_path=c_file)
+
+        return validate_c_file
+
+    def _check_directives(self, node):
         directives = node.directives
         if 'test_assert_path_exists' in directives:
             for path in directives['test_assert_path_exists']:
@@ -179,6 +234,19 @@ class TreeAssertVisitor(VisitorTransform):
                     Errors.error(
                         node.pos,
                         "Unexpected path '%s' found in result tree" % path)
+        if 'test_assert_c_code_has' in directives:
+            self._c_patterns.extend(directives['test_assert_c_code_has'])
+        if 'test_fail_if_c_code_has' in directives:
+            self._c_antipatterns.extend(directives['test_fail_if_c_code_has'])
+
+    def visit_ModuleNode(self, node):
+        self._module_pos = node.pos
+        self._check_directives(node)
+        self.visitchildren(node)
+        return node
+
+    def visit_CompilerDirectivesNode(self, node):
+        self._check_directives(node)
         self.visitchildren(node)
         return node
 
diff --git a/tests/compile/c_directives.pyx b/tests/compile/c_directives.pyx
index 0ede90ba8..ee19e652f 100644
--- a/tests/compile/c_directives.pyx
+++ b/tests/compile/c_directives.pyx
@@ -2,6 +2,8 @@
 # cython: boundscheck  =  False
 # cython: ignoreme = OK
 # cython: warn.undeclared = False
+# cython: test_assert_c_code_has = Generated by Cython
+# cython: test_fail_if_c_code_has = Generated by Python
 
 # This testcase is most useful if you inspect the generated C file
 
diff --git a/tests/run/c_file_validation.srctree b/tests/run/c_file_validation.srctree
new file mode 100644
index 000000000..cceb014ac
--- /dev/null
+++ b/tests/run/c_file_validation.srctree
@@ -0,0 +1,72 @@
+"""
+PYTHON run_test.py
+"""
+
+######## run_test.py ########
+
+import os
+from collections import defaultdict
+from os.path import basename, splitext
+
+from Cython.Compiler.Options import CompilationOptions
+from Cython.Compiler.Main import compile as cython_compile
+from Cython.Compiler.Options import default_options
+
+
+def validate_file(filename):
+    module_name = basename(filename)
+    c_file = splitext(filename)[0] + '.c'
+
+    options = CompilationOptions(
+        default_options,
+        language_level="3",
+        evaluate_tree_assertions=True,
+    )
+    result = cython_compile(filename, options=options)
+    return result.num_errors
+
+
+counts = defaultdict(int)
+failed = False
+
+for filename in sorted(os.listdir(".")):
+    if "run_test" in filename:
+        continue
+
+    print("Testing '%s'" % filename)
+    num_errors = validate_file(filename)
+    print(num_errors, filename)
+    counts[num_errors] += 1
+
+    if '_ok' in filename:
+        if num_errors > 0:
+            failed = True
+            print("ERROR: Compilation failed: %s (%s errors)" % (filename, num_errors))
+    else:
+        if num_errors == 0:
+            failed = True
+            print("ERROR: Expected failure, but compilation succeeded: %s" % filename)
+
+assert counts == {0: 2, 1: 2}, counts
+assert not failed
+
+
+######## assert_ok.py ########
+
+# cython: test_assert_c_code_has = Generated by Cython
+# cython: test_assert_c_code_has = CYTHON_HEX_VERSION
+
+
+######## assert_missing.py ########
+
+# cython: test_assert_c_code_has = Generated by Python
+
+
+######## fail_if_ok.py ########
+
+# cython: test_fail_if_c_code_has = Generated by Python
+
+
+######## fail_if_found.py ########
+
+# cython: test_fail_if_c_code_has = Generated by Cython