giscanner: store code before and after comment block

so we can later use them to re-write source files containing
broken GTK-Doc comment blocks where /** is preceded by and/or
*/ is followed by code...

3 files changed, 221 insertions, 60 deletions

diff --git a/tests/offsets/offsets.h b/tests/offsets/offsets.h
index 1985f39a..7bdc1ad7 100644
--- a/tests/offsets/offsets.h
+++ b/tests/offsets/offsets.h
@@ -119,8 +119,7 @@ struct _OffsetsArray
   gpointer some_ptrs[5];
 };
 
-/** Test object offsets
- */
+/* Test object offsets */
 
 typedef struct _OffsetsObj OffsetsObj;
 typedef struct _OffsetsObjClass OffsetsObjClass;
diff --git a/tests/scanner/annotationparser/gi/syntax.xml b/tests/scanner/annotationparser/gi/syntax.xml
index 32528fb0..b8c85b50 100644
--- a/tests/scanner/annotationparser/gi/syntax.xml
+++ b/tests/scanner/annotationparser/gi/syntax.xml
@@ -20,6 +20,14 @@
   <!--
   Not GTK-Doc
   -->
+  <input>/**
+//</input>
+</test>
+
+<test>
+  <!--
+  Not GTK-Doc
+  -->
   <input>/*Test*/</input>
 </test>
 
@@ -51,37 +59,77 @@ something */</input>
   <!--
   Not GTK-Doc
   -->
-  <input>/**Test*/</input>
+  <input>/**
+*/</input>
 </test>
 
 <test>
-  <!--
-  Not GTK-Doc
-  -->
-  <input>/** Test */</input>
+  <input>/** Test **/</input>
+  <parser>
+    <messages>
+      <message>1: Error: Test: Skipping invalid GTK-Doc comment block:
+/** Test **/
+^</message>
+    </messages>
+  </parser>
 </test>
 
 <test>
-  <!--
-  Not GTK-Doc
-  -->
-  <input>/** Test
-something */</input>
+  <input>/**Test
+*/</input>
+  <parser>
+    <docblock>
+      <identifier>
+        <name>Test</name>
+      </identifier>
+    </docblock>
+    <messages>
+      <message>1: Warning: Test: GTK-Doc comment block start token "/**" should not be followed by comment text:
+/**Test
+   ^</message>
+    </messages>
+  </parser>
 </test>
 
 <test>
-  <!--
-  Not GTK-Doc
-  -->
   <input>/**
-*/</input>
+Test */</input>
+  <parser>
+    <docblock>
+      <identifier>
+        <name>Test</name>
+      </identifier>
+    </docblock>
+    <messages>
+      <message>2: Warning: Test: GTK-Doc comment block end token "*/" should not be preceded by comment text:
+Test */
+    ^</message>
+    </messages>
+  </parser>
+</test>
+
+<test>
+  <input>/** Test
+something */</input>
+  <parser>
+    <docblock>
+      <identifier>
+        <name>Test</name>
+      </identifier>
+      <description>something</description>
+    </docblock>
+    <messages>
+      <message>1: Warning: Test: GTK-Doc comment block start token "/**" should not be followed by comment text:
+/** Test
+    ^</message>
+      <message>2: Warning: Test: GTK-Doc comment block end token "*/" should not be preceded by comment text:
+something */
+         ^</message>
+    </messages>
+  </parser>
 </test>
 
 <test>
-  <!--
-  Technically not GTK-Doc, but we need to support this for backwards compatibility
-  with the old annotationparser.
-  -->
   <input>/**
 Test
 something */</input>
@@ -93,7 +141,7 @@ something */</input>
       <description>something</description>
     </docblock>
     <messages>
-      <message>3: Warning: Test: Comments should end with */ on a new line:
+      <message>3: Warning: Test: GTK-Doc comment block end token "*/" should not be preceded by comment text:
 something */
          ^</message>
     </messages>
@@ -101,10 +149,6 @@ something */
 </test>
 
 <test>
-  <!--
-  Technically not GTK-Doc, but we need to support this for backwards compatibility
-  with the old annotationparser.
-  -->
   <input>/**
 Test
 something **/</input>
@@ -116,7 +160,7 @@ something **/</input>
       <description>something</description>
     </docblock>
     <messages>
-      <message>3: Warning: Test: Comments should end with */ on a new line:
+      <message>3: Warning: Test: GTK-Doc comment block end token "*/" should not be preceded by comment text:
 something **/
          ^</message>
     </messages>
@@ -124,8 +168,35 @@ something **/
 </test>
 
 <test>
+  <input>code goes here /** Test
+something */ code goes here</input>
+  <parser>
+    <docblock>
+      <identifier>
+        <name>Test</name>
+      </identifier>
+      <description>something</description>
+    </docblock>
+    <messages>
+      <message>1: Warning: Test: GTK-Doc comment block start token "/**" should not be preceded by code:
+code goes here /** Test
+              ^</message>
+      <message>1: Warning: Test: GTK-Doc comment block start token "/**" should not be followed by comment text:
+code goes here /** Test
+                   ^</message>
+      <message>2: Warning: Test: GTK-Doc comment block end token "*/" should not be followed by code:
+something */ code goes here
+                           ^</message>
+      <message>2: Warning: Test: GTK-Doc comment block end token "*/" should not be preceded by comment text:
+something */ code goes here
+         ^</message>
+    </messages>
+  </parser>
+</test>
+
+<test>
   <!--
-  Broken comment block, signal the start of the comment block description followed
+  Malformed comment block, signal the start of the comment block description followed
   by a parameter instead.
   -->
   <input>/**
diff --git a/tests/scanner/annotationparser/test_patterns.py b/tests/scanner/annotationparser/test_patterns.py
index 023f61f6..ce82cf90 100644
--- a/tests/scanner/annotationparser/test_patterns.py
+++ b/tests/scanner/annotationparser/test_patterns.py
@@ -31,13 +31,132 @@ against the expected output.
 '''
 
 
-from giscanner.annotationparser import (SECTION_RE, SYMBOL_RE, PROPERTY_RE,
+from giscanner.annotationparser import (COMMENT_BLOCK_START_RE, COMMENT_BLOCK_END_RE,
+                                        SECTION_RE, SYMBOL_RE, PROPERTY_RE,
                                         SIGNAL_RE, PARAMETER_RE, TAG_RE,
-                                        TAG_VALUE_VERSION_RE, TAG_VALUE_STABILITY_RE,
-                                        COMMENT_END_RE)
+                                        TAG_VALUE_VERSION_RE, TAG_VALUE_STABILITY_RE)
 from unittest import (TestCase, main)
 
 
+comment_start_tests = [
+    (COMMENT_BLOCK_START_RE, '/**',
+         {'code': '',
+          'token': '/**',
+          'comment': ''}),
+    (COMMENT_BLOCK_START_RE, '   /**',
+         {'code': '',
+          'token': '/**',
+          'comment': ''}),
+    (COMMENT_BLOCK_START_RE, ' /** ',
+         {'code': '',
+          'token': '/**',
+          'comment': ''}),
+    (COMMENT_BLOCK_START_RE, 'xyz /** ',
+         {'code': 'xyz',
+          'token': '/**',
+          'comment': ''}),
+    (COMMENT_BLOCK_START_RE, '    xyz    /** ',
+         {'code': '    xyz',
+          'token': '/**',
+          'comment': ''}),
+    (COMMENT_BLOCK_START_RE, '/** xyz',
+         {'code': '',
+          'token': '/**',
+          'comment': 'xyz'}),
+    (COMMENT_BLOCK_START_RE, ' /**xyz',
+         {'code': '',
+          'token': '/**',
+          'comment': 'xyz'}),
+    (COMMENT_BLOCK_START_RE, ' /** xyz',
+         {'code': '',
+          'token': '/**',
+          'comment': 'xyz'}),
+    (COMMENT_BLOCK_START_RE, '/***',
+         None),
+    (COMMENT_BLOCK_START_RE, ' /***',
+         None),
+    (COMMENT_BLOCK_START_RE, ' /*** ',
+         None),
+    (COMMENT_BLOCK_START_RE, '/*** xyz',
+         None),
+    (COMMENT_BLOCK_START_RE, '/***** xyz',
+         None),
+    (COMMENT_BLOCK_START_RE, ' /*****xyz',
+         None),
+]
+
+
+comment_end_tests = [
+    (COMMENT_BLOCK_END_RE, '*/',
+         {'comment': '',
+          'token': '*/',
+          'code': ''}),
+    (COMMENT_BLOCK_END_RE, '   */',
+         {'comment': '',
+          'token': '*/',
+          'code': ''}),
+    (COMMENT_BLOCK_END_RE, ' */ ',
+         {'comment': '',
+          'token': '*/',
+          'code': ''}),
+    (COMMENT_BLOCK_END_RE, '*/xyz',
+         {'comment': '',
+          'token': '*/',
+          'code': 'xyz'}),
+    (COMMENT_BLOCK_END_RE, '   */xyz',
+         {'comment': '',
+          'token': '*/',
+          'code': 'xyz'}),
+    (COMMENT_BLOCK_END_RE, ' */ xyz',
+         {'comment': '',
+          'token': '*/',
+          'code': ' xyz'}),
+    (COMMENT_BLOCK_END_RE, '**/',
+         {'comment': '',
+          'token': '**/',
+          'code': ''}),
+    (COMMENT_BLOCK_END_RE, ' **/',
+         {'comment': '',
+          'token': '**/',
+          'code': ''}),
+    (COMMENT_BLOCK_END_RE, ' **/ ',
+         {'comment': '',
+          'token': '**/',
+          'code': ''}),
+    (COMMENT_BLOCK_END_RE, 'test */',
+         {'comment': 'test',
+          'token': '*/',
+          'code': ''}),
+    (COMMENT_BLOCK_END_RE, ' test*/',
+         {'comment': 'test',
+          'token': '*/',
+          'code': ''}),
+    (COMMENT_BLOCK_END_RE, 'test */ xyz',
+         {'comment': 'test',
+          'token': '*/',
+          'code': ' xyz'}),
+    (COMMENT_BLOCK_END_RE, ' test*/  xyz  ',
+         {'comment': 'test',
+          'token': '*/',
+          'code': '  xyz'}),
+    (COMMENT_BLOCK_END_RE, 'test **/',
+         {'comment': 'test',
+          'token': '**/',
+          'code': ''}),
+    (COMMENT_BLOCK_END_RE, ' test**/',
+         {'comment': 'test',
+          'token': '**/',
+          'code': ''}),
+    (COMMENT_BLOCK_END_RE, 'test *****/',
+         {'comment': 'test',
+          'token': '*****/',
+          'code': ''}),
+    (COMMENT_BLOCK_END_RE, ' test*****/',
+         {'comment': 'test',
+          'token': '*****/',
+          'code': ''})]
+
+
 identifier_section_tests = [
     (SECTION_RE, 'TSIEOCN',
          None),
@@ -571,33 +690,6 @@ tag_value_stability_tests = [
           'description': 'xyz: abc'})]
 
 
-comment_end_tests = [
-    (COMMENT_END_RE, '*/',
-         {'description': ''}),
-    (COMMENT_END_RE, '   */',
-         {'description': ''}),
-    (COMMENT_END_RE, ' */ ',
-         {'description': ''}),
-    (COMMENT_END_RE, '**/',
-         {'description': ''}),
-    (COMMENT_END_RE, ' **/',
-         {'description': ''}),
-    (COMMENT_END_RE, ' **/ ',
-         {'description': ''}),
-    (COMMENT_END_RE, 'test */',
-         {'description': 'test'}),
-    (COMMENT_END_RE, ' test*/',
-         {'description': 'test'}),
-    (COMMENT_END_RE, 'test **/',
-         {'description': 'test'}),
-    (COMMENT_END_RE, ' test**/',
-         {'description': 'test'}),
-    (COMMENT_END_RE, 'test *****/',
-         {'description': 'test'}),
-    (COMMENT_END_RE, ' test*****/',
-         {'description': 'test'})]
-
-
 def create_tests(tests_name, testcases):
     for (index, testcase) in enumerate(testcases):
         real_test_name = '%s_%03d' % (tests_name, index)
@@ -639,16 +731,15 @@ class TestProgram(TestCase):
 
 
 if __name__ == '__main__':
-    # Create tests from data
+    create_tests('test_comment_start', comment_start_tests)
+    create_tests('test_comment_end', comment_end_tests)
     create_tests('test_identifier_section', identifier_section_tests)
     create_tests('test_identifier_symbol', identifier_symbol_tests)
     create_tests('test_identifier_property', identifier_property_tests)
     create_tests('test_identifier_signal', identifier_signal_tests)
     create_tests('test_parameter', parameter_tests)
     create_tests('test_tag', tag_tests)
-    create_tests('test_comment_end', comment_end_tests)
     create_tests('test_tag_value_version', tag_value_version_tests)
     create_tests('test_tag_value_stability', tag_value_stability_tests)
 
-    # Run test suite
     main()