giscanner: honor line and paragraph breaks in description fields

GTK-Doc parameter description fields are allowed to span
multiple lines, tag description fields are allowed to span
multiple lines and paragraphs. A tool fixing/rewriting
GTK-Doc comment blocks in source files would need to have
description fields parsed and stored (almost) exactly as
they appear in the source file.

13 files changed, 124 insertions, 18 deletions

diff --git a/tests/scanner/annotationparser/gi/annotation_array.xml b/tests/scanner/annotationparser/gi/annotation_array.xml
index a59c267a..b821dcfc 100644
--- a/tests/scanner/annotationparser/gi/annotation_array.xml
+++ b/tests/scanner/annotationparser/gi/annotation_array.xml
@@ -67,7 +67,8 @@
               </options>
             </annotation>
           </annotations>
-          <description>Sequence of numbers that are zero-terminated</description>
+          <description>Sequence of
+  numbers that are zero-terminated</description>
         </parameter>
         <parameter>
           <name>n_nums</name>
@@ -116,7 +117,8 @@
               </options>
             </annotation>
           </annotations>
-          <description>Sequence of numbers that are zero-terminated</description>
+          <description>Sequence of numbers that
+are zero-terminated</description>
         </parameter>
         <parameter>
           <name>n_nums</name>
diff --git a/tests/scanner/annotationparser/gi/parameter.xml b/tests/scanner/annotationparser/gi/parameter.xml
index d6c4ab37..809b9d89 100644
--- a/tests/scanner/annotationparser/gi/parameter.xml
+++ b/tests/scanner/annotationparser/gi/parameter.xml
@@ -88,7 +88,8 @@
               <name>allow-none</name>
             </annotation>
           </annotations>
-          <description>(transfer full): first parameter</description>
+          <description>
+(transfer full): first parameter</description>
         </parameter>
       </parameters>
       <description>Annotations spanning multiple lines are not valid</description>
@@ -122,7 +123,8 @@
               <name>allow-none</name>
             </annotation>
           </annotations>
-          <description>first parameter having a second line which looks (like an annotation), but isn't. This should not result in a warning.</description>
+          <description>first parameter having a second line which looks
+(like an annotation), but isn't. This should not result in a warning.</description>
         </parameter>
       </parameters>
       <description>Annotations spanning multiple lines are not valid</description>
diff --git a/tests/scanner/annotationparser/gi/parameter_varargs.xml b/tests/scanner/annotationparser/gi/parameter_varargs.xml
index b873de28..0952ee92 100644
--- a/tests/scanner/annotationparser/gi/parameter_varargs.xml
+++ b/tests/scanner/annotationparser/gi/parameter_varargs.xml
@@ -28,7 +28,8 @@
       <parameters>
         <parameter>
           <name>par1</name>
-          <description>description of parameter 1. These can extend over more than one line.</description>
+          <description>description of parameter 1. These can extend over more than
+one line.</description>
         </parameter>
         <parameter>
           <name>par2</name>
@@ -134,7 +135,8 @@ name it is called in.</description>
       <parameters>
         <parameter>
           <name>par1</name>
-          <description>description of parameter 1. These can extend over more than one line.</description>
+          <description>description of parameter 1. These can extend over more than
+one line.</description>
         </parameter>
         <parameter>
           <name>par2</name>
@@ -202,7 +204,8 @@ other declarations (which may be documented elsewhere).</description>
       <parameters>
         <parameter>
           <name>par1</name>
-          <description>description of parameter 1. These can extend over more than one line.</description>
+          <description>description of parameter 1. These can extend over more than
+one line.</description>
         </parameter>
         <parameter>
           <name>par2</name>
diff --git a/tests/scanner/annotationparser/gi/syntax_nested_tags.xml b/tests/scanner/annotationparser/gi/syntax_nested_tags.xml
index 4ebe0ea4..d5a30423 100644
--- a/tests/scanner/annotationparser/gi/syntax_nested_tags.xml
+++ b/tests/scanner/annotationparser/gi/syntax_nested_tags.xml
@@ -28,7 +28,12 @@
       <parameters>
         <parameter>
           <name>G_APPLICATION_NON_UNIQUE</name>
-          <description>Make no attempts to do any of the typical single-instance application negotiation, even if the application ID is given.  The application neither attempts to become the owner of the application ID nor does it check if an existing owner already exists.  Everything occurs in the local process. Since: 2.30.</description>
+          <description>Make no attempts to do any of the typical
+    single-instance application negotiation, even if the application
+    ID is given.  The application neither attempts to become the
+    owner of the application ID nor does it check if an existing
+    owner already exists.  Everything occurs in the local process.
+    Since: 2.30.</description>
         </parameter>
       </parameters>
       <description>Flags used to define the behaviour of a #GApplication.</description>
diff --git a/tests/scanner/annotationparser/gi/syntax_whitespace.xml b/tests/scanner/annotationparser/gi/syntax_whitespace.xml
new file mode 100644
index 00000000..e66836c2
--- /dev/null
+++ b/tests/scanner/annotationparser/gi/syntax_whitespace.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<tests xmlns="http://schemas.gnome.org/gobject-introspection/2013/test">
+
+<test>
+  <input>/**
+ * whitespace_test:
+ * @object: (skip):    
+ *       
+ *       A #AnnotationObject
+ *        
+ *        
+ * @func: The callback
+ *                
+ *
+ * Test overriding via the "Rename To" annotation.
+ *        
+ *
+ * Returns: nothing
+ *        
+ * 
+ */</input>
+  <parser>
+    <docblock>
+      <identifier>
+        <name>whitespace_test</name>
+      </identifier>
+      <parameters>
+        <parameter>
+          <name>object</name>
+          <annotations>
+            <annotation>
+              <name>skip</name>
+            </annotation>
+          </annotations>
+        </parameter>
+        <parameter>
+          <name>func</name>
+          <description>The callback</description>
+        </parameter>
+      </parameters>
+      <description>A #AnnotationObject
+       
+       
+
+Test overriding via the "Rename To" annotation.</description>
+      <tags>
+        <tag>
+          <name>returns</name>
+          <description>nothing</description>
+        </tag>
+      </tags>
+    </docblock>
+    <messages>
+      <message>8: Warning: Test: "@func" parameter unexpected at this location:
+ * @func: The callback
+    ^</message>
+    </messages>
+  </parser>
+</test>
+
+</tests>
diff --git a/tests/scanner/annotationparser/gi/tag.xml b/tests/scanner/annotationparser/gi/tag.xml
index 2dfe41aa..d7eca816 100644
--- a/tests/scanner/annotationparser/gi/tag.xml
+++ b/tests/scanner/annotationparser/gi/tag.xml
@@ -25,7 +25,8 @@
               <name>allow-none</name>
             </annotation>
           </annotations>
-          <description>(transfer full): something</description>
+          <description>
+(transfer full): something</description>
         </tag>
       </tags>
     </docblock>
@@ -60,7 +61,8 @@
               <name>allow-none</name>
             </annotation>
           </annotations>
-          <description>Returns tag having a second line which looks (like an annotation), but isn't. This should not result in a warning.</description>
+          <description>Returns tag having a second line which looks
+(like an annotation), but isn't. This should not result in a warning.</description>
         </tag>
       </tags>
     </docblock>
diff --git a/tests/scanner/annotationparser/gi/tag_returns.xml b/tests/scanner/annotationparser/gi/tag_returns.xml
index ec6e2c45..2f76e025 100644
--- a/tests/scanner/annotationparser/gi/tag_returns.xml
+++ b/tests/scanner/annotationparser/gi/tag_returns.xml
@@ -53,7 +53,9 @@
       <tags>
         <tag>
           <name>returns</name>
-          <description>something  Tags should go after the comment block description</description>
+          <description>something
+
+Tags should go after the comment block description</description>
         </tag>
       </tags>
     </docblock>
diff --git a/tests/scanner/annotationparser/gi/tag_since.xml b/tests/scanner/annotationparser/gi/tag_since.xml
index 502d8b6e..78ce7865 100644
--- a/tests/scanner/annotationparser/gi/tag_since.xml
+++ b/tests/scanner/annotationparser/gi/tag_since.xml
@@ -91,7 +91,8 @@
         <tag>
           <name>since</name>
           <value>2.0</value>
-          <description>one of these "Since:" tags is wrong...</description>
+          <description>one of these "Since:"
+tags is wrong...</description>
         </tag>
       </tags>
     </docblock>
diff --git a/tests/scanner/annotationparser/gtkdoc/annotations/tester.c.xml b/tests/scanner/annotationparser/gtkdoc/annotations/tester.c.xml
index a39b1502..9fda10b5 100644
--- a/tests/scanner/annotationparser/gtkdoc/annotations/tester.c.xml
+++ b/tests/scanner/annotationparser/gtkdoc/annotations/tester.c.xml
@@ -98,7 +98,8 @@
               <name>allow-none</name>
             </annotation>
           </annotations>
-          <description>an optional string, which is used in ways too complicated to describe in a single line, making it necessary to wrap it</description>
+          <description>an optional string, which is used in ways too
+ complicated to describe in a single line, making it necessary to wrap it</description>
         </parameter>
       </parameters>
       <description>Document optional parameters.</description>
@@ -118,7 +119,8 @@
               <name>allow-none</name>
             </annotation>
           </annotations>
-          <description>Returns stuff which you have to free after use, whose description is also rather long</description>
+          <description>Returns stuff which you have to
+ free after use, whose description is also rather long</description>
         </tag>
       </tags>
     </docblock>
diff --git a/tests/scanner/annotationparser/gtkdoc/gobject/giface.c.xml b/tests/scanner/annotationparser/gtkdoc/gobject/giface.c.xml
index 053b44b4..dc9ae531 100644
--- a/tests/scanner/annotationparser/gtkdoc/gobject/giface.c.xml
+++ b/tests/scanner/annotationparser/gtkdoc/gobject/giface.c.xml
@@ -100,7 +100,6 @@ or \# or even \@.]]></description>
  * Configure a new instance
  *
  * Returns: %TRUE for sucess or %FALSE in case of an error
- *
  * Since: 0.1
  */</input>
   <parser>
diff --git a/tests/scanner/annotationparser/gtkdoc/gobject/gobject.c.xml b/tests/scanner/annotationparser/gtkdoc/gobject/gobject.c.xml
index 7234ed8a..111b5d67 100644
--- a/tests/scanner/annotationparser/gtkdoc/gobject/gobject.c.xml
+++ b/tests/scanner/annotationparser/gtkdoc/gobject/gobject.c.xml
@@ -207,7 +207,8 @@ All the internal details go here or not:
         </parameter>
         <parameter>
           <name>value</name>
-          <description>the new otest value, whose description extends further than one line will allow</description>
+          <description>the new otest value, whose description extends further than one
+ line will allow</description>
         </parameter>
       </parameters>
       <description>Set the #GtkdocObject:otest property.</description>
diff --git a/tests/scanner/annotationparser/gtkdoc/gobject/gtypes.h.xml b/tests/scanner/annotationparser/gtkdoc/gobject/gtypes.h.xml
index 4965adbe..644d5946 100644
--- a/tests/scanner/annotationparser/gtkdoc/gobject/gtypes.h.xml
+++ b/tests/scanner/annotationparser/gtkdoc/gobject/gtypes.h.xml
@@ -23,7 +23,8 @@
         </parameter>
         <parameter>
           <name>GTKDOC_ENUM_V2</name>
-          <description>second Since: 0.10</description>
+          <description>second
+   Since: 0.10</description>
         </parameter>
       </parameters>
       <description>Enum values for the #GtkdocEnum type.</description>
diff --git a/tests/scanner/annotationparser/test_patterns.py b/tests/scanner/annotationparser/test_patterns.py
index ead487d7..207fdc63 100644
--- a/tests/scanner/annotationparser/test_patterns.py
+++ b/tests/scanner/annotationparser/test_patterns.py
@@ -32,7 +32,7 @@ against the expected output.
 
 
 from giscanner.annotationparser import (COMMENT_BLOCK_START_RE, COMMENT_BLOCK_END_RE,
-                                        COMMENT_ASTERISK_RE,
+                                        COMMENT_ASTERISK_RE, INDENTATION_RE, EMPTY_LINE_RE,
                                         SECTION_RE, SYMBOL_RE, PROPERTY_RE,
                                         SIGNAL_RE, PARAMETER_RE, TAG_RE,
                                         TAG_VALUE_VERSION_RE, TAG_VALUE_STABILITY_RE)
@@ -175,6 +175,28 @@ comment_asterisk_tests = [
          None)]
 
 
+indentaton_tests = [
+    (INDENTATION_RE, '',
+         {'indentation': ''}),
+    (INDENTATION_RE, ' ',
+         {'indentation': ' '}),
+    (INDENTATION_RE, '    ',
+         {'indentation': '    '}),
+    (INDENTATION_RE, '    x',
+         {'indentation': '    '}),
+    (INDENTATION_RE, '    *',
+         {'indentation': '    '})]
+
+
+empty_line_tests = [
+    (EMPTY_LINE_RE, '',
+         {}),
+    (EMPTY_LINE_RE, ' ',
+         {}),
+    (EMPTY_LINE_RE, ' .',
+         None)]
+
+
 identifier_section_tests = [
     (SECTION_RE, 'TSIEOCN',
          None),
@@ -752,6 +774,8 @@ if __name__ == '__main__':
     create_tests('test_comment_start', comment_start_tests)
     create_tests('test_comment_end', comment_end_tests)
     create_tests('test_comment_asterisk', comment_asterisk_tests)
+    create_tests('test_indentaton', indentaton_tests)
+    create_tests('test_empty_line', empty_line_tests)
     create_tests('test_identifier_section', identifier_section_tests)
     create_tests('test_identifier_symbol', identifier_symbol_tests)
     create_tests('test_identifier_property', identifier_property_tests)