giscanner: parse block comments for members and fields

There was previously no mechanism for tagging enum members
and struct fields with Since tags (or other, eg deprecation tags).

While the customary place to add Since tags for these symbols
is inline in the parent symbol's documentation eg:

/**
 * Foo:
 *
 * @FOO_BAR: some bar. Since X.Y
 */

And variations on that theme, implementing parsing for that scheme
would result in a pretty ambiguous grammar, especially if we also
want support for multiple tags.

Instead, the solution implemented here is to allow providing
documentation for individual members and fields through their
own separate block, as is done for virtual functions already.

Inline comments are still used, with a lower precedence.

Fixes #348

3 files changed, 66 insertions, 19 deletions

diff --git a/giscanner/annotationparser.py b/giscanner/annotationparser.py
index 63212963..f8257206 100644
--- a/giscanner/annotationparser.py
+++ b/giscanner/annotationparser.py
@@ -461,6 +461,27 @@ ACTION_RE = re.compile(
     ''',
     re.UNICODE | re.VERBOSE)
 
+# Pattern matching struct fields.
+FIELD_RE = re.compile(
+    r'''
+    ^                                                    # start
+    \s*                                                  # 0 or more whitespace characters
+    (?P<class_name>[\w]+)                                # class name
+    \s*                                                  # 0 or more whitespace characters
+    \.{1}                                                # 1 required dot
+    \s*                                                  # 0 or more whitespace characters
+    (?P<field_name>[\w-]*\w)                             # field name
+    \s*                                                  # 0 or more whitespace characters
+    (?P<delimiter>:?)                                    # delimiter
+    \s*                                                  # 0 or more whitespace characters
+    (?P<fields>.*?)                                      # annotations + description
+    \s*                                                  # 0 or more whitespace characters
+    :?                                                   # invalid delimiter
+    \s*                                                  # 0 or more whitespace characters
+    $                                                    # end
+    ''',
+    re.UNICODE | re.VERBOSE)
+
 # Pattern matching parameters.
 PARAMETER_RE = re.compile(
     r'''
@@ -1368,13 +1389,22 @@ class GtkDocCommentBlockParser(object):
                                 identifier_fields = None
                                 identifier_fields_start = None
                             else:
-                                result = SYMBOL_RE.match(line)
+                                result = FIELD_RE.match(line)
 
                                 if result:
-                                    identifier_name = '%s' % (result.group('symbol_name'), )
+                                    identifier_name = '%s.%s' % (result.group('class_name'),
+                                                                  result.group('field_name'))
                                     identifier_delimiter = result.group('delimiter')
                                     identifier_fields = result.group('fields')
                                     identifier_fields_start = result.start('fields')
+                                else:
+                                    result = SYMBOL_RE.match(line)
+
+                                    if result:
+                                        identifier_name = '%s' % (result.group('symbol_name'), )
+                                        identifier_delimiter = result.group('delimiter')
+                                        identifier_fields = result.group('fields')
+                                        identifier_fields_start = result.start('fields')
 
                 if result:
                     in_part = PART_IDENTIFIER
diff --git a/giscanner/girwriter.py b/giscanner/girwriter.py
index d1333cb7..41d3ead2 100644
--- a/giscanner/girwriter.py
+++ b/giscanner/girwriter.py
@@ -437,6 +437,7 @@ class GIRWriter(XMLWriter):
         attrs = [('name', member.name),
                  ('value', str(member.value)),
                  ('c:identifier', member.symbol)]
+        self._append_version(member, attrs)
         if member.nick is not None:
             attrs.append(('glib:nick', member.nick))
         with self.tagcontext('member', attrs):
@@ -626,6 +627,7 @@ class GIRWriter(XMLWriter):
                 raise AssertionError("Unknown field anonymous: %r" % (field.anonymous_node, ))
         else:
             attrs = [('name', field.name)]
+            self._append_version(field, attrs)
             self._append_node_generic(field, attrs)
             # Fields are assumed to be read-only
             # (see also girparser.c and generate.c)
diff --git a/giscanner/maintransformer.py b/giscanner/maintransformer.py
index 9468751d..3c4ef695 100644
--- a/giscanner/maintransformer.py
+++ b/giscanner/maintransformer.py
@@ -858,19 +858,28 @@ class MainTransformer(object):
         self._apply_annotations_params(node, node.parameters, block)
         self._apply_annotations_return(node, node.retval, block)
 
-    def _apply_annotations_field(self, parent, block, field):
-        if not block:
-            return
-        tag = block.params.get(field.name)
-        if not tag:
+    def _apply_annotations_field(self, parent, parent_block, field):
+        block = self._blocks.get('%s.%s' % (self._get_annotation_name(parent), field.name))
+
+        # Prioritize block level documentation
+        if block:
+            self._apply_annotations_annotated(field, block)
+            annotations = block.annotations
+        elif not parent_block:
             return
-        type_annotation = tag.annotations.get(ANN_TYPE)
+        else:
+            tag = parent_block.params.get(field.name)
+            if not tag:
+                return
+            annotations = tag.annotations
+            field.doc = tag.description
+            field.doc_position = tag.position
+
+        type_annotation = annotations.get(ANN_TYPE)
         if type_annotation:
             field.type = self._transformer.create_type_from_user_string(type_annotation[0])
-        field.doc = tag.description
-        field.doc_position = tag.position
         try:
-            self._adjust_container_type(parent, field, tag.annotations)
+            self._adjust_container_type(parent, field, annotations)
         except AttributeError as ex:
             print(ex)
 
@@ -938,15 +947,21 @@ class MainTransformer(object):
         if value_annotation:
             node.value = value_annotation[0]
 
-    def _apply_annotations_enum_members(self, node, block):
-        if block is None:
-            return
-
+    def _apply_annotations_enum_members(self, node, parent_block):
         for m in node.members:
-            param = block.params.get(m.symbol, None)
-            if param and param.description:
-                m.doc = param.description
-                m.doc_position = param.position
+            block = self._blocks.get(m.symbol)
+            # Prioritize block-level documentation
+            if block:
+                self._apply_annotations_annotated(m, block)
+            elif parent_block:
+                param = parent_block.params.get(m.symbol)
+
+                if not param:
+                    continue
+
+                if param.description:
+                    m.doc = param.description
+                    m.doc_position = param.position
 
     def _pass_read_annotations2(self, node, chain):
         if isinstance(node, ast.Function):