Implement semantic markup support for Ansible documentation in validate-modules. (#80243)

5 files changed, 330 insertions, 1 deletions

diff --git a/changelogs/fragments/80243-validate-modules.yml b/changelogs/fragments/80243-validate-modules.yml
new file mode 100644
index 0000000000..aa06db7547
--- /dev/null
+++ b/changelogs/fragments/80243-validate-modules.yml
@@ -0,0 +1,2 @@
+minor_changes:
+  - "validate-modules sanity test - add support for semantic markup (https://github.com/ansible/ansible/pull/80243)."
diff --git a/test/integration/targets/ansible-test-sanity-validate-modules/ansible_collections/ns/col/plugins/modules/semantic_markup.py b/test/integration/targets/ansible-test-sanity-validate-modules/ansible_collections/ns/col/plugins/modules/semantic_markup.py
new file mode 100644
index 0000000000..a7084499fd
--- /dev/null
+++ b/test/integration/targets/ansible-test-sanity-validate-modules/ansible_collections/ns/col/plugins/modules/semantic_markup.py
@@ -0,0 +1,107 @@
+#!/usr/bin/python
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+DOCUMENTATION = r'''
+module: semantic_markup
+short_description: Test semantic markup
+description:
+  - Test semantic markup.
+  - RV(does.not.exist=true).
+
+author:
+  - Ansible Core Team
+
+options:
+  foo:
+    description:
+      - Test.
+    type: str
+
+  a1:
+    description:
+      - O(foo)
+      - O(foo=bar)
+      - O(foo[1]=bar)
+      - O(ignore:bar=baz)
+      - O(ansible.builtin.copy#module:path=/)
+      - V(foo)
+      - V(bar(1\\2\)3)
+      - V(C(foo\)).
+      - E(env(var\))
+      - RV(ansible.builtin.copy#module:backup)
+      - RV(bar=baz)
+      - RV(ignore:bam)
+      - RV(ignore:bam.bar=baz)
+      - RV(bar).
+      - P(ansible.builtin.file#lookup)
+    type: str
+
+  a2:
+    description: V(C\(foo\)).
+    type: str
+
+  a3:
+    description: RV(bam).
+    type: str
+
+  a4:
+    description: P(foo.bar#baz).
+    type: str
+
+  a5:
+    description: P(foo.bar.baz).
+    type: str
+
+  a6:
+    description: P(foo.bar.baz#woof).
+    type: str
+
+  a7:
+    description: E(foo\(bar).
+    type: str
+
+  a8:
+    description: O(bar).
+    type: str
+
+  a9:
+    description: O(bar=bam).
+    type: str
+
+  a10:
+    description: O(foo.bar=1).
+    type: str
+'''
+
+EXAMPLES = '''#'''
+
+RETURN = r'''
+bar:
+  description: Bar.
+  type: int
+  returned: success
+  sample: 5
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+if __name__ == '__main__':
+    module = AnsibleModule(argument_spec=dict(
+        foo=dict(),
+        a1=dict(),
+        a2=dict(),
+        a3=dict(),
+        a4=dict(),
+        a5=dict(),
+        a6=dict(),
+        a7=dict(),
+        a8=dict(),
+        a9=dict(),
+        a10=dict(),
+    ))
+    module.exit_json()
diff --git a/test/integration/targets/ansible-test-sanity-validate-modules/expected.txt b/test/integration/targets/ansible-test-sanity-validate-modules/expected.txt
index 820a31f6e0..788045438a 100644
--- a/test/integration/targets/ansible-test-sanity-validate-modules/expected.txt
+++ b/test/integration/targets/ansible-test-sanity-validate-modules/expected.txt
@@ -9,3 +9,13 @@ plugins/modules/invalid_yaml_syntax.py:0:0: missing-documentation: No DOCUMENTAT
 plugins/modules/invalid_yaml_syntax.py:8:15: documentation-syntax-error: DOCUMENTATION is not valid YAML
 plugins/modules/invalid_yaml_syntax.py:12:15: invalid-examples: EXAMPLES is not valid YAML
 plugins/modules/invalid_yaml_syntax.py:16:15: return-syntax-error: RETURN is not valid YAML
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a2.description: Directive "V(C\(foo\))" contains unnecessarily quoted "(" for dictionary value @ data['options']['a2']['description']. Got 'V(C\\(foo\\)).'
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a4.description: Directive "P(foo.bar#baz)" must contain a FQCN; found "foo.bar" for dictionary value @ data['options']['a4']['description']. Got 'P(foo.bar#baz).'
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a5.description: Directive "P(foo.bar.baz)" must contain a "#" for dictionary value @ data['options']['a5']['description']. Got 'P(foo.bar.baz).'
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a6.description: Directive "P(foo.bar.baz#woof)" must contain a valid plugin type; found "woof" for dictionary value @ data['options']['a6']['description']. Got 'P(foo.bar.baz#woof).'
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a7.description: Directive "E(foo\(bar)" contains unnecessarily quoted "(" for dictionary value @ data['options']['a7']['description']. Got 'E(foo\\(bar).'
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: Directive "O(bar)" contains a non-existing option "bar"
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: Directive "O(bar=bam)" contains a non-existing option "bar"
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: Directive "O(foo.bar=1)" contains a non-existing option "foo.bar"
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: Directive "RV(bam)" contains a non-existing return value "bam"
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: Directive "RV(does.not.exist=true)" contains a non-existing return value "does.not.exist"
diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py
index b7d25318e1..4ca898be20 100644
--- a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py
+++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py
@@ -63,6 +63,7 @@ setup_collection_loader()
 
 from ansible import __version__ as ansible_version
 from ansible.executor.module_common import REPLACER_WINDOWS, NEW_STYLE_PYTHON_MODULE_RE
+from ansible.module_utils.common.collections import is_iterable
 from ansible.module_utils.common.parameters import DEFAULT_TYPE_VALIDATORS
 from ansible.module_utils.compat.version import StrictVersion, LooseVersion
 from ansible.module_utils.basic import to_bytes
@@ -74,7 +75,15 @@ from ansible.utils.version import SemanticVersion
 
 from .module_args import AnsibleModuleImportError, AnsibleModuleNotInitialized, get_argument_spec
 
-from .schema import ansible_module_kwargs_schema, doc_schema, return_schema
+from .schema import (
+    ansible_module_kwargs_schema,
+    doc_schema,
+    return_schema,
+    _SEM_OPTION_NAME,
+    _SEM_RET_VALUE,
+    _check_sem_quoting,
+    _parse_prefix,
+)
 
 from .utils import CaptureStd, NoArgsAnsibleModule, compare_unordered_lists, parse_yaml, parse_isodate
 
@@ -1028,6 +1037,8 @@ class ModuleValidator(Validator):
                     'invalid-documentation',
                 )
 
+            self._validate_all_semantic_markup(doc, returns)
+
             if not self.collection:
                 existing_doc = self._check_for_new_args(doc)
                 self._check_version_added(doc, existing_doc)
@@ -1153,6 +1164,112 @@ class ModuleValidator(Validator):
 
         return doc_info, doc
 
+    def _check_sem_option(self, directive, content):
+        try:
+            content = _check_sem_quoting(directive, content)
+            plugin_fqcn, plugin_type, option_link, option, value = _parse_prefix(directive, content)
+        except Exception:
+            # Validation errors have already been covered in the schema check
+            return
+        if plugin_fqcn is not None:
+            return
+        if tuple(option_link) not in self._all_options:
+            self.reporter.error(
+                path=self.object_path,
+                code='invalid-documentation-markup',
+                msg='Directive "%s" contains a non-existing option "%s"' % (directive, option)
+            )
+
+    def _check_sem_return_value(self, directive, content):
+        try:
+            content = _check_sem_quoting(directive, content)
+            plugin_fqcn, plugin_type, rv_link, rv, value = _parse_prefix(directive, content)
+        except Exception:
+            # Validation errors have already been covered in the schema check
+            return
+        if plugin_fqcn is not None:
+            return
+        if tuple(rv_link) not in self._all_return_values:
+            self.reporter.error(
+                path=self.object_path,
+                code='invalid-documentation-markup',
+                msg='Directive "%s" contains a non-existing return value "%s"' % (directive, rv)
+            )
+
+    def _validate_semantic_markup(self, object):
+        # Make sure we operate on strings
+        if is_iterable(object):
+            for entry in object:
+                self._validate_semantic_markup(entry)
+            return
+        if not isinstance(object, string_types):
+            return
+
+        for m in _SEM_OPTION_NAME.finditer(object):
+            self._check_sem_option(m.group(0), m.group(1))
+        for m in _SEM_RET_VALUE.finditer(object):
+            self._check_sem_return_value(m.group(0), m.group(1))
+
+    def _validate_semantic_markup_collect(self, destination, sub_key, data, all_paths):
+        if not isinstance(data, dict):
+            return
+        for key, value in data.items():
+            if not isinstance(value, dict):
+                continue
+            keys = {key}
+            if is_iterable(value.get('aliases')):
+                keys.update(value['aliases'])
+            new_paths = [path + [key] for path in all_paths for key in keys]
+            destination.update([tuple(path) for path in new_paths])
+            self._validate_semantic_markup_collect(destination, sub_key, value.get(sub_key), new_paths)
+
+    def _validate_semantic_markup_options(self, options):
+        if not isinstance(options, dict):
+            return
+        for key, value in options.items():
+            self._validate_semantic_markup(value.get('description'))
+            self._validate_semantic_markup_options(value.get('suboptions'))
+
+    def _validate_semantic_markup_return_values(self, return_vars):
+        if not isinstance(return_vars, dict):
+            return
+        for key, value in return_vars.items():
+            self._validate_semantic_markup(value.get('description'))
+            self._validate_semantic_markup(value.get('returned'))
+            self._validate_semantic_markup_return_values(value.get('contains'))
+
+    def _validate_all_semantic_markup(self, docs, return_docs):
+        if not isinstance(docs, dict):
+            docs = {}
+        if not isinstance(return_docs, dict):
+            return_docs = {}
+
+        self._all_options = set()
+        self._all_return_values = set()
+        self._validate_semantic_markup_collect(self._all_options, 'suboptions', docs.get('options'), [[]])
+        self._validate_semantic_markup_collect(self._all_return_values, 'contains', return_docs, [[]])
+
+        for string_keys in ('short_description', 'description', 'notes', 'requirements', 'todo'):
+            self._validate_semantic_markup(docs.get(string_keys))
+
+        if is_iterable(docs.get('seealso')):
+            for entry in docs.get('seealso'):
+                if isinstance(entry, dict):
+                    self._validate_semantic_markup(entry.get('description'))
+
+        if isinstance(docs.get('attributes'), dict):
+            for entry in docs.get('attributes').values():
+                if isinstance(entry, dict):
+                    for key in ('description', 'details'):
+                        self._validate_semantic_markup(entry.get(key))
+
+        if isinstance(docs.get('deprecated'), dict):
+            for key in ('why', 'alternative'):
+                self._validate_semantic_markup(docs.get('deprecated').get(key))
+
+        self._validate_semantic_markup_options(docs.get('options'))
+        self._validate_semantic_markup_return_values(return_docs)
+
     def _check_version_added(self, doc, existing_doc):
         version_added_raw = doc.get('version_added')
         try:
diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/schema.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/schema.py
index b979b156e1..9d6614637b 100644
--- a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/schema.py
+++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/schema.py
@@ -82,10 +82,26 @@ def date(error_code=None):
 
 
 _MODULE = re.compile(r"\bM\(([^)]+)\)")
+_PLUGIN = re.compile(r"\bP\(([^)]+)\)")
 _LINK = re.compile(r"\bL\(([^)]+)\)")
 _URL = re.compile(r"\bU\(([^)]+)\)")
 _REF = re.compile(r"\bR\(([^)]+)\)")
 
+_SEM_PARAMETER_STRING = r"\(((?:[^\\)]+|\\.)+)\)"
+_SEM_OPTION_NAME = re.compile(r"\bO" + _SEM_PARAMETER_STRING)
+_SEM_OPTION_VALUE = re.compile(r"\bV" + _SEM_PARAMETER_STRING)
+_SEM_ENV_VARIABLE = re.compile(r"\bE" + _SEM_PARAMETER_STRING)
+_SEM_RET_VALUE = re.compile(r"\bRV" + _SEM_PARAMETER_STRING)
+
+_UNESCAPE = re.compile(r"\\(.)")
+_CONTENT_LINK_SPLITTER_RE = re.compile(r'(?:\[[^\]]*\])?\.')
+_CONTENT_LINK_END_STUB_RE = re.compile(r'\[[^\]]*\]$')
+_FQCN_TYPE_PREFIX_RE = re.compile(r'^([^.]+\.[^.]+\.[^#]+)#([a-z]+):(.*)$')
+_IGNORE_MARKER = 'ignore:'
+_IGNORE_STRING = '(ignore)'
+
+_VALID_PLUGIN_TYPES = set(DOCUMENTABLE_PLUGINS)
+
 
 def _check_module_link(directive, content):
     if not FULLY_QUALIFIED_COLLECTION_RESOURCE_RE.match(content):
@@ -93,6 +109,21 @@ def _check_module_link(directive, content):
             Invalid('Directive "%s" must contain a FQCN' % directive), 'invalid-documentation-markup')
 
 
+def _check_plugin_link(directive, content):
+    if '#' not in content:
+        raise _add_ansible_error_code(
+            Invalid('Directive "%s" must contain a "#"' % directive), 'invalid-documentation-markup')
+    plugin_fqcn, plugin_type = content.split('#', 1)
+    if not FULLY_QUALIFIED_COLLECTION_RESOURCE_RE.match(plugin_fqcn):
+        raise _add_ansible_error_code(
+            Invalid('Directive "%s" must contain a FQCN; found "%s"' % (directive, plugin_fqcn)),
+            'invalid-documentation-markup')
+    if plugin_type not in _VALID_PLUGIN_TYPES:
+        raise _add_ansible_error_code(
+            Invalid('Directive "%s" must contain a valid plugin type; found "%s"' % (directive, plugin_type)),
+            'invalid-documentation-markup')
+
+
 def _check_link(directive, content):
     if ',' not in content:
         raise _add_ansible_error_code(
@@ -119,6 +150,58 @@ def _check_ref(directive, content):
             Invalid('Directive "%s" must contain a comma' % directive), 'invalid-documentation-markup')
 
 
+def _check_sem_quoting(directive, content):
+    for m in _UNESCAPE.finditer(content):
+        if m.group(1) not in ('\\', ')'):
+            raise _add_ansible_error_code(
+                Invalid('Directive "%s" contains unnecessarily quoted "%s"' % (directive, m.group(1))),
+                'invalid-documentation-markup')
+    return _UNESCAPE.sub(r'\1', content)
+
+
+def _parse_prefix(directive, content):
+    value = None
+    if '=' in content:
+        content, value = content.split('=', 1)
+    m = _FQCN_TYPE_PREFIX_RE.match(content)
+    if m:
+        plugin_fqcn = m.group(1)
+        plugin_type = m.group(2)
+        content = m.group(3)
+        if not FULLY_QUALIFIED_COLLECTION_RESOURCE_RE.match(plugin_fqcn):
+            raise _add_ansible_error_code(
+                Invalid('Directive "%s" must contain a FQCN; found "%s"' % (directive, plugin_fqcn)),
+                'invalid-documentation-markup')
+        if plugin_type not in _VALID_PLUGIN_TYPES:
+            raise _add_ansible_error_code(
+                Invalid('Directive "%s" must contain a valid plugin type; found "%s"' % (directive, plugin_type)),
+                'invalid-documentation-markup')
+    elif content.startswith(_IGNORE_MARKER):
+        content = content[len(_IGNORE_MARKER):]
+        plugin_fqcn = plugin_type = _IGNORE_STRING
+    else:
+        plugin_fqcn = plugin_type = None
+    if ':' in content or '#' in content:
+        raise _add_ansible_error_code(
+            Invalid('Directive "%s" contains wrongly specified FQCN/plugin type' % directive),
+            'invalid-documentation-markup')
+    content_link = _CONTENT_LINK_SPLITTER_RE.split(content)
+    for i, part in enumerate(content_link):
+        if i == len(content_link) - 1:
+            part = _CONTENT_LINK_END_STUB_RE.sub('', part)
+            content_link[i] = part
+        if '.' in part or '[' in part or ']' in part:
+            raise _add_ansible_error_code(
+                Invalid('Directive "%s" contains invalid name "%s"' % (directive, content)),
+                'invalid-documentation-markup')
+    return plugin_fqcn, plugin_type, content_link, content, value
+
+
+def _check_sem_option_return_value(directive, content):
+    content = _check_sem_quoting(directive, content)
+    _parse_prefix(directive, content)
+
+
 def doc_string(v):
     """Match a documentation string."""
     if not isinstance(v, string_types):
@@ -126,12 +209,22 @@ def doc_string(v):
             Invalid('Must be a string'), 'invalid-documentation')
     for m in _MODULE.finditer(v):
         _check_module_link(m.group(0), m.group(1))
+    for m in _PLUGIN.finditer(v):
+        _check_plugin_link(m.group(0), m.group(1))
     for m in _LINK.finditer(v):
         _check_link(m.group(0), m.group(1))
     for m in _URL.finditer(v):
         _check_url(m.group(0), m.group(1))
     for m in _REF.finditer(v):
         _check_ref(m.group(0), m.group(1))
+    for m in _SEM_OPTION_NAME.finditer(v):
+        _check_sem_option_return_value(m.group(0), m.group(1))
+    for m in _SEM_OPTION_VALUE.finditer(v):
+        _check_sem_quoting(m.group(0), m.group(1))
+    for m in _SEM_ENV_VARIABLE.finditer(v):
+        _check_sem_quoting(m.group(0), m.group(1))
+    for m in _SEM_RET_VALUE.finditer(v):
+        _check_sem_option_return_value(m.group(0), m.group(1))
     return v