Revamp the plugin_formatter doc generator

* Use a template to generate the category lists
* Refactor so that we first extract all of the data that we need to
  build the docs and then give that data to the templates to build with
* Add docs page listing modules ordered by support level

1 files changed, 260 insertions, 241 deletions

diff --git a/docs/bin/plugin_formatter.py b/docs/bin/plugin_formatter.py
index b858046b86..58092bd49e 100755
--- a/docs/bin/plugin_formatter.py
+++ b/docs/bin/plugin_formatter.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
 # (c) 2012-2014, Michael DeHaan <michael@ansible.com> and others
+# (c) 2017 Ansible Project
 #
 # This file is part of Ansible
 #
@@ -17,10 +18,10 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
-import cgi
+
 import datetime
 import glob
 import optparse
@@ -28,9 +29,17 @@ import os
 import re
 import sys
 import warnings
-import yaml
-
 from collections import defaultdict
+try:
+    from html import escape as html_escape
+except ImportError:
+    # Python-3.2 or later
+    import cgi
+
+    def html_escape(text, quote=True):
+        return cgi.escape(text, quote)
+
+import yaml
 from jinja2 import Environment, FileSystemLoader
 from six import iteritems
 
@@ -38,6 +47,7 @@ from ansible.errors import AnsibleError
 from ansible.module_utils._text import to_bytes
 from ansible.utils import plugin_docs
 
+
 #####################################################################################
 # constants and paths
 
@@ -82,7 +92,7 @@ def rst_ify(text):
 def html_ify(text):
     ''' convert symbols like I(this is in italics) to valid HTML '''
 
-    t = cgi.escape(text)
+    t = html_escape(text)
     t = _ITALIC.sub("<em>" + r"\1" + "</em>", t)
     t = _BOLD.sub("<b>" + r"\1" + "</b>", t)
     t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t)
@@ -104,25 +114,48 @@ def rst_xline(width, char="="):
     return char * width
 
 
-def write_data(text, options, outputname, module):
+def write_data(text, output_dir, outputname, module=None):
     ''' dumps module output to a file or the screen, as requested '''
 
-    if options.output_dir is not None:
-        fname = os.path.join(options.output_dir, outputname % module)
+    if output_dir is not None:
+        if module:
+            outputname = outputname % module
+        fname = os.path.join(output_dir, outputname)
         fname = fname.replace(".py", "")
-        f = open(fname, 'wb')
-        f.write(to_bytes(text))
-        f.close()
+        with open(fname, 'wb') as f:
+            f.write(to_bytes(text))
     else:
         print(text)
 
 
-def list_modules(module_dir, depth=0, limit_to_modules=None):
-    ''' returns a hash of categories, each category being a hash of module names to file paths '''
+def get_module_info(module_dir, limit_to_modules=None, verbose=False):
+    '''
+    Returns information about modules and the categories that they belong to
+
+    :arg module_dir: file system path to the top of the modules directory
+    :kwarg limit_to_modules: If given, this is a list of module names to
+        generate information for.  All other modules will be ignored.
+    :returns: Tuple of two dicts containing module_info, categories, and
+        aliases and a set listing deprecated modules:
+
+        :module_info: mapping of module names to information about them.  The fields of the dict are:
+
+            :path: filesystem path to the module
+            :deprecated: boolean.  True means the module is deprecated otherwise not.
+            :aliases: set of aliases to this module name
+            :metadata: The modules metadata (as recorded in the module)
+            :doc: The documentation structure for the module
+            :examples: The module's examples
+            :returndocs: The module's returndocs
+
+        :categories: maps category names to a dict.  The dict contains at
+            least one key, '_modules' which contains a list of module names in
+            that category.  Any other keys in the dict are subcategories with
+            the same structure.
+    '''
 
     categories = dict()
-    module_info = dict()
-    aliases = defaultdict(set)
+    module_info = defaultdict(dict)
 
     # * windows powershell modules have documentation stubs in python docstring
     #   format (they are not executed) so skip the ps1 format files
@@ -135,41 +168,72 @@ def list_modules(module_dir, depth=0, limit_to_modules=None):
     )
 
     for module_path in files:
+        # Do not list __init__.py files
         if module_path.endswith('__init__.py'):
             continue
+
+        # Do not list blacklisted modules
+        module = os.path.splitext(os.path.basename(module_path))[0]
+        if module in plugin_docs.BLACKLIST['MODULE']:
+            continue
+
+        # If requested, limit module documentation building only to passed-in
+        # modules.
+        if limit_to_modules is not None and module.lower() in limit_to_modules:
+            continue
+
+        deprecated = False
+        if module.startswith("_"):
+            if os.path.islink(module_path):
+                # Handle aliases
+                source = os.path.splitext(os.path.basename(os.path.realpath(module_path)))[0]
+                module = module.replace("_", "", 1)
+                aliases = module_info[source].get('aliases', set())
+                aliases.add(module)
+                # In case we just created this via get()'s fallback
+                module_info[source]['aliases'] = aliases
+                continue
+            else:
+                # Handle deprecations
+                module = module.replace("_", "", 1)
+                deprecated = True
+
+        #
+        # Regular module to process
+        #
+
         category = categories
-        mod_path_only = module_path
-        # Start at the second directory because we don't want the "vendor"
 
+        # Start at the second directory because we don't want the "vendor"
         mod_path_only = os.path.dirname(module_path[len(module_dir):])
 
-        # directories (core, extras)
+        # build up the categories that this module belongs to
         for new_cat in mod_path_only.split('/')[1:]:
             if new_cat not in category:
                 category[new_cat] = dict()
+                category[new_cat]['_modules'] = []
             category = category[new_cat]
 
-        module = os.path.splitext(os.path.basename(module_path))[0]
-        if module in plugin_docs.BLACKLIST['MODULE']:
-            # Do not list blacklisted modules
-            continue
-        if module.startswith("_") and os.path.islink(module_path):
-            source = os.path.splitext(os.path.basename(os.path.realpath(module_path)))[0]
-            module = module.replace("_", "", 1)
-            aliases[source].add(module)
-            continue
+        category['_modules'].append(module)
 
-        # If requested, limit module documentation building only to passed-in
-        # modules.
-        if limit_to_modules is None or module.lower() in limit_to_modules:
-            category[module] = module_path
-            module_info[module] = module_path
+        # use ansible core library to parse out doc metadata YAML and plaintext examples
+        doc, examples, returndocs, metadata = plugin_docs.get_docstring(module_path, verbose=verbose)
+
+        # save all the information
+        module_info[module] = {'path': module_path,
+                               'deprecated': deprecated,
+                               'aliases': set(),
+                               'metadata': metadata,
+                               'doc': doc,
+                               'examples': examples,
+                               'returndocs': returndocs,
+                               }
 
     # keep module tests out of becoming module docs
     if 'test' in categories:
         del categories['test']
 
-    return module_info, categories, aliases
+    return module_info, categories
 
 
 def generate_parser():
@@ -202,17 +266,21 @@ def jinja2_environment(template_dir, typ):
                       trim_blocks=True)
     env.globals['xline'] = rst_xline
 
+    templates = {}
     if typ == 'rst':
         env.filters['convert_symbols_to_format'] = rst_ify
         env.filters['html_ify'] = html_ify
         env.filters['fmt'] = rst_fmt
         env.filters['xline'] = rst_xline
-        template = env.get_template('plugin.rst.j2')
+        templates['plugin'] = env.get_template('plugin.rst.j2')
+        templates['category_list'] = env.get_template('modules_by_category.rst.j2')
+        templates['support_list'] = env.get_template('modules_by_support.rst.j2')
+        templates['list_of_CATEGORY_modules'] = env.get_template('list_of_CATEGORY_modules.rst.j2')
         outputname = "%s_module.rst"
     else:
         raise Exception("unknown module format type: %s" % typ)
 
-    return env, template, outputname
+    return templates, outputname
 
 
 def too_old(added):
@@ -225,196 +293,156 @@ def too_old(added):
     except ValueError as e:
         warnings.warn("Could not parse %s: %s" % (added, str(e)))
         return False
-    return (added_float < TO_OLD_TO_BE_NOTABLE)
-
-
-def process_module(module, options, env, template, outputname, module_map, aliases):
-
-    fname = module_map[module]
-    if isinstance(fname, dict):
-        return "SKIPPED"
-
-    basename = os.path.basename(fname)
-    deprecated = False
+    return added_float < TO_OLD_TO_BE_NOTABLE
 
-    # ignore files with extensions
-    if not basename.endswith(".py"):
-        return
-    elif module.startswith("_"):
-        if os.path.islink(fname):
-            return  # ignore, its an alias
-        deprecated = True
-        module = module.replace("_", "", 1)
 
-    print("rendering: %s" % module)
+def process_modules(module_map, templates, outputname, output_dir, ansible_version):
+    for module in module_map:
+        print("rendering: %s" % module)
 
-    # use ansible core library to parse out doc metadata YAML and plaintext examples
-    doc, examples, returndocs, metadata = plugin_docs.get_docstring(fname, verbose=options.verbose)
+        fname = module_map[module]['path']
 
-    # crash if module is missing documentation and not explicitly hidden from docs index
-    if doc is None:
-        sys.exit("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
+        # crash if module is missing documentation and not explicitly hidden from docs index
+        if module_map[module]['doc'] is None:
+            sys.exit("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
 
-    if deprecated and 'deprecated' not in doc:
-        sys.exit("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module))
+        # Going to reference this heavily so make a short name to reference it by
+        doc = module_map[module]['doc']
 
-    if module in aliases:
-        doc['aliases'] = aliases[module]
+        if module_map[module]['deprecated'] and 'deprecated' not in doc:
+            sys.exit("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module))
 
-    all_keys = []
+        if 'version_added' not in doc:
+            sys.exit("*** ERROR: missing version_added in: %s ***\n" % module)
 
-    if 'version_added' not in doc:
-        sys.exit("*** ERROR: missing version_added in: %s ***\n" % module)
+        #
+        # The present template gets everything from doc so we spend most of this
+        # function moving data into doc for the template to reference
+        #
 
-    added = 0
-    if doc['version_added'] == 'historical':
-        del doc['version_added']
-    else:
-        added = doc['version_added']
-
-    # don't show version added information if it's too old to be called out
-    if too_old(added):
-        del doc['version_added']
-
-    if 'options' in doc and doc['options']:
-        for (k, v) in iteritems(doc['options']):
-            # don't show version added information if it's too old to be called out
-            if 'version_added' in doc['options'][k] and too_old(doc['options'][k]['version_added']):
-                del doc['options'][k]['version_added']
-            if 'description' not in doc['options'][k]:
-                raise AnsibleError("Missing required description for option %s in %s " % (k, module))
-
-            required_value = doc['options'][k].get('required', False)
-            if not isinstance(required_value, bool):
-                raise AnsibleError("Invalid required value '%s' for option '%s' in '%s' (must be truthy)" % (required_value, k, module))
-            if not isinstance(doc['options'][k]['description'], list):
-                doc['options'][k]['description'] = [doc['options'][k]['description']]
-
-            all_keys.append(k)
-
-    all_keys = sorted(all_keys)
-
-    doc['option_keys'] = all_keys
-    doc['filename'] = fname
-    doc['docuri'] = doc['module'].replace('_', '-')
-    doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d')
-    doc['ansible_version'] = options.ansible_version
-    doc['plainexamples'] = examples  # plain text
-    doc['metadata'] = metadata
-
-    if returndocs:
-        try:
-            doc['returndocs'] = yaml.safe_load(returndocs)
-        except:
-            print("could not load yaml: %s" % returndocs)
-            raise
-    else:
-        doc['returndocs'] = None
+        if module_map[module]['aliases']:
+            doc['aliases'] = module_map[module]['aliases']
 
-    # here is where we build the table of contents...
-
-    try:
-        text = template.render(doc)
-    except Exception as e:
-        raise AnsibleError("Failed to render doc for %s: %s" % (fname, str(e)))
-    write_data(text, options, outputname, module)
-    return doc['short_description']
-
-
-def print_modules(module, category_file, deprecated, options, env, template, outputname, module_map, aliases):
-    modstring = module
-    if modstring.startswith('_'):
-        modstring = module[1:]
-    modname = modstring
-    if module in deprecated:
-        modstring = to_bytes(modstring) + DEPRECATED
-
-    category_file.write(b"  %s - %s <%s_module>\n" % (to_bytes(modstring), to_bytes(rst_ify(module_map[module][1])), to_bytes(modname)))
-
-
-def process_category(category, categories, options, env, template, outputname):
-
-    # FIXME:
-    # We no longer conceptually deal with a mapping of category names to
-    # modules to file paths.  Instead we want several different records:
-    # (1) Mapping of module names to file paths (what's presently used
-    #     as categories['all']
-    # (2) Mapping of category names to lists of module names (what you'd
-    #     presently get from categories[category_name][subcategory_name].keys()
-    # (3) aliases (what's presently in categories['_aliases']
-    #
-    # list_modules() now returns those.  Need to refactor this function and
-    # main to work with them.
-
-    module_map = categories[category]
-    module_info = categories['all']
-
-    aliases = {}
-    if '_aliases' in categories:
-        aliases = categories['_aliases']
-
-    category_file_path = os.path.join(options.output_dir, "list_of_%s_modules.rst" % category)
-    category_file = open(category_file_path, "wb")
-    print("*** recording category %s in %s ***" % (category, category_file_path))
-
-    # start a new category file
-
-    category = category.replace("_", " ")
-    category = category.title()
-
-    modules = []
-    deprecated = []
-    for module in module_map.keys():
-        if isinstance(module_map[module], dict):
-            for mod in (m for m in module_map[module].keys() if m in module_info):
-                if mod.startswith("_"):
-                    deprecated.append(mod)
+        # don't show version added information if it's too old to be called out
+        added = 0
+        if doc['version_added'] == 'historical':
+            del doc['version_added']
         else:
-            if module not in module_info:
-                continue
-            if module.startswith("_"):
-                deprecated.append(module)
-        modules.append(module)
-
-    modules.sort(key=lambda k: k[1:] if k.startswith('_') else k)
-
-    category_header = b"%s Modules" % (to_bytes(category.title()))
-    underscores = b"`" * len(category_header)
-
-    category_file.write(b"""\
-%s
-%s
-
-.. toctree:: :maxdepth: 1
-
-""" % (category_header, underscores))
-    sections = []
-    for module in modules:
-        if module in module_map and isinstance(module_map[module], dict):
-            sections.append(module)
-            continue
+            added = doc['version_added']
+
+        # Strip old version_added for the module
+        if too_old(added):
+            del doc['version_added']
+
+        option_names = []
+
+        if 'options' in doc and doc['options']:
+            for (k, v) in iteritems(doc['options']):
+                # Error out if there's no description
+                if 'description' not in doc['options'][k]:
+                    raise AnsibleError("Missing required description for option %s in %s " % (k, module))
+
+                # Error out if required isn't a boolean (people have been putting
+                # information on when something is required in here.  Those need
+                # to go in the description instead).
+                required_value = doc['options'][k].get('required', False)
+                if not isinstance(required_value, bool):
+                    raise AnsibleError("Invalid required value '%s' for option '%s' in '%s' (must be truthy)" % (required_value, k, module))
+
+                # Strip old version_added information for options
+                if 'version_added' in doc['options'][k] and too_old(doc['options'][k]['version_added']):
+                    del doc['options'][k]['version_added']
+
+                # Make sure description is a list of lines for later formatting
+                if not isinstance(doc['options'][k]['description'], list):
+                    doc['options'][k]['description'] = [doc['options'][k]['description']]
+
+                option_names.append(k)
+
+        option_names.sort()
+
+        doc['option_keys'] = option_names
+        doc['filename'] = fname
+        doc['docuri'] = doc['module'].replace('_', '-')
+        doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d')
+        doc['ansible_version'] = ansible_version
+        doc['plainexamples'] = module_map[module]['examples']  # plain text
+        doc['metadata'] = module_map[module]['metadata']
+
+        if module_map[module]['returndocs']:
+            try:
+                doc['returndocs'] = yaml.safe_load(module_map[module]['returndocs'])
+            except:
+                print("could not load yaml: %s" % module_map[module]['returndocs'])
+                raise
         else:
-            print_modules(module, category_file, deprecated, options, env, template, outputname, module_info, aliases)
-
-    sections.sort()
-    for section in sections:
-        category_file.write(b"\n%s\n%s\n\n" % (to_bytes(section.replace("_", " ").title()), b'-' * len(section)))
-        category_file.write(b".. toctree:: :maxdepth: 1\n\n")
-
-        section_modules = list(module_map[section].keys())
-        section_modules.sort(key=lambda k: k[1:] if k.startswith('_') else k)
-        # for module in module_map[section]:
-        for module in (m for m in section_modules if m in module_info):
-            print_modules(module, category_file, deprecated, options, env, template, outputname, module_info, aliases)
-
-    category_file.write(b"""\n\n
-.. note::
-    - %s: This marks a module as deprecated, which means a module is kept for backwards compatibility but usage is discouraged.
-       The module documentation details page may explain more about this rationale.
-""" % DEPRECATED)
-    category_file.close()
+            doc['returndocs'] = None
+
+        text = templates['plugin'].render(doc)
+
+        write_data(text, output_dir, outputname, module)
+
+
+def process_categories(mod_info, categories, templates, output_dir, output_name):
+    for category in sorted(categories.keys()):
+        module_map = categories[category]
+        category_filename = output_name % category
+
+        print("*** recording category %s in %s ***" % (category, category_filename))
+
+        # start a new category file
+
+        category = category.replace("_", " ")
+        category = category.title()
+
+        subcategories = dict((k, v) for k, v in module_map.items() if k != '_modules')
+        template_data = {'title': category,
+                         'category': module_map,
+                         'subcategories': subcategories,
+                         'module_info': mod_info,
+                         }
+
+        text = templates['list_of_CATEGORY_modules'].render(template_data)
+        write_data(text, output_dir, category_filename)
+
+
+def process_support_levels(mod_info, templates, output_dir):
+    supported_by = {'Ansible Core Team': {'slug': 'core_supported',
+                                          'modules': [],
+                                          'output': 'core_maintained.rst'},
+                    'Ansible Network Team': {'slug': 'network_supported',
+                                             'modules': [],
+                                             'output': 'network_maintained.rst'},
+                    'Ansible Partners': {'slug': 'partner_supported',
+                                         'modules': [],
+                                         'output': 'partner_maintained.rst'},
+                    'Ansible Community': {'slug': 'community_supported',
+                                          'modules': [],
+                                          'output': 'community_maintained.rst'},
+                    }
+
+    # Separate the modules by support_level
+    for module, info in mod_info.items():
+        if info['metadata']['supported_by'] == 'core':
+            supported_by['Ansible Core Team']['modules'].append(module)
+        elif info['metadata']['supported_by'] == 'network':
+            supported_by['Ansible Network Team']['modules'].append(module)
+        elif info['metadata']['supported_by'] == 'certified':
+            supported_by['Ansible Partners']['modules'].append(module)
+        elif info['metadata']['supported_by'] == 'community':
+            supported_by['Ansible Community']['modules'].append(module)
+        else:
+            raise AnsibleError('Unknown supported_by value: %s' % info['metadata']['supported_by'])
 
-    # TODO: end a new category file
+    # Render the module lists
+    for maintainers, data in supported_by.items():
+        template_data = {'maintainers': maintainers,
+                         'modules': data['modules'],
+                         'slug': data['slug'],
+                         'module_info': mod_info,
+                         }
+        text = templates['support_list'].render(template_data)
+        write_data(text, output_dir, data['output'])
 
 
 def validate_options(options):
@@ -435,43 +463,34 @@ def main():
     (options, args) = p.parse_args()
     validate_options(options)
 
-    env, template, outputname = jinja2_environment(options.template_dir, options.type)
+    templates, outputname = jinja2_environment(options.template_dir, options.type)
 
     # Convert passed-in limit_to_modules to None or list of modules.
     if options.limit_to_modules is not None:
         options.limit_to_modules = [s.lower() for s in options.limit_to_modules.split(",")]
 
-    mod_info, categories, aliases = list_modules(options.module_dir, limit_to_modules=options.limit_to_modules)
-    categories['all'] = mod_info
-    categories['_aliases'] = aliases
-    category_names = [c for c in categories.keys() if not c.startswith('_')]
-    category_names.sort()
+    mod_info, categories = get_module_info(options.module_dir, limit_to_modules=options.limit_to_modules,
+                                           verbose=options.verbose)
+
+    categories['all'] = {'_modules': mod_info.keys()}
+
+    # Transform the data
+    if options.type == 'rst':
+        for record in mod_info.values():
+            record['doc']['short_description'] = rst_ify(record['doc']['short_description'])
 
     # Write master category list
-    category_list_path = os.path.join(options.output_dir, "modules_by_category.rst")
-    with open(category_list_path, "wb") as category_list_file:
-        category_list_file.write(b"Module Index\n")
-        category_list_file.write(b"============\n")
-        category_list_file.write(b"\n\n")
-        category_list_file.write(b".. toctree::\n")
-        category_list_file.write(b"   :maxdepth: 1\n\n")
-
-        for category in category_names:
-            category_list_file.write(b"   list_of_%s_modules\n" % to_bytes(category))
-
-    # Import all the docs into memory
-    module_map = mod_info.copy()
-
-    for modname in module_map:
-        result = process_module(modname, options, env, template, outputname, module_map, aliases)
-        if result == 'SKIPPED':
-            del categories['all'][modname]
-        else:
-            categories['all'][modname] = (categories['all'][modname], result)
+    category_list_text = templates['category_list'].render(categories=sorted(categories.keys()))
+    write_data(category_list_text, options.output_dir, 'modules_by_category.rst')
+
+    # Render all the individual module pages
+    process_modules(mod_info, templates, outputname, options.output_dir, options.ansible_version)
+
+    # Render all the categories for modules
+    process_categories(mod_info, categories, templates, options.output_dir, "list_of_%s_modules.rst")
 
-    # Render all the docs to rst via category pages
-    for category in category_names:
-        process_category(category, categories, options, env, template, outputname)
+    # Render all the categories for modules
+    process_support_levels(mod_info, templates, options.output_dir)
 
 
 if __name__ == '__main__':