New metadata 1.0 (#22587)

Changes to the metadata format were approved here:
https://github.com/ansible/proposals/issues/54
* Update documentation to the new metadata format
* Changes to metadata-tool to account for new metadata
  * Add GPL license header
  * Add upgrade subcommand to upgrade metadata version
  * Change default metadata to the new format
  * Fix exclusion of non-modules from the metadata report
* Fix ansible-doc for new module metadata
* Exclude metadata version from ansible-doc output
* Fix website docs generation for the new metadata
* Update metadata schema in valiate-modules test
* Update the metadata in all modules to the new version

2 files changed, 134 insertions, 27 deletions

diff --git a/hacking/metadata-tool.py b/hacking/metadata-tool.py
index 795511f7f9..b37fe445af 100755
--- a/hacking/metadata-tool.py
+++ b/hacking/metadata-tool.py
@@ -1,4 +1,24 @@
 #!/usr/bin/env python
+# (c) 2016-2017, Toshio Kuratomi <tkuratomi@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
 
 import ast
 import csv
@@ -19,7 +39,7 @@ NONMODULE_PY_FILES = frozenset(('async_wrapper.py',))
 NONMODULE_MODULE_NAMES = frozenset(os.path.splitext(p)[0] for p in NONMODULE_PY_FILES)
 
 # Default metadata
-DEFAULT_METADATA = {'version': '1.0', 'status': ['preview'], 'supported_by':'community'}
+DEFAULT_METADATA = {'metadata_version': '1.0', 'status': ['preview'], 'supported_by':'community'}
 
 
 class ParseError(Exception):
@@ -34,9 +54,10 @@ class MissingModuleError(Exception):
 
 def usage():
     print("""Usage:
-      metadata-tester.py report [--version X]
-      metadata-tester.py add [--version X] [--overwrite] CSVFILE
-      metadata-tester.py add-default [--version X] [--overwrite]""")
+      metadata-tool.py report [--version X]
+      metadata-tool.py add [--version X] [--overwrite] CSVFILE
+      metadata-tool.py add-default [--version X] [--overwrite]
+      medatada-tool.py upgrade [--version X]""")
     sys.exit(1)
 
 
@@ -295,7 +316,7 @@ def parse_assigned_metadata_initial(csvfile):
         :2: Extras (x if so)
         :3: Category
         :4: Supported/SLA
-        :5: Committer
+        :5: Curated
         :6: Stable
         :7: Deprecated
         :8: Notes
@@ -310,7 +331,7 @@ def parse_assigned_metadata_initial(csvfile):
             if record[12] == 'core':
                 supported_by = 'core'
             elif record[12] == 'curated':
-                supported_by = 'committer'
+                supported_by = 'curated'
             elif record[12] == 'community':
                 supported_by = 'community'
             else:
@@ -326,7 +347,7 @@ def parse_assigned_metadata_initial(csvfile):
             if not status:
                 status.extend(DEFAULT_METADATA['status'])
 
-            yield (module, {'version': DEFAULT_METADATA['version'], 'supported_by': supported_by, 'status': status})
+            yield (module, {'version': DEFAULT_METADATA['metadata_version'], 'supported_by': supported_by, 'status': status})
 
 
 def parse_assigned_metadata(csvfile):
@@ -334,12 +355,11 @@ def parse_assigned_metadata(csvfile):
     Fields:
         :0: Module name
         :1: supported_by  string.  One of the valid support fields
-            core, community, unmaintained, committer
+            core, community, curated
         :2: stableinterface
         :3: preview
         :4: deprecated
         :5: removed
-        :6: tested
 
         https://github.com/ansible/proposals/issues/30
     """
@@ -355,12 +375,10 @@ def parse_assigned_metadata(csvfile):
                 status.append('deprecated')
             if record[5]:
                 status.append('removed')
-            if record[6]:
-                status.append('tested')
             if not status or record[3]:
                 status.append('preview')
 
-            yield (module, {'version': '1.0', 'supported_by': supported_by, 'status': status})
+            yield (module, {'metadata_version': '1.0', 'supported_by': supported_by, 'status': status})
 
 
 def write_metadata(filename, new_metadata, version=None, overwrite=False):
@@ -388,7 +406,8 @@ def write_metadata(filename, new_metadata, version=None, overwrite=False):
 
         module_data = insert_metadata(module_data, new_metadata, start_line, targets=('ANSIBLE_METADATA',))
 
-    elif overwrite or (version is not None and ('version' not in current_metadata or StrictVersion(current_metadata['version']) < StrictVersion(version))):
+    elif overwrite or (version is not None and ('metadata_version' not in current_metadata or
+                                                StrictVersion(current_metadata['metadata_version']) < StrictVersion(version))):
         # Current metadata that we do not want.  Remove the current
         # metadata and put the new version in its place
         module_data = remove_metadata(module_data, start_line, start_col, end_line, end_col)
@@ -404,7 +423,13 @@ def write_metadata(filename, new_metadata, version=None, overwrite=False):
 
 
 def return_metadata(plugins):
+    """Get the metadata for all modules
 
+    Handle duplicate module names
+
+    :arg plugins: List of plugins to look for
+    :returns: Mapping of plugin name to metadata dictionary
+    """
     metadata = {}
     for name, filename in plugins:
         # There may be several files for a module (if it is written in another
@@ -416,6 +441,7 @@ def return_metadata(plugins):
             metadata[name] = extract_metadata(module_data)[0]
     return metadata
 
+
 def metadata_summary(plugins, version=None):
     """Compile information about the metadata status for a list of modules
 
@@ -431,8 +457,8 @@ def metadata_summary(plugins, version=None):
     has_metadata = {}
     supported_by = defaultdict(set)
     status = defaultdict(set)
+    requested_version = StrictVersion(version)
 
-    plugins = list(plugins)
     all_mods_metadata = return_metadata(plugins)
     for name, filename in plugins:
         # Does the module have metadata?
@@ -440,7 +466,7 @@ def metadata_summary(plugins, version=None):
             metadata = all_mods_metadata[name]
             if metadata is None:
                 no_metadata[name] = filename
-            elif version is not None and ('version' not in metadata or StrictVersion(metadata['version']) < StrictVersion(version)):
+            elif version is not None and ('metadata_version' not in metadata or StrictVersion(metadata['metadata_version']) < requested_version):
                 no_metadata[name] = filename
             else:
                 has_metadata[name] = filename
@@ -458,6 +484,34 @@ def metadata_summary(plugins, version=None):
     return list(no_metadata.values()), list(has_metadata.values()), supported_by, status
 
 #
+# Filters to convert between metadata versions
+#
+
+def convert_metadata_pre_1_0_to_1_0(metadata):
+    """
+    Convert pre-1.0 to 1.0 metadata format
+
+    :arg metadata: The old metadata
+    :returns: The new metadata
+
+    Changes from pre-1.0 to 1.0:
+    * ``version`` field renamed to ``metadata_version``
+    * ``supported_by`` field value ``unmaintained`` has been removed (change to
+      ``community`` and let an external list track whether a module is unmaintained)
+    * ``supported_by`` field value ``committer`` has been renamed to ``curated``
+    """
+    new_metadata = {'metadata_version': '1.0',
+                    'supported_by': metadata['supported_by'],
+                    'status': metadata['status']
+                    }
+    if new_metadata['supported_by'] == 'unmaintained':
+        new_metadata['supported_by'] = 'community'
+    elif new_metadata['supported_by'] == 'committer':
+        new_metadata['supported_by'] = 'curated'
+
+    return new_metadata
+
+#
 # Subcommands
 #
 
@@ -515,6 +569,62 @@ def add_default(version=None, overwrite=False):
     return 0
 
 
+def upgrade_metadata(version=None):
+    """Implement the subcommand to upgrade the default metadata in modules.
+
+    :kwarg version: If given, the version of the metadata to upgrade to.  If
+        not given, upgrade to the latest format version.
+    """
+    if version is None:
+        # Number larger than any of the defined metadata formats.
+        version = 9999999
+    requested_version = StrictVersion(version)
+
+    # List all plugins
+    plugins = module_loader.all(path_only=True)
+    plugins = ((os.path.splitext((os.path.basename(p)))[0], p) for p in plugins)
+    plugins = (p for p in plugins if p[0] not in NONMODULE_MODULE_NAMES)
+
+    processed = set()
+    diagnostic_messages = []
+    for name, filename in (info for info in plugins if info[0] not in processed):
+        # For each plugin, read the existing metadata
+        with open(filename, 'rb') as f:
+            module_data = f.read()
+        metadata = extract_metadata(module_data)[0]
+
+        # If the metadata isn't the requested version, convert it to the new
+        # version
+        if 'metadata_version' not in metadata or metadata['metadata_version'] != version:
+            #
+            # With each iteration of metadata, add a new conditional to
+            # upgrade from the previous version
+            #
+
+            if 'metadata_version' not in metadata:
+                # First version, pre-1.0 final metadata
+                metadata = convert_metadata_pre_1_0_to_1_0(metadata)
+
+            if metadata['metadata_version'] == '1.0' and StrictVersion('1.0') < requested_version:
+                # 1.0 version => XXX.  We don't yet have anything beyond 1.0
+                # so there's nothing here
+                pass
+
+            # Replace the existing metadata with the new format
+            try:
+                write_metadata(filename, metadata, version, overwrite=True)
+            except ParseError as e:
+                diagnostic_messages.append(e.args[0])
+                continue
+
+        processed.add(name)
+
+    if diagnostic_messages:
+        pprint(diagnostic_messages)
+
+    return 0
+
+
 def report(version=None):
     """Implement the report subcommand
 
@@ -525,9 +635,8 @@ def report(version=None):
     """
     # List of all plugins
     plugins = module_loader.all(path_only=True)
-    plugins = list(plugins)
     plugins = ((os.path.splitext((os.path.basename(p)))[0], p) for p in plugins)
-    plugins = (p for p in plugins if p[0] != NONMODULE_MODULE_NAMES)
+    plugins = (p for p in plugins if p[0] not in NONMODULE_MODULE_NAMES)
     plugins = list(plugins)
 
     no_metadata, has_metadata, support, status = metadata_summary(plugins, version=version)
@@ -542,8 +651,8 @@ def report(version=None):
 
     print('== Supported by core ==')
     pprint(sorted(support['core']))
-    print('== Supported by committers ==')
-    pprint(sorted(support['committer']))
+    print('== Supported by value curated ==')
+    pprint(sorted(support['curated']))
     print('== Supported by community ==')
     pprint(sorted(support['community']))
     print('')
@@ -560,12 +669,10 @@ def report(version=None):
 
     print('== Summary ==')
     print('No Metadata: {0}             Has Metadata: {1}'.format(len(no_metadata), len(has_metadata)))
-    print('Supported by core: {0}      Supported by community: {1}    Supported by committer: {2}'.format(len(support['core']), len(support['community']),
-                                                                                                          len(support['committer'])))
+    print('Supported by core: {0}      Supported by community: {1}    Supported by value curated: {2}'.format(len(support['core']),
+                len(support['community']), len(support['curated'])))
     print('Status StableInterface: {0} Status Preview: {1}            Status Deprecated: {2}      Status Removed: {3}'.format(len(status['stableinterface']),
-                                                                                                                              len(status['preview']),
-                                                                                                                              len(status['deprecated']),
-                                                                                                                              len(status['removed'])))
+                len(status['preview']), len(status['deprecated']), len(status['removed'])))
 
     return 0
 
@@ -573,13 +680,13 @@ def report(version=None):
 if __name__ == '__main__':
     action, args = parse_args(sys.argv[1:])
 
-    ### TODO: Implement upgrade metadata and upgrade metadata from csvfile
     if action == 'report':
         rc = report(version=args['version'])
     elif action == 'add':
         rc = add_from_csv(args['csvfile'], version=args['version'], overwrite=args['overwrite'])
     elif action == 'add-default':
         rc = add_default(version=args['version'], overwrite=args['overwrite'])
+    elif action == 'upgrade':
+        rc = upgrade_metadata(version=args['version'])
 
     sys.exit(rc)
-
diff --git a/hacking/templates/rst.j2 b/hacking/templates/rst.j2
index f401fda6fa..1f9f692312 100644
--- a/hacking/templates/rst.j2
+++ b/hacking/templates/rst.j2
@@ -207,7 +207,7 @@ Notes
 {% endif %}
 
 {% if not deprecated %}
-{% set support = { 'core': 'This module is maintained by those with core commit privileges', 'committer': 'This module is supported mainly by the community and is curated by core committers.', 'community': 'This module is community maintained without core committer oversight.'} %}
+{% set support = { 'core': 'This module is maintained by those with core commit privileges', 'curated': 'This module is supported mainly by the community and is curated by core committers.', 'community': 'This module is community maintained without core committer oversight.'} %}
 {% set module_states = { 'preview': 'it is not guaranteed to have a backwards compatible interface', 'stableinterface': 'the maintainers for this module guarantee that no backward incompatible interface changes will be made'} %}
 
 {% if metadata %}