Machine Readable Sample Config

Adds the ability for the sample config generator to output the
config data in the machine readable formats yaml and json.

bp machine-readable-sample-config

Change-Id: I236918f0c1da27358aace66914aae5c34afef301
Co-Authored-By: Stephen Finucane <sfinucan@redhat.com>

3 files changed, 396 insertions, 10 deletions

diff --git a/oslo_config/generator.py b/oslo_config/generator.py
index d38ba7f..a509d75 100644
--- a/oslo_config/generator.py
+++ b/oslo_config/generator.py
@@ -24,13 +24,16 @@ Tool for generating a sample configuration file. See
 """
 
 import collections
+import copy
 import logging
 import operator
 import sys
 import textwrap
 
+import json
 import pkg_resources
 import six
+import yaml
 
 
 from oslo_config import cfg
@@ -61,6 +64,15 @@ _generator_opts = [
         default=False,
         help='Only output summaries of help text to config files. Retain '
         'longer help text for Sphinx documents.'),
+    cfg.StrOpt(
+        'format',
+        help='Desired format for the output. "ini" is the only one which can '
+             'be used directly with oslo.config. "json" and "yaml" are '
+             'intended for third-party tools that want to write config files '
+             'based on the sample config data.',
+        default='ini',
+        choices=['ini', 'json', 'yaml'],
+        dest='format_'),
 ]
 
 
@@ -491,6 +503,108 @@ def _get_groups(conf_ns):
     return groups
 
 
+def _build_entry(opt, group, namespace, conf):
+    """Return a dict representing the passed in opt
+
+    The dict will contain all public attributes of opt, as well as additional
+    entries for namespace, choices, min, and max.  Any DeprecatedOpts
+    contained in the deprecated_opts member will be converted to a dict with
+    the format: {'group': <deprecated group>, 'name': <deprecated name>}
+
+    :param opt: The Opt object to represent as a dict.
+    :param group: The name of the group containing opt.
+    :param namespace: The name of the namespace containing opt.
+    :param conf: The ConfigOpts object containing the options for the
+                 generator tool
+    """
+    entry = {key: value for key, value in opt.__dict__.items()
+             if not key.startswith('_')}
+    entry['namespace'] = namespace
+    # In some types, choices is explicitly set to None.  Force it to [] so it
+    # is always an iterable type.
+    entry['choices'] = getattr(entry['type'], 'choices', []) or []
+    entry['min'] = getattr(entry['type'], 'min', None)
+    entry['max'] = getattr(entry['type'], 'max', None)
+    entry['type'] = _format_type_name(entry['type'])
+    deprecated_opts = []
+    for deprecated_opt in entry['deprecated_opts']:
+        # NOTE(bnemec): opt names with a - are not valid in a config file,
+        # but it is possible to add a DeprecatedOpt with a - name.  We
+        # want to ignore those as they won't work anyway.
+        if not deprecated_opt.name or '-' not in deprecated_opt.name:
+            deprecated_opts.append(
+                {'group': deprecated_opt.group or group,
+                 'name': deprecated_opt.name or entry['name'],
+                 })
+    entry['deprecated_opts'] = deprecated_opts
+    return entry
+
+
+def _generate_machine_readable_data(groups, conf):
+    """Create data structure for machine readable sample config
+
+    Returns a dictionary with the top-level keys 'options',
+    'deprecated_options', and 'generator_options'.
+
+    'options' contains a dict mapping group names to a list of options in
+    that group.  Each option is represented by the result of a call to
+    _build_entry.  Only non-deprecated options are included in this list.
+
+    'deprecated_options' contains a dict mapping groups names to a list of
+    opts from that group which were deprecated.
+
+    'generator_options' is a dict mapping the options for the sample config
+    generator itself to their values.
+
+    :param groups: A dict of groups as returned by _get_groups.
+    :param conf: The ConfigOpts object containing the options for the
+                 generator tool
+    """
+    output_data = {'options': {},
+                   'deprecated_options': {},
+                   'generator_options': {}}
+    # See _get_groups for details on the structure of group_data
+    for group_name, group_data in groups.items():
+        output_data['options'][group_name] = {'opts': [], 'help': ''}
+        for namespace in group_data['namespaces']:
+            for opt in namespace[1]:
+                if group_data['object']:
+                    output_group = output_data['options'][group_name]
+                    output_group['help'] = group_data['object'].help
+                entry = _build_entry(opt, group_name, namespace[0], conf)
+                output_data['options'][group_name]['opts'].append(entry)
+                # Need copies of the opts because we modify them
+                for deprecated_opt in copy.deepcopy(entry['deprecated_opts']):
+                    group = deprecated_opt.pop('group')
+                    deprecated_options = output_data['deprecated_options']
+                    deprecated_options.setdefault(group, [])
+                    deprecated_opt['replacement_name'] = entry['name']
+                    deprecated_opt['replacement_group'] = group_name
+                    deprecated_options[group].append(deprecated_opt)
+    output_data['generator_options'] = conf
+    return output_data
+
+
+def _output_machine_readable(groups, output_file, conf):
+    """Write a machine readable sample config file
+
+    Take the data returned by _generate_machine_readable_data and write it in
+    the format specified by the format_ attribute of conf.
+
+    :param groups: A dict of groups as returned by _get_groups.
+    :param output_file: A file-like object to which the data should be written.
+    :param conf: The ConfigOpts object containing the options for the
+                 generator tool
+    """
+    output_data = _generate_machine_readable_data(groups, conf)
+    if conf.format_ == 'yaml':
+        output_file.write(yaml.safe_dump(output_data,
+                                         default_flow_style=False))
+    else:
+        output_file.write(json.dumps(output_data, sort_keys=True))
+    output_file.write('\n')
+
+
 def generate(conf):
     """Generate a sample config file.
 
@@ -504,21 +618,26 @@ def generate(conf):
     output_file = (open(conf.output_file, 'w')
                    if conf.output_file else sys.stdout)
 
-    formatter = _OptFormatter(output_file=output_file,
-                              wrap_width=conf.wrap_width)
-
     groups = _get_groups(_list_opts(conf.namespace))
 
-    # Output the "DEFAULT" section as the very first section
-    _output_opts(formatter, 'DEFAULT', groups.pop('DEFAULT'), conf.minimal,
-                 conf.summarize)
+    if conf.format_ == 'ini':
+        formatter = _OptFormatter(output_file=output_file,
+                                  wrap_width=conf.wrap_width)
 
-    # output all other config sections with groups in alphabetical order
-    for group, group_data in sorted(groups.items()):
-        formatter.write('\n\n')
-        _output_opts(formatter, group, group_data, conf.minimal,
+        # Output the "DEFAULT" section as the very first section
+        _output_opts(formatter, 'DEFAULT', groups.pop('DEFAULT'), conf.minimal,
                      conf.summarize)
 
+        # output all other config sections with groups in alphabetical order
+        for group, group_data in sorted(groups.items()):
+            formatter.write('\n\n')
+            _output_opts(formatter, group, group_data, conf.minimal,
+                         conf.summarize)
+    else:
+        _output_machine_readable(groups,
+                                 output_file=output_file,
+                                 conf=conf)
+
 
 def main(args=None):
     """The main function of oslo-config-generator."""
diff --git a/oslo_config/tests/test_generator.py b/oslo_config/tests/test_generator.py
index 3406b63..8b075e9 100644
--- a/oslo_config/tests/test_generator.py
+++ b/oslo_config/tests/test_generator.py
@@ -954,6 +954,265 @@ class GeneratorTestCase(base.BaseTestCase):
             self.assertFalse(mock_log.warning.called)
 
 
+GENERATOR_OPTS = {'format_': 'yaml',
+                  'minimal': False,
+                  'namespace': ['test'],
+                  'output_file': None,
+                  'summarize': False,
+                  'wrap_width': 70}
+
+
+class MachineReadableGeneratorTestCase(base.BaseTestCase):
+    all_opts = GeneratorTestCase.opts
+    all_groups = GeneratorTestCase.groups
+    content_scenarios = [
+        ('single_namespace',
+         dict(opts=[('test', [(None, [all_opts['foo']])])],
+              expected={'deprecated_options': {},
+                        'generator_options': GENERATOR_OPTS,
+                        'options': {
+                            'DEFAULT': {
+                                'help': '',
+                                'opts': [{'advanced': False,
+                                          'choices': [],
+                                          'default': None,
+                                          'deprecated_for_removal': False,
+                                          'deprecated_opts': [],
+                                          'deprecated_reason': None,
+                                          'deprecated_since': None,
+                                          'dest': 'foo',
+                                          'help': 'foo option',
+                                          'max': None,
+                                          'metavar': None,
+                                          'min': None,
+                                          'mutable': False,
+                                          'name': 'foo',
+                                          'namespace': 'test',
+                                          'positional': False,
+                                          'required': False,
+                                          'sample_default': None,
+                                          'secret': False,
+                                          'short': None,
+                                          'type': 'string value'}]}}})),
+        ('long_help',
+         dict(opts=[('test', [(None, [all_opts['long_help']])])],
+              expected={'deprecated_options': {},
+                        'generator_options': GENERATOR_OPTS,
+                        'options': {
+                            'DEFAULT': {
+                                'help': '',
+                                'opts': [{'advanced': False,
+                                          'choices': [],
+                                          'default': None,
+                                          'deprecated_for_removal': False,
+                                          'deprecated_opts': [],
+                                          'deprecated_reason': None,
+                                          'deprecated_since': None,
+                                          'dest': 'long_help',
+                                          'help': all_opts['long_help'].help,
+                                          'max': None,
+                                          'metavar': None,
+                                          'min': None,
+                                          'mutable': False,
+                                          'name': 'long_help',
+                                          'namespace': 'test',
+                                          'positional': False,
+                                          'required': False,
+                                          'sample_default': None,
+                                          'secret': False,
+                                          'short': None,
+                                          'type': 'string value'}]}}})),
+        ('long_help_pre',
+         dict(opts=[('test', [(None, [all_opts['long_help_pre']])])],
+              expected={'deprecated_options': {},
+                        'generator_options': GENERATOR_OPTS,
+                        'options': {
+                            'DEFAULT': {
+                                'help': '',
+                                'opts': [{'advanced': False,
+                                          'choices': [],
+                                          'default': None,
+                                          'deprecated_for_removal': False,
+                                          'deprecated_opts': [],
+                                          'deprecated_reason': None,
+                                          'deprecated_since': None,
+                                          'dest': 'long_help_pre',
+                                          'help':
+                                              all_opts['long_help_pre'].help,
+                                          'max': None,
+                                          'metavar': None,
+                                          'min': None,
+                                          'mutable': False,
+                                          'name': 'long_help_pre',
+                                          'namespace': 'test',
+                                          'positional': False,
+                                          'required': False,
+                                          'sample_default': None,
+                                          'secret': False,
+                                          'short': None,
+                                          'type': 'string value'}]}}})),
+        ('opt_with_DeprecatedOpt',
+         dict(opts=[('test', [(None, [all_opts['opt_with_DeprecatedOpt']])])],
+              expected={
+                  'deprecated_options': {
+                      'deprecated': [{'name': 'foo_bar',
+                                      'replacement_group': 'DEFAULT',
+                                      'replacement_name': 'foo-bar'}]},
+                  'generator_options': GENERATOR_OPTS,
+                  'options': {
+                      'DEFAULT': {
+                          'help': '',
+                          'opts': [{
+                              'advanced': False,
+                              'choices': [],
+                              'default': None,
+                              'deprecated_for_removal': False,
+                              'deprecated_opts': [{'group': 'deprecated',
+                                                   'name': 'foo_bar'}],
+                              'deprecated_reason': None,
+                              'deprecated_since': None,
+                              'dest': 'foo_bar',
+                              'help':
+                                  all_opts['opt_with_DeprecatedOpt'].help,
+                              'max': None,
+                              'metavar': None,
+                              'min': None,
+                              'mutable': False,
+                              'name': 'foo-bar',
+                              'namespace': 'test',
+                              'positional': False,
+                              'required': False,
+                              'sample_default': None,
+                              'secret': False,
+                              'short': None,
+                              'type': 'boolean value'}]}}})),
+        ('choices_opt',
+         dict(opts=[('test', [(None, [all_opts['choices_opt']])])],
+              expected={'deprecated_options': {},
+                        'generator_options': GENERATOR_OPTS,
+                        'options': {
+                            'DEFAULT': {
+                                'help': '',
+                                'opts': [{'advanced': False,
+                                          'choices': (None, '', 'a', 'b', 'c'),
+                                          'default': 'a',
+                                          'deprecated_for_removal': False,
+                                          'deprecated_opts': [],
+                                          'deprecated_reason': None,
+                                          'deprecated_since': None,
+                                          'dest': 'choices_opt',
+                                          'help': all_opts['choices_opt'].help,
+                                          'max': None,
+                                          'metavar': None,
+                                          'min': None,
+                                          'mutable': False,
+                                          'name': 'choices_opt',
+                                          'namespace': 'test',
+                                          'positional': False,
+                                          'required': False,
+                                          'sample_default': None,
+                                          'secret': False,
+                                          'short': None,
+                                          'type': 'string value'}]}}})),
+        ('int_opt',
+         dict(opts=[('test', [(None, [all_opts['int_opt']])])],
+              expected={'deprecated_options': {},
+                        'generator_options': GENERATOR_OPTS,
+                        'options': {
+                            'DEFAULT': {
+                                'help': '',
+                                'opts': [{'advanced': False,
+                                          'choices': [],
+                                          'default': 10,
+                                          'deprecated_for_removal': False,
+                                          'deprecated_opts': [],
+                                          'deprecated_reason': None,
+                                          'deprecated_since': None,
+                                          'dest': 'int_opt',
+                                          'help': all_opts['int_opt'].help,
+                                          'max': 20,
+                                          'metavar': None,
+                                          'min': 1,
+                                          'mutable': False,
+                                          'name': 'int_opt',
+                                          'namespace': 'test',
+                                          'positional': False,
+                                          'required': False,
+                                          'sample_default': None,
+                                          'secret': False,
+                                          'short': None,
+                                          'type': 'integer value'}]}}})),
+        ('group_help',
+         dict(opts=[('test', [(all_groups['group1'], [all_opts['foo']])])],
+              expected={'deprecated_options': {},
+                        'generator_options': GENERATOR_OPTS,
+                        'options': {
+                            'DEFAULT': {
+                                'help': '',
+                                'opts': []
+                            },
+                            'group1': {
+                                'help': all_groups['group1'].help,
+                                'opts': [{'advanced': False,
+                                          'choices': [],
+                                          'default': None,
+                                          'deprecated_for_removal': False,
+                                          'deprecated_opts': [],
+                                          'deprecated_reason': None,
+                                          'deprecated_since': None,
+                                          'dest': 'foo',
+                                          'help': all_opts['foo'].help,
+                                          'max': None,
+                                          'metavar': None,
+                                          'min': None,
+                                          'mutable': False,
+                                          'name': 'foo',
+                                          'namespace': 'test',
+                                          'positional': False,
+                                          'required': False,
+                                          'sample_default': None,
+                                          'secret': False,
+                                          'short': None,
+                                          'type': 'string value'}]}}})),
+        ]
+
+    def setUp(self):
+        super(MachineReadableGeneratorTestCase, self).setUp()
+
+        self.conf = cfg.ConfigOpts()
+        self.config_fixture = config_fixture.Config(self.conf)
+        self.config = self.config_fixture.config
+        self.useFixture(self.config_fixture)
+
+    @classmethod
+    def generate_scenarios(cls):
+        cls.scenarios = testscenarios.multiply_scenarios(
+            cls.content_scenarios)
+
+    @mock.patch.object(generator, '_get_raw_opts_loaders')
+    def test_generate(self, raw_opts_loader):
+        generator.register_cli_opts(self.conf)
+        namespaces = [i[0] for i in self.opts]
+        self.config(namespace=namespaces, format_='yaml')
+
+        # We have a static data structure matching what should be
+        # returned by _list_opts() but we're mocking out a lower level
+        # function that needs to return a namespace and a callable to
+        # return options from that namespace. We have to pass opts to
+        # the lambda to cache a reference to the name because the list
+        # comprehension changes the thing pointed to by the name each
+        # time through the loop.
+        raw_opts_loader.return_value = [
+            (ns, lambda opts=opts: opts)
+            for ns, opts in self.opts
+        ]
+        test_groups = generator._get_groups(
+            generator._list_opts(self.conf.namespace))
+        self.assertEqual(self.expected,
+                         generator._generate_machine_readable_data(test_groups,
+                                                                   self.conf))
+
+
 class IgnoreDoublesTestCase(base.BaseTestCase):
 
     opts = [cfg.StrOpt('foo', help='foo option'),
@@ -1377,3 +1636,4 @@ class AdvancedOptionsTestCase(base.BaseTestCase):
 
 
 GeneratorTestCase.generate_scenarios()
+MachineReadableGeneratorTestCase.generate_scenarios()
diff --git a/releasenotes/notes/machine-readable-sample-config-e8f8ba43ababcf99.yaml b/releasenotes/notes/machine-readable-sample-config-e8f8ba43ababcf99.yaml
new file mode 100644
index 0000000..179027b
--- /dev/null
+++ b/releasenotes/notes/machine-readable-sample-config-e8f8ba43ababcf99.yaml
@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    The sample config generator can now generate machine-readable formats of
+    the sample config data.  This can be consumed by deployment tools to
+    automatically generate configuration files that contain all of the
+    information in the traditional sample configs.