From f2ca66fdc94faaff7ebe6b7962723bf06793df5f Mon Sep 17 00:00:00 2001 From: David Vallee Delisle Date: Thu, 22 Oct 2020 21:14:20 -0400 Subject: Adding --check-defaults to validator When troubleshooting, it's often interesting to see the deltas between the default or sample configs and the running configuration. The oslo_config.validator is a great tool to integrate this feature across all projects. It can also be easily captured by data collection tools like sosreports with the current deployment packages. Change-Id: I172d82f19a81282093b0d5f7ae4c1817801cd887 Signed-off-by: David Vallee Delisle --- oslo_config/tests/test_validator.py | 58 +++++++++++++++++++++--- oslo_config/validator.py | 90 ++++++++++++++++++++++++++++++++++++- 2 files changed, 141 insertions(+), 7 deletions(-) (limited to 'oslo_config') diff --git a/oslo_config/tests/test_validator.py b/oslo_config/tests/test_validator.py index de912ac..383340f 100644 --- a/oslo_config/tests/test_validator.py +++ b/oslo_config/tests/test_validator.py @@ -21,13 +21,51 @@ from oslo_config import fixture from oslo_config import validator -OPT_DATA = {'options': {'foo': {'opts': [{'name': 'opt'}]}, - 'bar': {'opts': [{'name': 'opt'}]}, - }, - 'deprecated_options': {'bar': [{'name': 'opt'}]}} +OPT_DATA = { + "options": { + "foo": { + "opts": [ + { + "name": "opt", + "default": 1 + } + ] + }, + "bar": { + "opts": [ + { + "name": "opt", + "default": 2 + }, + { + "name": "foo-bar", + "dest": "foo_bar", + "default": 2 + }, + { + "name": "bar-foo", + "dest": "bar_foo", + "default": 2 + } + ] + } + }, + "deprecated_options": { + "bar": [ + { + "name": "opt", + "default": 3 + } + ] + } +} VALID_CONF = """ [foo] -opt = value +opt = 1 +[bar] +opt = 3 +foo-bar = 3 +bar_foo = 3 """ DEPRECATED_CONF = """ [bar] @@ -106,5 +144,15 @@ class TestValidator(base.BaseTestCase): with mock.patch('builtins.open', m): self.assertEqual(0, validator._validate(self.conf)) + @mock.patch('oslo_config.validator.load_opt_data') + def test_check_defaults(self, mock_lod): + mock_lod.return_value = OPT_DATA + self.conf_fixture.config(opt_data='mocked.yaml', + input_file='mocked.conf', + check_defaults=True) + m = mock.mock_open(read_data=VALID_CONF) + with mock.patch('builtins.open', m): + self.assertEqual(0, validator._validate(self.conf)) + def test_invalid_options(self): self.assertRaises(RuntimeError, validator._validate, self.conf) diff --git a/oslo_config/validator.py b/oslo_config/validator.py index 725a868..938ee65 100644 --- a/oslo_config/validator.py +++ b/oslo_config/validator.py @@ -22,6 +22,7 @@ project then it returns those errors. """ import logging +import re import sys try: @@ -36,6 +37,11 @@ import yaml from oslo_config import cfg from oslo_config import generator +VALIDATE_DEFAULTS_EXCLUSIONS = [ + '.*_ur(i|l)', '.*connection', 'password', 'username', 'my_ip', + 'host(name)?', 'glance_api_servers', 'osapi_volume_listen', + 'osapi_compute_listen', +] _validator_opts = [ cfg.MultiStrOpt( @@ -50,6 +56,16 @@ _validator_opts = [ 'opt-data', help='Path to a YAML file containing definitions of options, as ' 'output by the config generator.'), + cfg.BoolOpt( + 'check-defaults', + default=False, + help='Report differences between the sample values and current ' + 'values.'), + cfg.ListOpt( + 'exclude-options', + default=VALIDATE_DEFAULTS_EXCLUSIONS, + help='Exclude options matching these patterns when comparing ' + 'the current and sample configurations.'), cfg.BoolOpt( 'fatal-warnings', default=False, @@ -86,6 +102,73 @@ def _validate_deprecated_opt(group, option, opt_data): return option in name_data +def _validate_defaults(sections, opt_data, conf): + """Compares the current and sample configuration and reports differences + + :param section: ConfigParser instance + :param opt_data: machine readable data from the generator instance + :param conf: ConfigOpts instance + :returns: boolean wether or not warnings were reported + """ + warnings = False + # Generating regex objects from ListOpt + exclusion_regexes = [] + for pattern in conf.exclude_options: + exclusion_regexes.append(re.compile(pattern)) + for group, opts in opt_data['options'].items(): + if group in conf.exclude_group: + continue + if group not in sections: + logging.warning( + 'Group %s from the sample config is not defined in ' + 'input-file', group) + continue + for opt in opts['opts']: + # We need to convert the defaults into a list to find + # intersections. defaults are only a list if they can + # be defined multiple times, but configparser only + # returns list + if not isinstance(opt['default'], list): + defaults = [str(opt['default'])] + else: + defaults = opt['default'] + + # Apparently, there's multiple naming conventions for + # options, 'name' is mostly with hyphens, and 'dest' + # is represented with underscores. + opt_names = set([opt['name'], opt.get('dest')]) + if not opt_names.intersection(sections[group]): + continue + try: + value = sections[group][opt['name']] + keyname = opt['name'] + except KeyError: + value = sections[group][opt.get('dest')] + keyname = opt.get('dest') + + if any(rex.fullmatch(keyname) for rex in exclusion_regexes): + logging.info( + '%s/%s Ignoring option because it is part of the excluded ' + 'patterns. This can be changed with the --exclude-options ' + 'argument', group, keyname) + continue + + if len(value) > 1: + logging.info( + '%s/%s defined %s times', group, keyname, len(value)) + if not opt['default']: + logging.warning( + '%s/%s sample value is empty but input-file has %s', + group, keyname, ", ".join(value)) + warnings = True + elif not frozenset(defaults).intersection(value): + logging.warning( + '%s/%s sample value %s is not in %s', + group, keyname, defaults, value) + warnings = True + return warnings + + def _validate_opt(group, option, opt_data): if group not in opt_data['options']: return False @@ -114,12 +197,14 @@ def _validate(conf): parser.parse() warnings = False errors = False + if conf.check_defaults: + warnings = _validate_defaults(sections, opt_data, conf) for section, options in sections.items(): if section in conf.exclude_group: continue for option in options: if _validate_deprecated_opt(section, option, opt_data): - logging.warn('Deprecated opt %s/%s found', section, option) + logging.warning('Deprecated opt %s/%s found', section, option) warnings = True elif not _validate_opt(section, option, opt_data): if section in KNOWN_BAD_GROUPS: @@ -129,7 +214,8 @@ def _validate(conf): 'cannot be validated properly.', option, section) continue - logging.error('%s/%s not found', section, option) + logging.error('%s/%s is not part of the sample config', + section, option) errors = True if errors or (warnings and conf.fatal_warnings): return 1 -- cgit v1.2.1