diff options
25 files changed, 524 insertions, 46 deletions
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..9d94556 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,35 @@ +# We from the Oslo project decided to pin repos based on the +# commit hash instead of the version tag to prevend arbitrary +# code from running in developer's machines. To update to a +# newer version, run `pre-commit autoupdate` and then replace +# the newer versions with their commit hash. + +default_language_version: + python: python3 + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: ebc15addedad713c86ef18ae9632c88e187dd0af # v3.1.0 + hooks: + - id: trailing-whitespace + # Replaces or checks mixed line ending + - id: mixed-line-ending + args: ['--fix', 'lf'] + exclude: '.*\.(svg)$' + # Forbid files which have a UTF-8 byte-order marker + - id: check-byte-order-marker + # Checks that non-binary executables have a proper shebang + - id: check-executables-have-shebangs + # Check for files that contain merge conflict strings. + - id: check-merge-conflict + # Check for debugger imports and py37+ breakpoint() + # calls in python source + - id: debug-statements + - id: check-yaml + files: .*\.(yaml|yml)$ + - repo: https://gitlab.com/pycqa/flake8 + rev: 181bb46098dddf7e2d45319ea654b4b4d58c2840 # 3.8.3 + hooks: + - id: flake8 + additional_dependencies: + - hacking>=3.0.1,<3.1.0 diff --git a/.zuul.d/project.yaml b/.zuul.yaml index 2c16442..8425030 100644 --- a/.zuul.d/project.yaml +++ b/.zuul.yaml @@ -2,8 +2,7 @@ templates: - check-requirements - lib-forward-testing-python3 - - openstack-lower-constraints-jobs - - openstack-python3-victoria-jobs + - openstack-python3-wallaby-jobs - periodic-stable-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 diff --git a/doc/source/cli/validator.rst b/doc/source/cli/validator.rst index d829b8d..23ae424 100644 --- a/doc/source/cli/validator.rst +++ b/doc/source/cli/validator.rst @@ -32,7 +32,7 @@ Here's an example of using the validator on Nova as installed by Devstack (with the option [foo]/bar added to demonstrate a failure):: $ oslo-config-validator --config-file /opt/stack/nova/etc/nova/nova-config-generator.conf --input-file /etc/nova/nova.conf - ERROR:root:foo/bar not found + ERROR:root:foo/bar is not part of the sample config INFO:root:Ignoring missing option "project_domain_name" from group "keystone_authtoken" because the group is known to have incomplete sample config data and thus cannot be validated properly. INFO:root:Ignoring missing option "project_name" from group "keystone_authtoken" because the group is known to have incomplete sample config data and thus cannot be validated properly. INFO:root:Ignoring missing option "user_domain_name" from group "keystone_authtoken" because the group is known to have incomplete sample config data and thus cannot be validated properly. @@ -58,7 +58,7 @@ a sample config file ``config-data.yaml`` created by the config generator (with the option [foo]/bar added to demonstrate a failure):: $ oslo-config-validator --opt-data config-data.yaml --input-file /etc/nova/nova.conf - ERROR:root:foo/bar not found + ERROR:root:foo/bar is not part of the sample config INFO:root:Ignoring missing option "project_domain_name" from group "keystone_authtoken" because the group is known to have incomplete sample config data and thus cannot be validated properly. INFO:root:Ignoring missing option "project_name" from group "keystone_authtoken" because the group is known to have incomplete sample config data and thus cannot be validated properly. INFO:root:Ignoring missing option "user_domain_name" from group "keystone_authtoken" because the group is known to have incomplete sample config data and thus cannot be validated properly. @@ -66,6 +66,158 @@ the option [foo]/bar added to demonstrate a failure):: INFO:root:Ignoring missing option "username" from group "keystone_authtoken" because the group is known to have incomplete sample config data and thus cannot be validated properly. INFO:root:Ignoring missing option "auth_url" from group "keystone_authtoken" because the group is known to have incomplete sample config data and thus cannot be validated properly. +Comparing configuration with the default configuration +------------------------------------------------------ + +.. note:: For most accurate results, the validation should be done using the + same version of the code as the system whose config file is being + validated. + +Comparing the default configuration with the current configuration can help +operators with troubleshooting issues. Since the generator config is not always +available in production environment, we can pass the ``--namespace`` arguments. +In addition to the ``--namespace``, we need to pass a ``--input-file`` as well +as the ``--check-defaults``. + +Some options are ignored by default but this behavior can be overridden with +the ``--exclude-options`` list argument. + +Here's an example of using the validator on Nova:: + + $ oslo-config-validator --input-file /etc/nova/nova.conf \ + --check-defaults \ + --namespace nova.conf \ + --namespace oslo.log \ + --namespace oslo.messaging \ + --namespace oslo.policy \ + --namespace oslo.privsep \ + --namespace oslo.service.periodic_task \ + --namespace oslo.service.service \ + --namespace oslo.db \ + --namespace oslo.db.concurrency \ + --namespace oslo.middleware \ + --namespace oslo.concurrency \ + --namespace keystonemiddleware.auth_token \ + --namespace osprofiler + INFO:keyring.backend:Loading Gnome + INFO:keyring.backend:Loading Google + INFO:keyring.backend:Loading Windows (alt) + INFO:keyring.backend:Loading file + INFO:keyring.backend:Loading keyczar + INFO:keyring.backend:Loading multi + INFO:keyring.backend:Loading pyfs + WARNING:root:DEFAULT/compute_driver sample value is empty but input-file has libvirt.LibvirtDriver + WARNING:root:DEFAULT/allow_resize_to_same_host sample value is empty but input-file has True + WARNING:root:DEFAULT/default_ephemeral_format sample value is empty but input-file has ext4 + WARNING:root:DEFAULT/pointer_model sample value ['usbtablet'] is not in ['ps2mouse'] + WARNING:root:DEFAULT/instances_path sample value ['$state_path/instances'] is not in ['/opt/stack/data/nova/instances'] + WARNING:root:DEFAULT/shutdown_timeout sample value ['60'] is not in ['0'] + INFO:root:DEFAULT/my_ip Ignoring option because it is part of the excluded patterns. This can be changed with the --exclude-options argument. + WARNING:root:DEFAULT/state_path sample value ['$pybasedir'] is not in ['/opt/stack/data/nova'] + INFO:root:DEFAULT/osapi_compute_listen Ignoring option because it is part of the excluded patterns. This can be changed with the --exclude-options argument. + WARNING:root:DEFAULT/osapi_compute_workers sample value is empty but input-file has 2 + WARNING:root:DEFAULT/metadata_workers sample value is empty but input-file has 2 + WARNING:root:DEFAULT/graceful_shutdown_timeout sample value ['60'] is not in ['5'] + INFO:root:DEFAULT/transport_url Ignoring option because it is part of the excluded patterns. This can be changed with the --exclude-options argument. + WARNING:root:DEFAULT/debug sample value is empty but input-file has True + WARNING:root:DEFAULT/logging_context_format_string sample value ['%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s'] is not in ['%(color)s%(levelname)s %(name)s [\x1b[01;36m%(global_request_id)s %(request_id)s \x1b[00;36m%(project_name)s %(user_name)s%(color)s] \x1b[01;35m%(instance)s%(color)s%(message)s\x1b[00m'] + WARNING:root:DEFAULT/logging_default_format_string sample value ['%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s'] is not in ['%(color)s%(levelname)s %(name)s [\x1b[00;36m-%(color)s] \x1b[01;35m%(instance)s%(color)s%(message)s\x1b[00m'] + WARNING:root:DEFAULT/logging_debug_format_suffix sample value ['%(funcName)s %(pathname)s:%(lineno)d'] is not in ['\x1b[00;33m{{(pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d}}\x1b[00m'] + WARNING:root:DEFAULT/logging_exception_prefix sample value ['%(asctime)s.%(msecs)03d %(process)d ERROR %(name)s %(instance)s'] is not in ['ERROR %(name)s \x1b[01;35m%(instance)s\x1b[00m'] + WARNING:root:Group api from the sample config is not defined in input-file + WARNING:root:cache/backend sample value ['dogpile.cache.null'] is not in ['dogpile.cache.memcached'] + WARNING:root:cache/enabled sample value is empty but input-file has True + WARNING:root:cinder/os_region_name sample value is empty but input-file has RegionOne + WARNING:root:cinder/auth_type sample value is empty but input-file has password + INFO:root:cinder/auth_url Ignoring option because it is part of the excluded patterns. This can be changed with the --exclude-options argument. + WARNING:root:cinder/project_name sample value is empty but input-file has service + WARNING:root:cinder/project_domain_name sample value is empty but input-file has Default + INFO:root:cinder/username Ignoring option because it is part of the excluded patterns. This can be changed with the --exclude-options argument. + WARNING:root:cinder/user_domain_name sample value is empty but input-file has Default + INFO:root:cinder/password Ignoring option because it is part of the excluded patterns. This can be changed with the --exclude-options argument. + WARNING:root:Group compute from the sample config is not defined in input-file + WARNING:root:conductor/workers sample value is empty but input-file has 2 + WARNING:root:Group console from the sample config is not defined in input-file + WARNING:root:Group consoleauth from the sample config is not defined in input-file + WARNING:root:Group cyborg from the sample config is not defined in input-file + INFO:root:api_database/connection Ignoring option because it is part of the excluded patterns. This can be changed with the --exclude-options argument. + WARNING:root:Group devices from the sample config is not defined in input-file + WARNING:root:Group ephemeral_storage_encryption from the sample config is not defined in input-file + WARNING:root:Group glance from the sample config is not defined in input-file + WARNING:root:Group guestfs from the sample config is not defined in input-file + WARNING:root:Group hyperv from the sample config is not defined in input-file + WARNING:root:Group image_cache from the sample config is not defined in input-file + WARNING:root:Group ironic from the sample config is not defined in input-file + WARNING:root:key_manager/fixed_key sample value is empty but input-file has bae3516cc1c0eb18b05440eba8012a4a880a2ee04d584a9c1579445e675b12defdc716ec + WARNING:root:key_manager/backend sample value ['barbican'] is not in ['nova.keymgr.conf_key_mgr.ConfKeyManager'] + WARNING:root:Group barbican from the sample config is not defined in input-file + WARNING:root:Group vault from the sample config is not defined in input-file + WARNING:root:Group keystone from the sample config is not defined in input-file + INFO:root:libvirt/live_migration_uri Ignoring option because it is part of the excluded patterns. This can be changed with the --exclude-options argument. + WARNING:root:libvirt/cpu_mode sample value is empty but input-file has none + WARNING:root:Group mks from the sample config is not defined in input-file + WARNING:root:neutron/default_floating_pool sample value ['nova'] is not in ['public'] + WARNING:root:neutron/service_metadata_proxy sample value is empty but input-file has True + WARNING:root:neutron/auth_type sample value is empty but input-file has password + INFO:root:neutron/auth_url Ignoring option because it is part of the excluded patterns. This can be changed with the --exclude-options argument. + WARNING:root:neutron/project_name sample value is empty but input-file has service + WARNING:root:neutron/project_domain_name sample value is empty but input-file has Default + INFO:root:neutron/username Ignoring option because it is part of the excluded patterns. This can be changed with the --exclude-options argument. + WARNING:root:neutron/user_domain_name sample value is empty but input-file has Default + INFO:root:neutron/password Ignoring option because it is part of the excluded patterns. This can be changed with the --exclude-options argument. + WARNING:root:neutron/region_name sample value is empty but input-file has RegionOne + WARNING:root:Group pci from the sample config is not defined in input-file + WARNING:root:placement/auth_type sample value is empty but input-file has password + INFO:root:placement/auth_url Ignoring option because it is part of the excluded patterns. This can be changed with the --exclude-options argument. + WARNING:root:placement/project_name sample value is empty but input-file has service + WARNING:root:placement/project_domain_name sample value is empty but input-file has Default + INFO:root:placement/username Ignoring option because it is part of the excluded patterns. This can be changed with the --exclude-options argument. + WARNING:root:placement/user_domain_name sample value is empty but input-file has Default + INFO:root:placement/password Ignoring option because it is part of the excluded patterns. This can be changed with the --exclude-options argument. + WARNING:root:placement/region_name sample value is empty but input-file has RegionOne + WARNING:root:Group powervm from the sample config is not defined in input-file + WARNING:root:Group quota from the sample config is not defined in input-file + WARNING:root:Group rdp from the sample config is not defined in input-file + WARNING:root:Group remote_debug from the sample config is not defined in input-file + WARNING:root:scheduler/workers sample value is empty but input-file has 2 + WARNING:root:filter_scheduler/track_instance_changes sample value ['True'] is not in ['False'] + WARNING:root:filter_scheduler/enabled_filters sample value ['AvailabilityZoneFilter', 'ComputeFilter', 'ComputeCapabilitiesFilter', 'ImagePropertiesFilter', 'ServerGroupAntiAffinityFilter', 'ServerGroupAffinityFilter'] is not in ['AvailabilityZoneFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,SameHostFilter,DifferentHostFilter'] + WARNING:root:Group metrics from the sample config is not defined in input-file + WARNING:root:Group serial_console from the sample config is not defined in input-file + WARNING:root:Group service_user from the sample config is not defined in input-file + WARNING:root:Group spice from the sample config is not defined in input-file + WARNING:root:upgrade_levels/compute sample value is empty but input-file has auto + WARNING:root:Group vendordata_dynamic_auth from the sample config is not defined in input-file + WARNING:root:Group vmware from the sample config is not defined in input-file + WARNING:root:Group vnc from the sample config is not defined in input-file + WARNING:root:Group workarounds from the sample config is not defined in input-file + WARNING:root:wsgi/api_paste_config sample value ['api-paste.ini'] is not in ['/etc/nova/api-paste.ini'] + WARNING:root:Group zvm from the sample config is not defined in input-file + WARNING:root:oslo_concurrency/lock_path sample value is empty but input-file has /opt/stack/data/nova + WARNING:root:Group oslo_middleware from the sample config is not defined in input-file + WARNING:root:Group cors from the sample config is not defined in input-file + WARNING:root:Group healthcheck from the sample config is not defined in input-file + WARNING:root:Group oslo_messaging_amqp from the sample config is not defined in input-file + WARNING:root:oslo_messaging_notifications/driver sample value is empty but input-file has messagingv2 + INFO:root:oslo_messaging_notifications/transport_url Ignoring option because it is part of the excluded patterns. This can be changed with the --exclude-options argument. + WARNING:root:Group oslo_messaging_rabbit from the sample config is not defined in input-file + WARNING:root:Group oslo_messaging_kafka from the sample config is not defined in input-file + WARNING:root:Group oslo_policy from the sample config is not defined in input-file + WARNING:root:Group privsep from the sample config is not defined in input-file + WARNING:root:Group profiler from the sample config is not defined in input-file + INFO:root:database/connection Ignoring option because it is part of the excluded patterns. This can be changed with the --exclude-options argument. + WARNING:root:keystone_authtoken/interface sample value ['internal'] is not in ['public'] + WARNING:root:keystone_authtoken/cafile sample value is empty but input-file has /opt/stack/data/ca-bundle.pem + WARNING:root:keystone_authtoken/memcached_servers sample value is empty but input-file has localhost:11211 + WARNING:root:keystone_authtoken/auth_type sample value is empty but input-file has password + ERROR:root:neutron/auth_strategy is not part of the sample config + INFO:root:Ignoring missing option "project_domain_name" from group "keystone_authtoken" because the group is known to have incomplete sample config data and thus cannot be validated properly. + INFO:root:Ignoring missing option "project_name" from group "keystone_authtoken" because the group is known to have incomplete sample config data and thus cannot be validated properly. + INFO:root:Ignoring missing option "user_domain_name" from group "keystone_authtoken" because the group is known to have incomplete sample config data and thus cannot be validated properly. + INFO:root:Ignoring missing option "password" from group "keystone_authtoken" because the group is known to have incomplete sample config data and thus cannot be validated properly. + INFO:root:Ignoring missing option "username" from group "keystone_authtoken" because the group is known to have incomplete sample config data and thus cannot be validated properly. + INFO:root:Ignoring missing option "auth_url" from group "keystone_authtoken" because the group is known to have incomplete sample config data and thus cannot be validated properly + Handling Dynamic Groups ----------------------- diff --git a/doc/source/conf.py b/doc/source/conf.py index 5ce2034..3535e20 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,4 +1,17 @@ # -*- coding: utf-8 -*- +# Copyright (C) 2020 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. import os import sys diff --git a/doc/source/reference/configuration-files.rst b/doc/source/reference/configuration-files.rst index b4e299d..435a135 100644 --- a/doc/source/reference/configuration-files.rst +++ b/doc/source/reference/configuration-files.rst @@ -7,7 +7,7 @@ and ``--config-dir``: .. code-block:: python - class ConfigOpts(object): + class ConfigOpts: def __call__(self, ...): diff --git a/doc/source/reference/defining.rst b/doc/source/reference/defining.rst index d8e1866..19a70ec 100644 --- a/doc/source/reference/defining.rst +++ b/doc/source/reference/defining.rst @@ -81,7 +81,7 @@ the option is referenced: .. code-block:: python - class ExtensionManager(object): + class ExtensionManager: enabled_apis_opt = cfg.ListOpt(...) diff --git a/lower-constraints.txt b/lower-constraints.txt index 1e336f2..73d3236 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -57,6 +57,7 @@ requests-mock==1.5.0 requestsexceptions==1.2.0 restructuredtext-lint==1.3.1 rfc3986==1.2.0 +rst2txt==1.1.0 six==1.15.0 smmap==0.9.0 snowballstemmer==1.2.1 diff --git a/oslo_config/cfg.py b/oslo_config/cfg.py index 5324e52..39a9878 100644 --- a/oslo_config/cfg.py +++ b/oslo_config/cfg.py @@ -424,7 +424,7 @@ def _report_deprecation(format_str, format_dict): @functools.total_ordering -class Opt(object): +class Opt: """Base class for all configuration options. @@ -809,7 +809,7 @@ class Opt(object): return hash(self) < hash(another) -class DeprecatedOpt(object): +class DeprecatedOpt: """Represents a Deprecated option. @@ -1416,7 +1416,7 @@ class _ConfigDirOpt(Opt): return kwargs -class OptGroup(object): +class OptGroup: """Represents a group of opts. @@ -3155,7 +3155,7 @@ class ConfigOpts(abc.Mapping): """Return the number of options and option groups.""" return len(self._group._opts) - class SubCommandAttr(object): + class SubCommandAttr: """Helper class. @@ -3189,7 +3189,7 @@ class ConfigOpts(abc.Mapping): except AttributeError: raise NoSuchOptError(name) - class StrSubWrapper(object): + class StrSubWrapper: """Helper class. diff --git a/oslo_config/generator.py b/oslo_config/generator.py index f422587..9fdde09 100644 --- a/oslo_config/generator.py +++ b/oslo_config/generator.py @@ -29,7 +29,18 @@ import json import logging import operator import sys -import textwrap + +try: + # For oslo.config[rst-generator] + from docutils import core as docutils_core + from docutils.parsers.rst import nodes as docutils_nodes + from docutils.parsers.rst import roles as docutils_roles + import rst2txt + from sphinx import roles as sphinx_roles +except ImportError: + import textwrap + + rst2txt = None try: # For Python 3.8 and later @@ -53,6 +64,7 @@ _generator_opts = [ cfg.IntOpt( 'wrap-width', default=70, + min=0, help='The maximum length of help lines.'), cfg.MultiStrOpt( 'namespace', @@ -162,7 +174,7 @@ def _format_type_name(opt_type): return 'unknown value' -class _OptFormatter(object): +class _OptFormatter: """Format configuration option descriptions to a file.""" @@ -173,16 +185,43 @@ class _OptFormatter(object): :param output_file: a writeable file object """ self.output_file = output_file or sys.stdout - self.wrap_width = conf.wrap_width + self.wrap_width = conf.wrap_width or 998 # arbitrary line length self.minimal = conf.minimal self.summarize = conf.summarize + if rst2txt: + # register the default Sphinx roles + for rolename, nodeclass in sphinx_roles.generic_docroles.items(): + generic = docutils_roles.GenericRole(rolename, nodeclass) + docutils_roles.register_local_role(rolename, generic) + + for rolename, func in sphinx_roles.specific_docroles.items(): + docutils_roles.register_local_role(rolename, func) + + # plus our custom roles + for rolename in ('oslo.config:option', 'oslo.config:group'): + generic = docutils_roles.GenericRole(rolename, + docutils_nodes.strong) + docutils_roles.register_local_role(rolename, generic) + def _format_help(self, help_text): """Format the help for a group or option to the output file. :param help_text: The text of the help string """ - if self.wrap_width is not None and self.wrap_width > 0: + if rst2txt: + help_text = docutils_core.publish_string( + source=help_text, + writer=rst2txt.Writer(), + settings_overrides={'wrap_width': self.wrap_width} + ).decode() + + lines = '' + for line in help_text.splitlines(): + lines += '# ' + line + '\n' if line else '#\n' + + lines = [lines] + elif self.wrap_width > 0: wrapped = "" for line in help_text.splitlines(): text = "\n".join(textwrap.wrap(line, self.wrap_width, @@ -195,6 +234,7 @@ class _OptFormatter(object): lines = [wrapped] else: lines = ['# ' + help_text + '\n'] + return lines def _get_choice_text(self, choice): diff --git a/oslo_config/iniparser.py b/oslo_config/iniparser.py index 90218e4..b79ee08 100644 --- a/oslo_config/iniparser.py +++ b/oslo_config/iniparser.py @@ -23,7 +23,7 @@ class ParseError(Exception): return 'at line %d, %s: %r' % (self.lineno, self.msg, self.line) -class BaseParser(object): +class BaseParser: lineno = 0 parse_exc = ParseError diff --git a/oslo_config/sources/__init__.py b/oslo_config/sources/__init__.py index 92b557d..a1fa0c5 100644 --- a/oslo_config/sources/__init__.py +++ b/oslo_config/sources/__init__.py @@ -33,10 +33,10 @@ import abc _NoValue = object() -class ConfigurationSourceDriver(object, metaclass=abc.ABCMeta): +class ConfigurationSourceDriver(metaclass=abc.ABCMeta): """A backend driver option for oslo.config. - For each group name listed in **config_source** on the **DEFAULT** group, + For each group name listed in **config_source** in the **DEFAULT** group, a :class:`ConfigurationSourceDriver` is responsible for creating one new instance of a :class:`ConfigurationSource`. The proper driver class to be used is selected based on the **driver** option inside each one of the @@ -96,7 +96,7 @@ class ConfigurationSourceDriver(object, metaclass=abc.ABCMeta): """ -class ConfigurationSource(object, metaclass=abc.ABCMeta): +class ConfigurationSource(metaclass=abc.ABCMeta): """A configuration source option for oslo.config. A configuration source is able to fetch configuration values based on diff --git a/oslo_config/sphinxext.py b/oslo_config/sphinxext.py index 4a11163..6aadf73 100644 --- a/oslo_config/sphinxext.py +++ b/oslo_config/sphinxext.py @@ -495,6 +495,15 @@ class ConfigDomain(Domain): ) return None + def merge_domaindata(self, docnames, otherdata): + for target, docname in otherdata['options'].items(): + if docname in docnames: + self.data['options'][target] = docname + + for target, docname in otherdata['groups'].items(): + if docname in docnames: + self.data['groups'][target] = docname + def setup(app): # NOTE(dhellmann): Try to turn off lazy translation from oslo_i18n diff --git a/oslo_config/tests/test_cfg.py b/oslo_config/tests/test_cfg.py index ac45780..505cb00 100644 --- a/oslo_config/tests/test_cfg.py +++ b/oslo_config/tests/test_cfg.py @@ -3958,7 +3958,7 @@ class FindFileTestCase(BaseTestCase): class OptDumpingTestCase(BaseTestCase): - class FakeLogger(object): + class FakeLogger: def __init__(self, test_case, expected_lvl): self.test_case = test_case diff --git a/oslo_config/tests/test_generator.py b/oslo_config/tests/test_generator.py index a069e24..d75b4d8 100644 --- a/oslo_config/tests/test_generator.py +++ b/oslo_config/tests/test_generator.py @@ -405,16 +405,16 @@ class GeneratorTestCase(base.BaseTestCase): # ''' # noqa -'# Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod ' -'tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, ' -'quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo ' -'consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse ' -'cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat ' -'non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. ' -'(string value)' +'# Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod ' # noqa +'tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, ' # noqa +'quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo ' # noqa +'consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse ' # noqa +'cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat ' # noqa +'non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. ' # noqa +'(string value)' # noqa ''' #long_help = <None> -''')), +''')), # noqa ('long_help_with_preformatting', dict(opts=[('test', [(None, [opts['long_help_pre']])])], wrap_width=70, @@ -1638,7 +1638,7 @@ class GeneratorRaiseErrorTestCase(base.BaseTestCase): class FakeException(Exception): pass - class FakeEP(object): + class FakeEP: def __init__(self): self.name = 'callback_is_expected' diff --git a/oslo_config/tests/test_types.py b/oslo_config/tests/test_types.py index 644a239..dd56b98 100644 --- a/oslo_config/tests/test_types.py +++ b/oslo_config/tests/test_types.py @@ -37,7 +37,7 @@ class ConfigTypeTests(unittest.TestCase): MyString() -class TypeTestHelper(object): +class TypeTestHelper: def setUp(self): super(TypeTestHelper, self).setUp() self.type_instance = self.type @@ -766,6 +766,27 @@ class HostAddressTypeTests(TypeTestHelper, unittest.TestCase): self.assertConvertedValue('abc.0-0', 'abc.0-0') +class HostDomainTypeTests(TypeTestHelper, unittest.TestCase): + type = types.HostDomain() + + def test_invalid_host_addresses(self): + self.assertInvalid('-1') + self.assertInvalid('3.14') + self.assertInvalid('10.0') + self.assertInvalid('host..name') + self.assertInvalid('org.10') + self.assertInvalid('0.0.00') + + def test_valid_host_addresses(self): + self.assertConvertedValue('_foo', '_foo') + self.assertConvertedValue('host_name', 'host_name') + self.assertConvertedValue( + 'overcloud-novacompute_edge1-0.internalapi.localdomain', + 'overcloud-novacompute_edge1-0.internalapi.localdomain') + self.assertConvertedValue('host_01.co.uk', 'host_01.co.uk') + self.assertConvertedValue('_site01001', '_site01001') + + class HostnameTypeTests(TypeTestHelper, unittest.TestCase): type = types.Hostname() @@ -796,8 +817,8 @@ class HostnameTypeTests(TypeTestHelper, unittest.TestCase): self.assertInvalid("h'ost'") self.assertInvalid("h'ost") self.assertInvalid("h$ost") - self.assertInvalid("h%ost") self.assertInvalid("host_01.co.uk") + self.assertInvalid("h%ost") self.assertInvalid("host;name=99") self.assertInvalid('___site0.1001') self.assertInvalid('_site01001') @@ -808,6 +829,7 @@ class HostnameTypeTests(TypeTestHelper, unittest.TestCase): def test_invalid_hostnames_with_numeric_characters(self): self.assertInvalid("10.0.0.0") self.assertInvalid("3.14") + self.assertInvalid('___site0.1001') self.assertInvalid("org.10") self.assertInvalid('0.0.00') diff --git a/oslo_config/tests/test_validator.py b/oslo_config/tests/test_validator.py index de912ac..383340f 100644 --- a/oslo_config/tests/test_validator.py +++ b/oslo_config/tests/test_validator.py @@ -21,13 +21,51 @@ from oslo_config import fixture from oslo_config import validator -OPT_DATA = {'options': {'foo': {'opts': [{'name': 'opt'}]}, - 'bar': {'opts': [{'name': 'opt'}]}, - }, - 'deprecated_options': {'bar': [{'name': 'opt'}]}} +OPT_DATA = { + "options": { + "foo": { + "opts": [ + { + "name": "opt", + "default": 1 + } + ] + }, + "bar": { + "opts": [ + { + "name": "opt", + "default": 2 + }, + { + "name": "foo-bar", + "dest": "foo_bar", + "default": 2 + }, + { + "name": "bar-foo", + "dest": "bar_foo", + "default": 2 + } + ] + } + }, + "deprecated_options": { + "bar": [ + { + "name": "opt", + "default": 3 + } + ] + } +} VALID_CONF = """ [foo] -opt = value +opt = 1 +[bar] +opt = 3 +foo-bar = 3 +bar_foo = 3 """ DEPRECATED_CONF = """ [bar] @@ -106,5 +144,15 @@ class TestValidator(base.BaseTestCase): with mock.patch('builtins.open', m): self.assertEqual(0, validator._validate(self.conf)) + @mock.patch('oslo_config.validator.load_opt_data') + def test_check_defaults(self, mock_lod): + mock_lod.return_value = OPT_DATA + self.conf_fixture.config(opt_data='mocked.yaml', + input_file='mocked.conf', + check_defaults=True) + m = mock.mock_open(read_data=VALID_CONF) + with mock.patch('builtins.open', m): + self.assertEqual(0, validator._validate(self.conf)) + def test_invalid_options(self): self.assertRaises(RuntimeError, validator._validate, self.conf) diff --git a/oslo_config/types.py b/oslo_config/types.py index dfd89bc..72242cb 100644 --- a/oslo_config/types.py +++ b/oslo_config/types.py @@ -30,7 +30,7 @@ import netaddr import rfc3986 -class ConfigType(object, metaclass=abc.ABCMeta): +class ConfigType(metaclass=abc.ABCMeta): def __init__(self, type_name='unknown type'): self.type_name = type_name @@ -761,11 +761,12 @@ class Hostname(ConfigType): :param type_name: Type name to be used in the sample config file. """ + HOSTNAME_REGEX = '(?!-)[A-Z0-9-]{1,63}(?<!-)$' def __init__(self, type_name='hostname value'): super(Hostname, self).__init__(type_name=type_name) - def __call__(self, value): + def __call__(self, value, regex=HOSTNAME_REGEX): """Check hostname is valid. Ensures that each segment @@ -789,7 +790,7 @@ class Hostname(ConfigType): % value) if value.endswith("."): value = value[:-1] - allowed = re.compile("(?!-)[A-Z0-9-]{1,63}(?<!-)$", re.IGNORECASE) + allowed = re.compile(regex, re.IGNORECASE) if not re.search('[a-zA-Z-]', value.split(".")[-1]): raise ValueError('%s contains no non-numeric characters in the ' 'top-level domain part of the host name and is ' @@ -844,7 +845,8 @@ class HostAddress(ConfigType): try: value = self.hostname(value) except ValueError: - raise ValueError("%s is not a valid host address" % (value,)) + raise ValueError( + "%s is not a valid host address" % (value,)) return value def __repr__(self): @@ -857,6 +859,45 @@ class HostAddress(ConfigType): return value +class HostDomain(HostAddress): + """Host Domain type. + + Like HostAddress with the support of _ character. + + :param version: defines which version should be explicitly + checked (4 or 6) in case of an IP address + :param type_name: Type name to be used in the sample config file. + """ + # DOMAIN_REGEX is HOSTNAME_REGEX with the _ character added + DOMAIN_REGEX = '(?!-)[A-Z0-9-_]{1,63}(?<!-)$' + + def __call__(self, value): + """Checks if is a valid IP/hostname. + + If not a valid IP, makes sure it is not a mistyped IP before + performing checks for it as a hostname. + + """ + + try: + value = super(HostDomain, self).__call__(value) + except ValueError: + # Check if domain is valid + # Add support of underscore + # https://www.ietf.org/rfc/rfc1912, + # http://domainkeys.sourceforge.net/underscore.html + # https://bugs.launchpad.net/oslo.config/+bug/1892044 + try: + value = self.hostname(value, regex=self.DOMAIN_REGEX) + except ValueError: + raise ValueError( + "%s is not a valid host address" % (value,)) + return value + + def __repr__(self): + return 'HostDomain' + + class URI(ConfigType): """URI type diff --git a/oslo_config/validator.py b/oslo_config/validator.py index 725a868..938ee65 100644 --- a/oslo_config/validator.py +++ b/oslo_config/validator.py @@ -22,6 +22,7 @@ project then it returns those errors. """ import logging +import re import sys try: @@ -36,6 +37,11 @@ import yaml from oslo_config import cfg from oslo_config import generator +VALIDATE_DEFAULTS_EXCLUSIONS = [ + '.*_ur(i|l)', '.*connection', 'password', 'username', 'my_ip', + 'host(name)?', 'glance_api_servers', 'osapi_volume_listen', + 'osapi_compute_listen', +] _validator_opts = [ cfg.MultiStrOpt( @@ -51,6 +57,16 @@ _validator_opts = [ help='Path to a YAML file containing definitions of options, as ' 'output by the config generator.'), cfg.BoolOpt( + 'check-defaults', + default=False, + help='Report differences between the sample values and current ' + 'values.'), + cfg.ListOpt( + 'exclude-options', + default=VALIDATE_DEFAULTS_EXCLUSIONS, + help='Exclude options matching these patterns when comparing ' + 'the current and sample configurations.'), + cfg.BoolOpt( 'fatal-warnings', default=False, help='Report failure if any warnings are found.'), @@ -86,6 +102,73 @@ def _validate_deprecated_opt(group, option, opt_data): return option in name_data +def _validate_defaults(sections, opt_data, conf): + """Compares the current and sample configuration and reports differences + + :param section: ConfigParser instance + :param opt_data: machine readable data from the generator instance + :param conf: ConfigOpts instance + :returns: boolean wether or not warnings were reported + """ + warnings = False + # Generating regex objects from ListOpt + exclusion_regexes = [] + for pattern in conf.exclude_options: + exclusion_regexes.append(re.compile(pattern)) + for group, opts in opt_data['options'].items(): + if group in conf.exclude_group: + continue + if group not in sections: + logging.warning( + 'Group %s from the sample config is not defined in ' + 'input-file', group) + continue + for opt in opts['opts']: + # We need to convert the defaults into a list to find + # intersections. defaults are only a list if they can + # be defined multiple times, but configparser only + # returns list + if not isinstance(opt['default'], list): + defaults = [str(opt['default'])] + else: + defaults = opt['default'] + + # Apparently, there's multiple naming conventions for + # options, 'name' is mostly with hyphens, and 'dest' + # is represented with underscores. + opt_names = set([opt['name'], opt.get('dest')]) + if not opt_names.intersection(sections[group]): + continue + try: + value = sections[group][opt['name']] + keyname = opt['name'] + except KeyError: + value = sections[group][opt.get('dest')] + keyname = opt.get('dest') + + if any(rex.fullmatch(keyname) for rex in exclusion_regexes): + logging.info( + '%s/%s Ignoring option because it is part of the excluded ' + 'patterns. This can be changed with the --exclude-options ' + 'argument', group, keyname) + continue + + if len(value) > 1: + logging.info( + '%s/%s defined %s times', group, keyname, len(value)) + if not opt['default']: + logging.warning( + '%s/%s sample value is empty but input-file has %s', + group, keyname, ", ".join(value)) + warnings = True + elif not frozenset(defaults).intersection(value): + logging.warning( + '%s/%s sample value %s is not in %s', + group, keyname, defaults, value) + warnings = True + return warnings + + def _validate_opt(group, option, opt_data): if group not in opt_data['options']: return False @@ -114,12 +197,14 @@ def _validate(conf): parser.parse() warnings = False errors = False + if conf.check_defaults: + warnings = _validate_defaults(sections, opt_data, conf) for section, options in sections.items(): if section in conf.exclude_group: continue for option in options: if _validate_deprecated_opt(section, option, opt_data): - logging.warn('Deprecated opt %s/%s found', section, option) + logging.warning('Deprecated opt %s/%s found', section, option) warnings = True elif not _validate_opt(section, option, opt_data): if section in KNOWN_BAD_GROUPS: @@ -129,7 +214,8 @@ def _validate(conf): 'cannot be validated properly.', option, section) continue - logging.error('%s/%s not found', section, option) + logging.error('%s/%s is not part of the sample config', + section, option) errors = True if errors or (warnings and conf.fatal_warnings): return 1 diff --git a/releasenotes/notes/add-HostDomain-implement-rfc1033-c985a3054f824e9d.yaml b/releasenotes/notes/add-HostDomain-implement-rfc1033-c985a3054f824e9d.yaml new file mode 100644 index 0000000..1f8d153 --- /dev/null +++ b/releasenotes/notes/add-HostDomain-implement-rfc1033-c985a3054f824e9d.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Add ``HostDomain`` to handle address with underscore. Underscore are + allowed in domain by RFC1033 [1][2][3]. + + - [1] https://www.ietf.org/rfc/rfc1912.txt + - [2] https://www.ietf.org/rfc/rfc1033.txt + - [3] http://domainkeys.sourceforge.net/underscore.html diff --git a/releasenotes/notes/validator-check-defaults-e7b596a2fde781a8.yaml b/releasenotes/notes/validator-check-defaults-e7b596a2fde781a8.yaml new file mode 100644 index 0000000..3fdbfe6 --- /dev/null +++ b/releasenotes/notes/validator-check-defaults-e7b596a2fde781a8.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Add a ``--check-defaults`` flag to ``oslo-config-validator``. When set, + ``oslo-config-validator`` will compare the ``input-file`` with the sample + generated configuration and flag any discrepancies. It also obeys the + standard ``--exclude-group`` and ``--fatal-warning`` options. diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index 2051f56..0ebefc1 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,7 @@ :maxdepth: 1 unreleased + victoria ussuri train stein diff --git a/releasenotes/source/victoria.rst b/releasenotes/source/victoria.rst new file mode 100644 index 0000000..4efc7b6 --- /dev/null +++ b/releasenotes/source/victoria.rst @@ -0,0 +1,6 @@ +============================= +Victoria Series Release Notes +============================= + +.. release-notes:: + :branch: stable/victoria @@ -22,6 +22,13 @@ classifier = Programming Language :: Python :: 3 :: Only Programming Language :: Python :: Implementation :: CPython +[extras] +# package dependencies for optional behavior in the config generator. +# e.g.: oslo.config[generator] +rst-generator = + rst2txt>=1.1.0 # BSD + sphinx>=1.8.0,!=2.1.0 # BSD + [files] packages = oslo_config diff --git a/test-requirements.txt b/test-requirements.txt index bfe3ac5..8cb29d3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,3 +25,5 @@ requests_mock>=1.5.0 # Apache-2.0 # Bandit security code scanner bandit>=1.6.0,<1.7.0 # Apache-2.0 + +pre-commit>=2.6.0 # MIT @@ -1,7 +1,7 @@ [tox] minversion = 3.2.0 distribute = False -envlist = py37,pep8 +envlist = py3,pep8 ignore_basepython_conflict = true [testenv] @@ -31,7 +31,7 @@ deps = [testenv:pep8] commands = - flake8 + pre-commit run -a # TODO(stephenfin): Add mypy here once it's passing {[testenv:bandit]commands} @@ -56,7 +56,7 @@ deps = -r{toxinidir}/doc/requirements.txt commands = rm -fr doc/build - sphinx-build -W --keep-going -b html doc/source doc/build/html {posargs} + sphinx-build -W --keep-going -b html -j auto doc/source doc/build/html {posargs} [testenv:mypy] commands = @@ -72,7 +72,7 @@ whitelist_externals = deps = {[testenv:docs]deps} commands = rm -rf releasenotes/build - sphinx-build -a -E -W -d releasenotes/build/doctrees --keep-going -b html releasenotes/source releasenotes/build/html + sphinx-build -W --keep-going -b html -j auto releasenotes/source releasenotes/build/html {posargs} [hacking] import_exceptions = oslo_config._i18n |