diff options
-rw-r--r-- | doc/source/cli/common/convert-opts.rst | 8 | ||||
-rw-r--r-- | doc/source/cli/index.rst | 1 | ||||
-rw-r--r-- | doc/source/cli/oslopolicy-convert-json-to-yaml.rst | 85 | ||||
-rw-r--r-- | oslo_policy/generator.py | 110 | ||||
-rw-r--r-- | oslo_policy/tests/test_generator.py | 143 | ||||
-rw-r--r-- | releasenotes/notes/add-policy-convert-json-to-yaml-tool-3c93604aee79f58a.yaml | 5 | ||||
-rw-r--r-- | setup.cfg | 1 |
7 files changed, 348 insertions, 5 deletions
diff --git a/doc/source/cli/common/convert-opts.rst b/doc/source/cli/common/convert-opts.rst new file mode 100644 index 0000000..c7f69b6 --- /dev/null +++ b/doc/source/cli/common/convert-opts.rst @@ -0,0 +1,8 @@ +.. option:: --namespace NAMESPACE + + Option namespace(s) under "oslo.policy.policies" in which to query for + options. + +.. option:: --policy-file POLICY_FILE + + Path to the policy file which need to be converted to ``yaml`` format. diff --git a/doc/source/cli/index.rst b/doc/source/cli/index.rst index b8e54d8..500a7fe 100644 --- a/doc/source/cli/index.rst +++ b/doc/source/cli/index.rst @@ -13,3 +13,4 @@ This document describes the various command line tools exposed by oslopolicy-list-redundant oslopolicy-policy-generator oslopolicy-sample-generator + oslopolicy-convert-json-to-yaml diff --git a/doc/source/cli/oslopolicy-convert-json-to-yaml.rst b/doc/source/cli/oslopolicy-convert-json-to-yaml.rst new file mode 100644 index 0000000..1689aed --- /dev/null +++ b/doc/source/cli/oslopolicy-convert-json-to-yaml.rst @@ -0,0 +1,85 @@ +=============================== +oslopolicy-convert-json-to-yaml +=============================== + +.. program:: oslopolicy-convert-json-to-yaml + +Synopsis +-------- + +:: + + oslopolicy-convert-json-to-yaml [-h] [--config-dir DIR] [--config-file PATH] + [--namespace NAMESPACE] + [--policy-file POLICY_FILE] + [--output-file OUTPUT_FILE] + + +Description +----------- + +The ``oslopolicy-convert-json-to-yaml`` tool can be used to convert the JSON +format policy file to YAML format. It takes JSON formatted policy file as input +and convert it to a YAML formatted policy file similar to +``oslopolicy-sample-generator`` tool except keeping the overridden rule +as uncommented. It does the following: + +* Comment out any rules that match the default from policy-in-code. +* Keep rules uncommented if rule is overridden. +* Does not auto add the deprecated rules in the file unless it not already + present in the file. +* Keep any extra rules or already exist deprecated rules uncommented + but at the end of the file with a warning text. + +When to use: +~~~~~~~~~~~~ + +Oslo policy still support the policy file in JSON format, but that lead to +`multiple issues <https://specs.openstack.org/openstack/oslo-specs/specs/victoria/policy-json-to-yaml.html#problem-description>`_ . +One of the key issue came up while nova switched to the new policy with new +defaults and scope feature from keystone. +Refer `this bug <https://bugs.launchpad.net/nova/+bug/1875418>`_ for details. + +In future release, oslo policy will remove the JSON formatted policy +file support and to have a smooth migration to YAML formatted policy file +you can use this tool to convert your existing JSON formatted file to YAML +file. + +Options +------- + +.. include:: common/default-opts.rst + +.. include:: common/generator-opts.rst + +.. include:: common/convert-opts.rst + +Examples +-------- + +To convert a JSON policy file for a namespace called ``keystone``: + +.. code-block:: bash + + oslopolicy-convert-json-to-yaml --namespace keystone \ + --policy-file keystone-policy.json + +To convert a JSON policy file to yaml format directly to a file: + +.. code-block:: bash + + oslopolicy-convert-json-to-yaml --namespace keystone \ + --policy-file keystone-policy.json \ + --output-file keystone-policy.yaml + +Use the following to generate help text for additional options and arguments +supported by ``oslopolicy-convert-json-to-yaml``: + +.. code-block:: bash + + oslopolicy-convert-json-to-yaml --help + +See Also +-------- + +:program:`oslopolicy-sample-generator`, :program:`oslopolicy-policy-generator`, :program:`oslopolicy-upgrade` diff --git a/oslo_policy/generator.py b/oslo_policy/generator.py index 40f374d..5262784 100644 --- a/oslo_policy/generator.py +++ b/oslo_policy/generator.py @@ -51,6 +51,17 @@ UPGRADE_OPTS = [ help='Path to the policy file which need to be updated.') ] +CONVERT_OPTS = [ + cfg.MultiStrOpt('namespace', + required=True, + help='Option namespace(s) under "oslo.policy.policies" in ' + 'which to query for options.'), + cfg.StrOpt('policy-file', + required=True, + help='Path to the policy file which need to be converted to ' + 'yaml format.') +] + def get_policies_dict(namespaces): """Find the options available via the given namespaces. @@ -139,10 +150,16 @@ def _format_help_text(description): return '\n'.join(formatted_lines) -def _format_rule_default_yaml(default, include_help=True): +def _format_rule_default_yaml(default, include_help=True, comment_rule=True, + add_deprecated_rules=True): """Create a yaml node from policy.RuleDefault or policy.DocumentedRuleDefault. :param default: A policy.RuleDefault or policy.DocumentedRuleDefault object + :param comment_rule: By default rules will be commented out in generated + yaml format text. If you want to keep few or all rules + uncommented then pass this arg as False. + :param add_deprecated_rules: Whether to add the deprecated rules in format + text. :returns: A string containing a yaml representation of the RuleDefault """ text = ('"%(name)s": "%(check_str)s"\n' % @@ -161,14 +178,15 @@ def _format_rule_default_yaml(default, include_help=True): intended_scope = ( '# Intended scope(s): ' + ', '.join(default.scope_types) + '\n' ) - - text = ('%(help)s\n%(op)s%(scope)s#%(text)s\n' % + comment = '#' if comment_rule else '' + text = ('%(help)s\n%(op)s%(scope)s%(comment)s%(text)s\n' % {'help': _format_help_text(default.description), 'op': op, 'scope': intended_scope, + 'comment': comment, 'text': text}) - if default.deprecated_for_removal: + if add_deprecated_rules and default.deprecated_for_removal: text = ( '# DEPRECATED\n# "%(name)s" has been deprecated since ' '%(since)s.\n%(reason)s\n%(text)s' @@ -177,7 +195,7 @@ def _format_rule_default_yaml(default, include_help=True): 'since': default.deprecated_since, 'reason': _format_help_text(default.deprecated_reason), 'text': text} - elif default.deprecated_rule: + elif add_deprecated_rules and default.deprecated_rule: # This issues a deprecation warning but aliases the old policy name # with the new policy name for compatibility. deprecated_text = ( @@ -392,6 +410,75 @@ def _validate_policy(namespace): return return_code +def _convert_policy_json_to_yaml(namespace, policy_file, output_file=None): + with open(policy_file, 'r') as rule_data: + file_policies = jsonutils.loads(rule_data.read()) + + yaml_format_rules = [] + default_policies = get_policies_dict(namespace) + for section in sorted(default_policies): + default_rules = default_policies[section] + for default_rule in default_rules: + if default_rule.name not in file_policies: + continue + file_rule_check_str = file_policies.pop(default_rule.name) + # Some rules might be still RuleDefault object so let's prepare + # empty 'operations' list for those. + operations = [{ + 'method': '', + 'path': '' + }] + if hasattr(default_rule, 'operations'): + operations = default_rule.operations + # Converting JSON file rules to DocumentedRuleDefault rules so + # that we can convert the JSON file to YAML including + # descriptions which is what 'oslopolicy-sample-generator' + # tool does. + file_rule = policy.DocumentedRuleDefault( + default_rule.name, + file_rule_check_str, + default_rule.description, + operations, + default_rule.deprecated_rule, + default_rule.deprecated_for_removal, + default_rule.deprecated_reason, + default_rule.deprecated_since, + scope_types=default_rule.scope_types) + if file_rule == default_rule: + rule_text = _format_rule_default_yaml( + file_rule, add_deprecated_rules=False) + else: + # NOTE(gmann): If json file rule is not same as default + # means rule is overridden then do not comment out it in + # yaml file. + rule_text = _format_rule_default_yaml( + file_rule, comment_rule=False, + add_deprecated_rules=False) + yaml_format_rules.append(rule_text) + + extra_rules_text = ("# WARNING: Below rules are either deprecated rules\n" + "# or extra rules in policy file, it is strongly\n" + "# recommended to switch to new rules.\n") + # NOTE(gmann): If policy json file still using the deprecated rules which + # will not be present in default rules list. Or it can be case of any + # extra rule (old rule which is now removed) present in json file. + # so let's keep these as it is (not commented out) to avoid breaking + # existing deployment. + if file_policies: + yaml_format_rules.append(extra_rules_text) + for file_rule, check_str in file_policies.items(): + rule_text = ('"%(name)s": "%(check_str)s"\n' % + {'name': file_rule, + 'check_str': check_str}) + yaml_format_rules.append(rule_text) + + if output_file: + with open(output_file, 'w') as fh: + fh.writelines(yaml_format_rules) + else: + sys.stdout.writelines(yaml_format_rules) + + def on_load_failure_callback(*args, **kwargs): raise @@ -490,3 +577,16 @@ def validate_policy(args=None): conf.register_opts(ENFORCER_OPTS) conf(args) sys.exit(_validate_policy(conf.namespace)) + + +def convert_policy_json_to_yaml(args=None, conf=None): + logging.basicConfig(level=logging.WARN) + # Allow the caller to pass in a local conf object for unit testing + if conf is None: + conf = cfg.CONF + conf.register_cli_opts(GENERATOR_OPTS + CONVERT_OPTS) + conf.register_opts(GENERATOR_OPTS + CONVERT_OPTS) + conf(args) + _check_for_namespace_opt(conf) + _convert_policy_json_to_yaml(conf.namespace, conf.policy_file, + conf.output_file) diff --git a/oslo_policy/tests/test_generator.py b/oslo_policy/tests/test_generator.py index 1f74aa3..ab726dc 100644 --- a/oslo_policy/tests/test_generator.py +++ b/oslo_policy/tests/test_generator.py @@ -799,3 +799,146 @@ class ValidatorTestCase(base.PolicyBaseTestCase): def test_missing_policy_file(self): self._test_policy('', missing_file=True) + + +class ConvertJsonToYamlTestCase(base.PolicyBaseTestCase): + def setUp(self): + super(ConvertJsonToYamlTestCase, self).setUp() + policy_json_contents = jsonutils.dumps({ + "rule1_name": "rule:admin", + "rule2_name": "rule:overridden", + "deprecated_rule1_name": "rule:admin" + }) + self.create_config_file('policy.json', policy_json_contents) + self.output_file_path = self.get_config_file_fullname( + 'converted_policy.yaml') + deprecated_policy = policy.DeprecatedRule( + name='deprecated_rule1_name', + check_str='rule:admin' + ) + self.registered_policy = [ + policy.DocumentedRuleDefault( + name='rule1_name', + check_str='rule:admin', + description='test_rule1', + operations=[{'path': '/test', 'method': 'GET'}], + deprecated_rule=deprecated_policy, + deprecated_reason='testing', + deprecated_since='ussuri', + scope_types=['system'] + ), + policy.DocumentedRuleDefault( + name='rule2_name', + check_str='rule:admin', + description='test_rule2', + operations=[{'path': '/test', 'method': 'PUT'}], + deprecated_rule=deprecated_policy, + deprecated_reason='testing2', + deprecated_since='ussuri', + scope_types=['system', 'project'] + ) + ] + self.extensions = [] + ext = stevedore.extension.Extension(name='test', + entry_point=None, + plugin=None, + obj=self.registered_policy) + self.extensions.append(ext) + # Just used for cli opt parsing + self.local_conf = cfg.ConfigOpts() + + self.expected = '''# test_rule1 +# GET /test +# Intended scope(s): system +#"rule1_name": "rule:admin" + +# test_rule2 +# PUT /test +# Intended scope(s): system, project +"rule2_name": "rule:overridden" + +# WARNING: Below rules are either deprecated rules +# or extra rules in policy file, it is strongly +# recommended to switch to new rules. +"deprecated_rule1_name": "rule:admin" +''' + + def _is_yaml(self, data): + is_yaml = False + try: + jsonutils.loads(data) + except ValueError: + try: + yaml.safe_load(data) + is_yaml = True + except yaml.scanner.ScannerError: + pass + return is_yaml + + def _test_convert_json_to_yaml_file(self, output_to_file=True): + test_mgr = stevedore.named.NamedExtensionManager.make_test_instance( + extensions=self.extensions, namespace='test') + converted_policy_data = None + with mock.patch('stevedore.named.NamedExtensionManager', + return_value=test_mgr): + testargs = ['oslopolicy-convert-json-to-yaml', + '--namespace', 'test', + '--policy-file', + self.get_config_file_fullname('policy.json')] + if output_to_file: + testargs.extend(['--output-file', + self.output_file_path]) + with mock.patch('sys.argv', testargs): + generator.convert_policy_json_to_yaml(conf=self.local_conf) + if output_to_file: + with open(self.output_file_path, 'r') as fh: + converted_policy_data = fh.read() + return converted_policy_data + + def test_convert_json_to_yaml_file(self): + converted_policy_data = self._test_convert_json_to_yaml_file() + self.assertTrue(self._is_yaml(converted_policy_data)) + self.assertEqual(self.expected, converted_policy_data) + + def test_convert_policy_to_stdout(self): + stdout = self._capture_stdout() + self._test_convert_json_to_yaml_file(output_to_file=False) + self.assertEqual(self.expected, stdout.getvalue()) + + def test_converted_yaml_is_loadable(self): + self._test_convert_json_to_yaml_file() + enforcer = policy.Enforcer(self.conf, + policy_file=self.output_file_path) + enforcer.load_rules() + for rule in ['rule2_name', 'deprecated_rule1_name']: + self.assertIn(rule, enforcer.rules) + + def test_default_rules_comment_out_in_yaml_file(self): + converted_policy_data = self._test_convert_json_to_yaml_file() + commented_default_rule = '''# test_rule1 +# GET /test +# Intended scope(s): system +#"rule1_name": "rule:admin" + +''' + self.assertIn(commented_default_rule, converted_policy_data) + + def test_overridden_rules_uncommented_in_yaml_file(self): + converted_policy_data = self._test_convert_json_to_yaml_file() + uncommented_overridden_rule = '''# test_rule2 +# PUT /test +# Intended scope(s): system, project +"rule2_name": "rule:overridden" + +''' + self.assertIn(uncommented_overridden_rule, converted_policy_data) + + def test_existing_deprecated_rules_kept_uncommented_in_yaml_file(self): + converted_policy_data = self._test_convert_json_to_yaml_file() + existing_deprecated_rule_with_warning = '''# WARNING: Below rules are either deprecated rules +# or extra rules in policy file, it is strongly +# recommended to switch to new rules. +"deprecated_rule1_name": "rule:admin" +''' + self.assertIn(existing_deprecated_rule_with_warning, + converted_policy_data) diff --git a/releasenotes/notes/add-policy-convert-json-to-yaml-tool-3c93604aee79f58a.yaml b/releasenotes/notes/add-policy-convert-json-to-yaml-tool-3c93604aee79f58a.yaml new file mode 100644 index 0000000..230a00d --- /dev/null +++ b/releasenotes/notes/add-policy-convert-json-to-yaml-tool-3c93604aee79f58a.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add ``oslopolicy-convert-json-to-yaml`` tool to convert the json formatted + policy file to yaml format in compatible way. Refer to `this document <https://docs.openstack.org/oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html>`_ for details. @@ -36,6 +36,7 @@ console_scripts = oslopolicy-list-redundant = oslo_policy.generator:list_redundant oslopolicy-policy-upgrade = oslo_policy.generator:upgrade_policy oslopolicy-validator = oslo_policy.generator:validate_policy + oslopolicy-convert-json-to-yaml = oslo_policy.generator:convert_policy_json_to_yaml oslo.policy.rule_checks = http = oslo_policy._external:HttpCheck |