Add oslopolicy-convert-json-to-yaml tool

Add ``oslopolicy-convert-json-to-yaml`` tool which can be
used to convert the json formatted policy file to yaml format.
It takes json formatted policy file as input and convert it to
a yaml formatted policy file similar to 'oslopolicy-sample-generator'
tool except keeping the overridden rule as uncommented.

This tool does the following:

* Comment out any rules that match the default from policy-in-code.
* Keep rules uncommented if rule is overridden.
* Does not auto add the deprecated rules in the file unless it not already
  present in the file.
* Keep any extra rules or already exist deprecated rules uncommented
  but at the end of the file with a warning text.

I did not add the new functionality in existing 'oslopolicy-policy-upgrade'
tool because the above listed features of new tool end up creating a
complete different code path instead of reusing it from existing tool so it
better to have separate tool which can be removed in future once all deployments
are migrated to YAML formatted file.

This commits add doc and reno also for this tool

Partial implement blueprint policy-json-to-yaml

Change-Id: Icc245951b2992cc09a891516ffd14f3d4c009920

2 files changed, 248 insertions, 5 deletions

diff --git a/oslo_policy/generator.py b/oslo_policy/generator.py
index 40f374d..5262784 100644
--- a/oslo_policy/generator.py
+++ b/oslo_policy/generator.py
@@ -51,6 +51,17 @@ UPGRADE_OPTS = [
                help='Path to the policy file which need to be updated.')
 ]
 
+CONVERT_OPTS = [
+    cfg.MultiStrOpt('namespace',
+                    required=True,
+                    help='Option namespace(s) under "oslo.policy.policies" in '
+                         'which to query for options.'),
+    cfg.StrOpt('policy-file',
+               required=True,
+               help='Path to the policy file which need to be converted to '
+                    'yaml format.')
+]
+
 
 def get_policies_dict(namespaces):
     """Find the options available via the given namespaces.
@@ -139,10 +150,16 @@ def _format_help_text(description):
     return '\n'.join(formatted_lines)
 
 
-def _format_rule_default_yaml(default, include_help=True):
+def _format_rule_default_yaml(default, include_help=True, comment_rule=True,
+                              add_deprecated_rules=True):
     """Create a yaml node from policy.RuleDefault or policy.DocumentedRuleDefault.
 
     :param default: A policy.RuleDefault or policy.DocumentedRuleDefault object
+    :param comment_rule: By default rules will be commented out in generated
+                         yaml format text. If you want to keep few or all rules
+                         uncommented then pass this arg as False.
+    :param add_deprecated_rules: Whether to add the deprecated rules in format
+                                 text.
     :returns: A string containing a yaml representation of the RuleDefault
     """
     text = ('"%(name)s": "%(check_str)s"\n' %
@@ -161,14 +178,15 @@ def _format_rule_default_yaml(default, include_help=True):
             intended_scope = (
                 '# Intended scope(s): ' + ', '.join(default.scope_types) + '\n'
             )
-
-        text = ('%(help)s\n%(op)s%(scope)s#%(text)s\n' %
+        comment = '#' if comment_rule else ''
+        text = ('%(help)s\n%(op)s%(scope)s%(comment)s%(text)s\n' %
                 {'help': _format_help_text(default.description),
                  'op': op,
                  'scope': intended_scope,
+                 'comment': comment,
                  'text': text})
 
-    if default.deprecated_for_removal:
+    if add_deprecated_rules and default.deprecated_for_removal:
         text = (
             '# DEPRECATED\n# "%(name)s" has been deprecated since '
             '%(since)s.\n%(reason)s\n%(text)s'
@@ -177,7 +195,7 @@ def _format_rule_default_yaml(default, include_help=True):
              'since': default.deprecated_since,
              'reason': _format_help_text(default.deprecated_reason),
              'text': text}
-    elif default.deprecated_rule:
+    elif add_deprecated_rules and default.deprecated_rule:
         # This issues a deprecation warning but aliases the old policy name
         # with the new policy name for compatibility.
         deprecated_text = (
@@ -392,6 +410,75 @@ def _validate_policy(namespace):
     return return_code
 
 
+def _convert_policy_json_to_yaml(namespace, policy_file, output_file=None):
+    with open(policy_file, 'r') as rule_data:
+        file_policies = jsonutils.loads(rule_data.read())
+
+    yaml_format_rules = []
+    default_policies = get_policies_dict(namespace)
+    for section in sorted(default_policies):
+        default_rules = default_policies[section]
+        for default_rule in default_rules:
+            if default_rule.name not in file_policies:
+                continue
+            file_rule_check_str = file_policies.pop(default_rule.name)
+            # Some rules might be still RuleDefault object so let's prepare
+            # empty 'operations' list for those.
+            operations = [{
+                'method': '',
+                'path': ''
+            }]
+            if hasattr(default_rule, 'operations'):
+                operations = default_rule.operations
+            # Converting JSON file rules to DocumentedRuleDefault rules so
+            # that we can convert the JSON file to YAML including
+            # descriptions which is what 'oslopolicy-sample-generator'
+            # tool does.
+            file_rule = policy.DocumentedRuleDefault(
+                default_rule.name,
+                file_rule_check_str,
+                default_rule.description,
+                operations,
+                default_rule.deprecated_rule,
+                default_rule.deprecated_for_removal,
+                default_rule.deprecated_reason,
+                default_rule.deprecated_since,
+                scope_types=default_rule.scope_types)
+            if file_rule == default_rule:
+                rule_text = _format_rule_default_yaml(
+                    file_rule, add_deprecated_rules=False)
+            else:
+                # NOTE(gmann): If json file rule is not same as default
+                # means rule is overridden then do not comment out it in
+                # yaml file.
+                rule_text = _format_rule_default_yaml(
+                    file_rule, comment_rule=False,
+                    add_deprecated_rules=False)
+            yaml_format_rules.append(rule_text)
+
+    extra_rules_text = ("# WARNING: Below rules are either deprecated rules\n"
+                        "# or extra rules in policy file, it is strongly\n"
+                        "# recommended to switch to new rules.\n")
+    # NOTE(gmann): If policy json file still using the deprecated rules which
+    # will not be present in default rules list. Or it can be case of any
+    # extra rule (old rule which is now removed) present in json file.
+    # so let's keep these as it is (not commented out) to avoid breaking
+    # existing deployment.
+    if file_policies:
+        yaml_format_rules.append(extra_rules_text)
+    for file_rule, check_str in file_policies.items():
+        rule_text = ('"%(name)s": "%(check_str)s"\n' %
+                     {'name': file_rule,
+                      'check_str': check_str})
+        yaml_format_rules.append(rule_text)
+
+    if output_file:
+        with open(output_file, 'w') as fh:
+            fh.writelines(yaml_format_rules)
+    else:
+        sys.stdout.writelines(yaml_format_rules)
+
+
 def on_load_failure_callback(*args, **kwargs):
     raise
 
@@ -490,3 +577,16 @@ def validate_policy(args=None):
     conf.register_opts(ENFORCER_OPTS)
     conf(args)
     sys.exit(_validate_policy(conf.namespace))
+
+
+def convert_policy_json_to_yaml(args=None, conf=None):
+    logging.basicConfig(level=logging.WARN)
+    # Allow the caller to pass in a local conf object for unit testing
+    if conf is None:
+        conf = cfg.CONF
+    conf.register_cli_opts(GENERATOR_OPTS + CONVERT_OPTS)
+    conf.register_opts(GENERATOR_OPTS + CONVERT_OPTS)
+    conf(args)
+    _check_for_namespace_opt(conf)
+    _convert_policy_json_to_yaml(conf.namespace, conf.policy_file,
+                                 conf.output_file)
diff --git a/oslo_policy/tests/test_generator.py b/oslo_policy/tests/test_generator.py
index 1f74aa3..ab726dc 100644
--- a/oslo_policy/tests/test_generator.py
+++ b/oslo_policy/tests/test_generator.py
@@ -799,3 +799,146 @@ class ValidatorTestCase(base.PolicyBaseTestCase):
 
     def test_missing_policy_file(self):
         self._test_policy('', missing_file=True)
+
+
+class ConvertJsonToYamlTestCase(base.PolicyBaseTestCase):
+    def setUp(self):
+        super(ConvertJsonToYamlTestCase, self).setUp()
+        policy_json_contents = jsonutils.dumps({
+            "rule1_name": "rule:admin",
+            "rule2_name": "rule:overridden",
+            "deprecated_rule1_name": "rule:admin"
+        })
+        self.create_config_file('policy.json', policy_json_contents)
+        self.output_file_path = self.get_config_file_fullname(
+            'converted_policy.yaml')
+        deprecated_policy = policy.DeprecatedRule(
+            name='deprecated_rule1_name',
+            check_str='rule:admin'
+        )
+        self.registered_policy = [
+            policy.DocumentedRuleDefault(
+                name='rule1_name',
+                check_str='rule:admin',
+                description='test_rule1',
+                operations=[{'path': '/test', 'method': 'GET'}],
+                deprecated_rule=deprecated_policy,
+                deprecated_reason='testing',
+                deprecated_since='ussuri',
+                scope_types=['system']
+            ),
+            policy.DocumentedRuleDefault(
+                name='rule2_name',
+                check_str='rule:admin',
+                description='test_rule2',
+                operations=[{'path': '/test', 'method': 'PUT'}],
+                deprecated_rule=deprecated_policy,
+                deprecated_reason='testing2',
+                deprecated_since='ussuri',
+                scope_types=['system', 'project']
+            )
+        ]
+        self.extensions = []
+        ext = stevedore.extension.Extension(name='test',
+                                            entry_point=None,
+                                            plugin=None,
+                                            obj=self.registered_policy)
+        self.extensions.append(ext)
+        # Just used for cli opt parsing
+        self.local_conf = cfg.ConfigOpts()
+
+        self.expected = '''# test_rule1
+# GET  /test
+# Intended scope(s): system
+#"rule1_name": "rule:admin"
+
+# test_rule2
+# PUT  /test
+# Intended scope(s): system, project
+"rule2_name": "rule:overridden"
+
+# WARNING: Below rules are either deprecated rules
+# or extra rules in policy file, it is strongly
+# recommended to switch to new rules.
+"deprecated_rule1_name": "rule:admin"
+'''
+
+    def _is_yaml(self, data):
+        is_yaml = False
+        try:
+            jsonutils.loads(data)
+        except ValueError:
+            try:
+                yaml.safe_load(data)
+                is_yaml = True
+            except yaml.scanner.ScannerError:
+                pass
+        return is_yaml
+
+    def _test_convert_json_to_yaml_file(self, output_to_file=True):
+        test_mgr = stevedore.named.NamedExtensionManager.make_test_instance(
+            extensions=self.extensions, namespace='test')
+        converted_policy_data = None
+        with mock.patch('stevedore.named.NamedExtensionManager',
+                        return_value=test_mgr):
+            testargs = ['oslopolicy-convert-json-to-yaml',
+                        '--namespace', 'test',
+                        '--policy-file',
+                        self.get_config_file_fullname('policy.json')]
+            if output_to_file:
+                testargs.extend(['--output-file',
+                                 self.output_file_path])
+            with mock.patch('sys.argv', testargs):
+                generator.convert_policy_json_to_yaml(conf=self.local_conf)
+                if output_to_file:
+                    with open(self.output_file_path, 'r') as fh:
+                        converted_policy_data = fh.read()
+        return converted_policy_data
+
+    def test_convert_json_to_yaml_file(self):
+        converted_policy_data = self._test_convert_json_to_yaml_file()
+        self.assertTrue(self._is_yaml(converted_policy_data))
+        self.assertEqual(self.expected, converted_policy_data)
+
+    def test_convert_policy_to_stdout(self):
+        stdout = self._capture_stdout()
+        self._test_convert_json_to_yaml_file(output_to_file=False)
+        self.assertEqual(self.expected, stdout.getvalue())
+
+    def test_converted_yaml_is_loadable(self):
+        self._test_convert_json_to_yaml_file()
+        enforcer = policy.Enforcer(self.conf,
+                                   policy_file=self.output_file_path)
+        enforcer.load_rules()
+        for rule in ['rule2_name', 'deprecated_rule1_name']:
+            self.assertIn(rule, enforcer.rules)
+
+    def test_default_rules_comment_out_in_yaml_file(self):
+        converted_policy_data = self._test_convert_json_to_yaml_file()
+        commented_default_rule = '''# test_rule1
+# GET  /test
+# Intended scope(s): system
+#"rule1_name": "rule:admin"
+
+'''
+        self.assertIn(commented_default_rule, converted_policy_data)
+
+    def test_overridden_rules_uncommented_in_yaml_file(self):
+        converted_policy_data = self._test_convert_json_to_yaml_file()
+        uncommented_overridden_rule = '''# test_rule2
+# PUT  /test
+# Intended scope(s): system, project
+"rule2_name": "rule:overridden"
+
+'''
+        self.assertIn(uncommented_overridden_rule, converted_policy_data)
+
+    def test_existing_deprecated_rules_kept_uncommented_in_yaml_file(self):
+        converted_policy_data = self._test_convert_json_to_yaml_file()
+        existing_deprecated_rule_with_warning = '''# WARNING: Below rules are either deprecated rules
+# or extra rules in policy file, it is strongly
+# recommended to switch to new rules.
+"deprecated_rule1_name": "rule:admin"
+'''
+        self.assertIn(existing_deprecated_rule_with_warning,
+                      converted_policy_data)