Implement "openstack stack failures list"

A command which prints failed resources in the stack to help debug
stack issues.

Failed resources are added by recursing from the top level stack into
failed nested stack resources. A failed nested stack resource is only
added to the failed list if it contains no failed resources so that the
user only sees root-cause failed resources.

As an added convenience to debugging deployment resources, deployment
outputs deploy_stdout and deploy_stderr are printed as well (the full
outputs are printed when --long is specified).

The output is a yamlish format where the key is a dot-delimited resource
name path and the value is a dict of resource information.

Change-Id: Ifa44175d8dbab2f7b62691b839e1696e3af7f4f3
Closes-Bug: #1585820

4 files changed, 400 insertions, 0 deletions

diff --git a/heatclient/common/format_utils.py b/heatclient/common/format_utils.py
index 5cb691a..2d950c9 100644
--- a/heatclient/common/format_utils.py
+++ b/heatclient/common/format_utils.py
@@ -81,6 +81,8 @@ def print_software_deployment_output(data, name, out=sys.stdout, long=False):
     The format attempts to be valid yaml, but is primarily aimed at showing
     useful information to the user in a helpful layout.
     """
+    if data is None:
+        data = {}
     if name in ('deploy_stdout', 'deploy_stderr'):
         output = indent_and_truncate(
             data.get(name),
diff --git a/heatclient/osc/v1/stack_failures.py b/heatclient/osc/v1/stack_failures.py
new file mode 100644
index 0000000..bc9e88c
--- /dev/null
+++ b/heatclient/osc/v1/stack_failures.py
@@ -0,0 +1,136 @@
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+import collections
+
+from cliff import command
+
+from heatclient.common import format_utils
+from heatclient import exc
+from heatclient.openstack.common._i18n import _
+
+
+class ListStackFailures(command.Command):
+    """Show information about failed stack resources."""
+
+    def take_action(self, parsed_args):
+        self.heat_client = self.app.client_manager.orchestration
+        failures = self._build_failed_resources(parsed_args.stack)
+        deployment_failures = self._build_software_deployments(failures)
+        self._print_failures(failures, deployment_failures,
+                             long=parsed_args.long)
+
+    def get_parser(self, prog_name):
+        parser = super(ListStackFailures, self).get_parser(prog_name)
+        parser.add_argument(
+            'stack',
+            metavar='<stack>',
+            help=_('Stack to display (name or ID)'),
+        )
+        parser.add_argument(
+            '--long',
+            action='store_true',
+            default=False,
+            help=_('Show full deployment logs in output'),
+        )
+        return parser
+
+    def _build_failed_resources(self, stack):
+        """List information about FAILED stack resources.
+
+        Failed resources are added by recursing from the top level stack into
+        failed nested stack resources. A failed nested stack resource is only
+        added to the failed list if it contains no failed resources.
+        """
+        s = self.heat_client.stacks.get(stack)
+        if s.status != 'FAILED':
+            return []
+        resources = self.heat_client.resources.list(s.id)
+        failures = collections.OrderedDict()
+        self._append_failed_resources(failures, resources, [s.stack_name])
+        return failures
+
+    def _append_failed_resources(self, failures, resources, resource_path):
+        """Recursively build list of failed resources."""
+        appended = False
+        for r in resources:
+            if not r.resource_status.endswith('FAILED'):
+                continue
+            # determine if this resources is a nested stack
+            links_rel = list([l['rel'] for l in r.links])
+            is_nested = 'nested' in links_rel
+            nested_appended = False
+            next_resource_path = list(resource_path)
+            next_resource_path.append(r.resource_name)
+            if is_nested:
+                try:
+                    nested_resources = self.heat_client.resources.list(
+                        r.physical_resource_id)
+                    nested_appended = self._append_failed_resources(
+                        failures, nested_resources, next_resource_path)
+                except exc.HTTPNotFound:
+                    # there is a failed resource but no stack
+                    pass
+            if not nested_appended:
+                failures['.'.join(next_resource_path)] = r
+            appended = True
+        return appended
+
+    def _build_software_deployments(self, resources):
+        """Build a dict of software deployments from the supplied resources.
+
+        The key is the deployment ID.
+        """
+        df = {}
+        if not resources:
+            return df
+        for r in resources.values():
+            if r.resource_type not in ('OS::Heat::StructuredDeployment',
+                                       'OS::Heat::SoftwareDeployment'):
+                continue
+            try:
+                sd = self.heat_client.software_deployments.get(
+                    deployment_id=r.physical_resource_id)
+                df[r.physical_resource_id] = sd
+            except exc.HTTPNotFound:
+                pass
+        return df
+
+    def _print_failures(self, failures, deployment_failures, long=False):
+        """Print failed resources.
+
+        If the resource is a deployment resource, look up the deployment and
+        print deploy_stdout and deploy_stderr.
+        """
+        out = self.app.stdout
+
+        if not failures:
+            return
+        for k, f in failures.items():
+            out.write('%s:\n' % k)
+            out.write('  resource_type: %s\n' % f.resource_type)
+            out.write('  physical_resource_id: %s\n' %
+                      f.physical_resource_id)
+            out.write('  status: %s\n' % f.resource_status)
+            reason = format_utils.indent_and_truncate(
+                f.resource_status_reason,
+                spaces=4,
+                truncate=not long,
+                truncate_prefix='...\n')
+            out.write('  status_reason: |\n%s\n' % reason)
+            df = deployment_failures.get(f.physical_resource_id)
+            if df:
+                for output in ('deploy_stdout', 'deploy_stderr'):
+                    format_utils.print_software_deployment_output(
+                        data=df.output_values, name=output, long=long, out=out)
diff --git a/heatclient/tests/unit/osc/v1/test_stack_failures.py b/heatclient/tests/unit/osc/v1/test_stack_failures.py
new file mode 100644
index 0000000..1ecfbf5
--- /dev/null
+++ b/heatclient/tests/unit/osc/v1/test_stack_failures.py
@@ -0,0 +1,261 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+
+import mock
+
+from heatclient import exc
+from heatclient.osc.v1 import stack_failures
+from heatclient.tests.unit.osc.v1 import fakes as orchestration_fakes
+
+
+class ListStackFailuresTest(orchestration_fakes.TestOrchestrationv1):
+
+    def setUp(self):
+        super(ListStackFailuresTest, self).setUp()
+        self.cmd = stack_failures.ListStackFailures(self.app, None)
+        self.cmd.heat_client = self.app.client_manager.orchestration
+        self.stack_client = self.app.client_manager.orchestration.stacks
+        self.resource_client = self.app.client_manager.orchestration.resources
+        self.software_deployments_client = \
+            self.app.client_manager.orchestration.software_deployments
+
+        self.stack = mock.MagicMock(id='123', status='FAILED',
+                                    stack_name='stack')
+        self.stack_client.get.return_value = self.stack
+        self.failed_template_resource = mock.MagicMock(
+            physical_resource_id='aaaa',
+            resource_type='My::TemplateResource',
+            resource_status='CREATE_FAILED',
+            links=[{'rel': 'nested'}],
+            resource_name='my_templateresource',
+            resource_status_reason='All gone Pete Tong',
+            logical_resource_id='my_templateresource',
+        )
+        self.failed_resource = mock.MagicMock(
+            physical_resource_id='cccc',
+            resource_type='OS::Nova::Server',
+            resource_status='CREATE_FAILED',
+            links=[],
+            resource_name='my_server',
+            resource_status_reason='All gone Pete Tong',
+            logical_resource_id='my_server',
+        )
+        self.other_failed_template_resource = mock.MagicMock(
+            physical_resource_id='dddd',
+            resource_type='My::OtherTemplateResource',
+            resource_status='CREATE_FAILED',
+            links=[{'rel': 'nested'}],
+            resource_name='my_othertemplateresource',
+            resource_status_reason='RPC timeout',
+            logical_resource_id='my_othertemplateresource',
+        )
+        self.working_resource = mock.MagicMock(
+            physical_resource_id='bbbb',
+            resource_type='OS::Nova::Server',
+            resource_status='CREATE_COMPLETE',
+            resource_name='my_server',
+        )
+        self.failed_deployment_resource = mock.MagicMock(
+            physical_resource_id='eeee',
+            resource_type='OS::Heat::SoftwareDeployment',
+            resource_status='CREATE_FAILED',
+            links=[],
+            resource_name='my_deployment',
+            resource_status_reason='Returned deploy_statuscode 1',
+            logical_resource_id='my_deployment',
+        )
+        self.failed_deployment = mock.MagicMock(
+            id='eeee',
+            output_values={
+                'deploy_statuscode': '1',
+                'deploy_stderr': 'It broke',
+                'deploy_stdout': ('1\n2\n3\n4\n5\n6\n7\n8\n9\n10'
+                                  '\n11\n12')
+            },
+        )
+        self.software_deployments_client.get.return_value = (
+            self.failed_deployment)
+
+    def test_build_failed_none(self):
+        self.stack = mock.MagicMock(id='123', status='COMPLETE',
+                                    stack_name='stack')
+        failures = self.cmd._build_failed_resources('stack')
+        expected = collections.OrderedDict()
+        self.assertEqual(expected, failures)
+
+    def test_build_failed_resources(self):
+        self.resource_client.list.side_effect = [[
+            # resource-list stack
+            self.failed_template_resource,
+            self.other_failed_template_resource,
+            self.working_resource,
+        ], [  # resource-list aaaa
+            self.failed_resource
+        ], [  # resource-list dddd
+        ]]
+        failures = self.cmd._build_failed_resources('stack')
+        expected = collections.OrderedDict()
+        expected['stack.my_templateresource.my_server'] = self.failed_resource
+        expected['stack.my_othertemplateresource'] = (
+            self.other_failed_template_resource)
+        self.assertEqual(expected, failures)
+
+    def test_build_failed_resources_not_found(self):
+        self.resource_client.list.side_effect = [[
+            # resource-list stack
+            self.failed_template_resource,
+            self.other_failed_template_resource,
+            self.working_resource,
+        ], exc.HTTPNotFound(), [  # resource-list dddd
+        ]]
+
+        failures = self.cmd._build_failed_resources('stack')
+        expected = collections.OrderedDict()
+        expected['stack.my_templateresource'] = self.failed_template_resource
+        expected['stack.my_othertemplateresource'] = (
+            self.other_failed_template_resource)
+        self.assertEqual(expected, failures)
+
+    def test_build_software_deployments(self):
+        resources = {
+            'stack.my_server': self.working_resource,
+            'stack.my_deployment': self.failed_deployment_resource
+        }
+        deployments = self.cmd._build_software_deployments(resources)
+        self.assertEqual({
+            'eeee': self.failed_deployment
+        }, deployments)
+
+    def test_build_software_deployments_not_found(self):
+        resources = {
+            'stack.my_server': self.working_resource,
+            'stack.my_deployment': self.failed_deployment_resource
+        }
+        self.software_deployments_client.get.side_effect = exc.HTTPNotFound()
+        deployments = self.cmd._build_software_deployments(resources)
+        self.assertEqual({}, deployments)
+
+    def test_build_software_deployments_no_resources(self):
+        resources = {}
+        self.software_deployments_client.get.side_effect = exc.HTTPNotFound()
+        deployments = self.cmd._build_software_deployments(resources)
+        self.assertEqual({}, deployments)
+
+    def test_list_stack_failures(self):
+        self.resource_client.list.side_effect = [[
+            # resource-list stack
+            self.failed_template_resource,
+            self.other_failed_template_resource,
+            self.working_resource,
+            self.failed_deployment_resource
+        ], [  # resource-list aaaa
+            self.failed_resource
+        ], [  # resource-list dddd
+        ]]
+
+        arglist = ['stack']
+        parsed_args = self.check_parser(self.cmd, arglist, [])
+        self.cmd.take_action(parsed_args)
+
+        self.assertEqual(
+            self.app.stdout.make_string(),
+            '''stack.my_templateresource.my_server:
+  resource_type: OS::Nova::Server
+  physical_resource_id: cccc
+  status: CREATE_FAILED
+  status_reason: |
+    All gone Pete Tong
+stack.my_othertemplateresource:
+  resource_type: My::OtherTemplateResource
+  physical_resource_id: dddd
+  status: CREATE_FAILED
+  status_reason: |
+    RPC timeout
+stack.my_deployment:
+  resource_type: OS::Heat::SoftwareDeployment
+  physical_resource_id: eeee
+  status: CREATE_FAILED
+  status_reason: |
+    Returned deploy_statuscode 1
+  deploy_stdout: |
+    ...
+    3
+    4
+    5
+    6
+    7
+    8
+    9
+    10
+    11
+    12
+    (truncated, view all with --long)
+  deploy_stderr: |
+    It broke
+''')
+
+    def test_list_stack_failures_long(self):
+        self.resource_client.list.side_effect = [[
+            # resource-list stack
+            self.failed_template_resource,
+            self.other_failed_template_resource,
+            self.working_resource,
+            self.failed_deployment_resource
+        ], [  # resource-list aaaa
+            self.failed_resource
+        ], [  # resource-list dddd
+        ]]
+
+        arglist = ['--long', 'stack']
+        parsed_args = self.check_parser(self.cmd, arglist, [])
+        self.cmd.take_action(parsed_args)
+
+        self.assertEqual(
+            self.app.stdout.make_string(),
+            '''stack.my_templateresource.my_server:
+  resource_type: OS::Nova::Server
+  physical_resource_id: cccc
+  status: CREATE_FAILED
+  status_reason: |
+    All gone Pete Tong
+stack.my_othertemplateresource:
+  resource_type: My::OtherTemplateResource
+  physical_resource_id: dddd
+  status: CREATE_FAILED
+  status_reason: |
+    RPC timeout
+stack.my_deployment:
+  resource_type: OS::Heat::SoftwareDeployment
+  physical_resource_id: eeee
+  status: CREATE_FAILED
+  status_reason: |
+    Returned deploy_statuscode 1
+  deploy_stdout: |
+    1
+    2
+    3
+    4
+    5
+    6
+    7
+    8
+    9
+    10
+    11
+    12
+  deploy_stderr: |
+    It broke
+''')
diff --git a/setup.cfg b/setup.cfg
index 4d6a6c5..1f19425 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -56,6 +56,7 @@ openstack.orchestration.v1 =
     stack_event_list = heatclient.osc.v1.event:ListEvent
     stack_event_show = heatclient.osc.v1.event:ShowEvent
     stack_environment_show = heatclient.osc.v1.stack:EnvironmentShowStack
+    stack_failures_list = heatclient.osc.v1.stack_failures:ListStackFailures
     stack_hook_clear = heatclient.osc.v1.stack:StackHookClear
     stack_hook_poll = heatclient.osc.v1.stack:StackHookPoll
     stack_list = heatclient.osc.v1.stack:ListStack