From 619999b6db425e4710a56db6d174e6df0c6c1a34 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Wed, 24 Aug 2022 06:06:53 +1000 Subject: zuul-tox-remote: capture callback errors in remote zuul jobs We have a couple of places that can trigger errors in the zuul_stream callback that this testing currently misses. To try and catch this case better we grab the Ansible console output (where the failure of the callback plugin is noted) and check it for Ansible failure strings. Just for review purposes, follow-on changes will correct current errors so this test can be inverted. Change-Id: I9f569a411729f8a067de17d99ef6b9d74fc21543 --- tests/remote/test_remote_zuul_stream.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/remote/test_remote_zuul_stream.py b/tests/remote/test_remote_zuul_stream.py index 1c705127e..c163b3488 100644 --- a/tests/remote/test_remote_zuul_stream.py +++ b/tests/remote/test_remote_zuul_stream.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. +import io +import logging import os import re import textwrap @@ -31,6 +33,12 @@ class FunctionalZuulStreamMixIn: self.executor_server.log_console_port = self.log_console_port self.wait_timeout = 180 self.fake_nodepool.remote_ansible = True + # This catches the Ansible output; rather than the callback + # output captured in the job log. For example if the callback + # fails, there will be an error output in this stream. + self.logger = logging.getLogger('zuul.AnsibleJob') + self.console_output = io.StringIO() + self.logger.addHandler(logging.StreamHandler(self.console_output)) ansible_remote = os.environ.get('ZUUL_REMOTE_IPV4') self.assertIsNotNone(ansible_remote) @@ -120,6 +128,21 @@ class FunctionalZuulStreamMixIn: build = self.history[-1] self.assertEqual(build.result, 'SUCCESS') + console_output = self.console_output.getvalue() + # This should be generic enough to match any callback + # plugin failures, which look something like + # + # [WARNING]: Failure using method (v2_runner_on_ok) in \ + # callback plugin + # (): 'dict' object has no attribute 'startswith'" + # Callback Exception: + # ... + # + # NOTE(ianw) 2022-08-24 : needs a couple of fixes before + # we can turn on. + self.assertIn('[WARNING]: Failure using method', console_output) + text = self._get_job_output(build) self.assertLogLine( r'RUN START: \[untrusted : review.example.com/org/project/' -- cgit v1.2.1