zuul-tox-remote: capture callback errors in remote zuul jobs

We have a couple of places that can trigger errors in the zuul_stream
callback that this testing currently misses.

To try and catch this case better we grab the Ansible console output
(where the failure of the callback plugin is noted) and check it for
Ansible failure strings.

Just for review purposes, follow-on changes will correct current
errors so this test can be inverted.

Change-Id: I9f569a411729f8a067de17d99ef6b9d74fc21543

1 files changed, 23 insertions, 0 deletions

diff --git a/tests/remote/test_remote_zuul_stream.py b/tests/remote/test_remote_zuul_stream.py
index 1c705127e..c163b3488 100644
--- a/tests/remote/test_remote_zuul_stream.py
+++ b/tests/remote/test_remote_zuul_stream.py
@@ -12,6 +12,8 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import io
+import logging
 import os
 import re
 import textwrap
@@ -31,6 +33,12 @@ class FunctionalZuulStreamMixIn:
         self.executor_server.log_console_port = self.log_console_port
         self.wait_timeout = 180
         self.fake_nodepool.remote_ansible = True
+        # This catches the Ansible output; rather than the callback
+        # output captured in the job log.  For example if the callback
+        # fails, there will be an error output in this stream.
+        self.logger = logging.getLogger('zuul.AnsibleJob')
+        self.console_output = io.StringIO()
+        self.logger.addHandler(logging.StreamHandler(self.console_output))
 
         ansible_remote = os.environ.get('ZUUL_REMOTE_IPV4')
         self.assertIsNotNone(ansible_remote)
@@ -120,6 +128,21 @@ class FunctionalZuulStreamMixIn:
             build = self.history[-1]
             self.assertEqual(build.result, 'SUCCESS')
 
+            console_output = self.console_output.getvalue()
+            # This should be generic enough to match any callback
+            # plugin failures, which look something like
+            #
+            #  [WARNING]: Failure using method (v2_runner_on_ok) in \
+            #                                                  callback plugin
+            #  (<ansible.plugins.callback.zuul_stream.CallbackModule object at'
+            #  0x7f89f72a20b0>): 'dict' object has no attribute 'startswith'"
+            #  Callback Exception:
+            #  ...
+            #
+            # NOTE(ianw) 2022-08-24 : needs a couple of fixes before
+            # we can turn on.
+            self.assertIn('[WARNING]: Failure using method', console_output)
+
             text = self._get_job_output(build)
             self.assertLogLine(
                 r'RUN START: \[untrusted : review.example.com/org/project/'