zuul-stream: start new console for stream tests

The zuul-stream-functional tests currently run Ansible against two
hosts (node1 and node2) started by the "infrastructure" Zuul.  However
the tests are working with the zuul_console running on the default
port 19885 -- which is the zuul_console started by the setup jobs at
[1].

The result of this is that this test is only talking to a zuul_console
instance started from the current executor's code, not the Zuul
checkout uder test -- i.e. changes to zuul_console.py aren't tested by
this change.

This modifies the job playbook to have another step that can start its
own zuul_console service on the two hosts running at another port
(19887).  This way we can test against the zuul_console code from the
Zuul checkout.  A new step is added to run a playbook overriding to
this port.

We retain the existing test (against the already running port 19885)
as a backwards compatability test; although we don't exactly know what
this is running (as it comes from OpenDev's production executors) we
want changes to the console log/stream callback to be able to run
against it, as it may represent what is running a static node.

The test results are pulled apart a bit to be more explicit, logging
and testing each run separately.

[1] https://opendev.org/zuul/zuul-jobs/src/branch/master/roles/prepare-workspace/tasks/main.yaml#L2

Change-Id: Ib11b77cfdc6c59d12807c6d9684c3e653ccad863

4 files changed, 100 insertions, 46 deletions

diff --git a/playbooks/zuul-stream/fixtures/test-stream.yaml b/playbooks/zuul-stream/fixtures/test-stream.yaml
index 05b75daf9..0c10304b2 100644
--- a/playbooks/zuul-stream/fixtures/test-stream.yaml
+++ b/playbooks/zuul-stream/fixtures/test-stream.yaml
@@ -1,3 +1,16 @@
+- name: Start zuul stream daemon
+  hosts: node
+  tasks:
+
+    # NOTE : when new_console is set, this playbook runs with
+    # ZUUL_CONSOLE_PORT=19887 so that we test with the zuul_console
+    # from the Zuul checkout, and not the one started by test
+    # infrastructure.
+    - name: Start zuul_console on non-default port
+      zuul_console:
+        port: 19887
+      when: new_console | default(false)
+
 - name: Run some commands to show that logging works
   hosts: node
   tasks:
diff --git a/playbooks/zuul-stream/functional.yaml b/playbooks/zuul-stream/functional.yaml
index ee1643fbc..3b79e7b40 100644
--- a/playbooks/zuul-stream/functional.yaml
+++ b/playbooks/zuul-stream/functional.yaml
@@ -7,71 +7,73 @@
         # the python version of the platform is changed.
         python_path: "/usr/local/lib/python3.10/dist-packages"
 
-    - name: Run ansible that should succeed
+    - name: Run ansible that should succeed against testing console
       command: >
         /usr/lib/zuul/ansible/{{ zuul_ansible_version }}/bin/ansible-playbook
+        -e "new_console=true"
         src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/test-stream.yaml
       environment:
+        # Setup by test-stream.yaml so we start a new zuul_console
+        # from this checkout.
+        ZUUL_CONSOLE_PORT: 19887
         ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
         ZUUL_JOBDIR: "{{ ansible_user_dir}}"
         PYTHONPATH: "{{ python_path }}"
 
-    - name: Run ansible playbook that should fail
+    - name: Save output
+      shell: |
+          mv job-output.txt job-output-success-19887.txt
+          mv job-output.json job-output-success-19887.json
+
+    # NOTE(ianw) 2022-07 : we deliberatly have this second step to run
+    # against the console setup by the infrastructure executor in the
+    # job pre playbooks as a backwards compatability sanity check.
+    - name: Run ansible that should succeed against extant console
       command: >
         /usr/lib/zuul/ansible/{{ zuul_ansible_version }}/bin/ansible-playbook
-        src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/test-stream-failure.yaml
-      register: failed_results
-      failed_when: "failed_results.rc != 2"
+        -e "new_console=false"
+        src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/test-stream.yaml
       environment:
         ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
         ZUUL_JOBDIR: "{{ ansible_user_dir}}"
         PYTHONPATH: "{{ python_path }}"
 
-    - name: Validate output - setupvar
-      shell: |
-        egrep "^.*\| node1 \|\s+\"setupvar\": {" job-output.txt
-        egrep "^.*\| node2 \|\s+\"setupvar\": {" job-output.txt
-
-    - name: Validate output - shell task
+    - name: Save output
       shell: |
-        egrep "^.*\| node1 \| 1: lo:" job-output.txt
-        egrep "^.*\| node2 \| 1: lo:" job-output.txt
+          mv job-output.txt job-output-success-19885.txt
+          mv job-output.json job-output-success-19885.json
 
-    - name: Validate output - loop with items
-      shell: |
-        egrep "^.+\| node1 \| ok: Item: item1" job-output.txt
-        egrep "^.+\| node1 \| ok: Item: item2" job-output.txt
-        egrep "^.+\| node1 \| ok: Item: item3" job-output.txt
-        egrep "^.+\| node2 \| ok: Item: item1" job-output.txt
-        egrep "^.+\| node2 \| ok: Item: item2" job-output.txt
-        egrep "^.+\| node2 \| ok: Item: item3" job-output.txt
+    - name: Validate outputs
+      include_tasks: validate.yaml
+      loop:
+        - job-output-success-19887.txt
+        - job-output-success-19885.txt
 
-    - name: Validate output - loop with complex items
-      shell: |
-        egrep "^.+\| node1 \| ok: Item: Runtime" job-output.txt
-        egrep "^.+\| node2 \| ok: Item: Runtime" job-output.txt
+    # failure case
 
-    - name: Validate output - failed shell task
-      shell: |
-        egrep "^.+\| node1 \| Exception: Test module failure exception task" job-output.txt
-        egrep "^.+\| node2 \| Exception: Test module failure exception task" job-output.txt
+    - name: Run ansible playbook that should fail
+      command: >
+        /usr/lib/zuul/ansible/{{ zuul_ansible_version }}/bin/ansible-playbook
+        src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/test-stream-failure.yaml
+      register: failed_results
+      failed_when: "failed_results.rc != 2"
+      environment:
+        ZUUL_CONSOLE_PORT: 19887
+        ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
+        ZUUL_JOBDIR: "{{ ansible_user_dir}}"
+        PYTHONPATH: "{{ python_path }}"
 
-    - name: Validate output - item loop with exception
+    - name: Save output
       shell: |
-        egrep "^.+\| node1 \| Exception: Test module failure exception loop" job-output.txt
-        egrep "^.+\| node2 \| Exception: Test module failure exception loop" job-output.txt
+          mv job-output.txt job-output-failure.txt
+          mv job-output.json job-output-failure.json
 
     - name: Validate output - failure shell task with exception
       shell: |
-        egrep "^.+\| node1 \| Exception: Test module failure exception fail-task" job-output.txt
-        egrep "^.+\| node2 \| Exception: Test module failure exception fail-task" job-output.txt
+        egrep "^.+\| node1 \| Exception: Test module failure exception fail-task" job-output-failure.txt
+        egrep "^.+\| node2 \| Exception: Test module failure exception fail-task" job-output-failure.txt
 
     - name: Validate output - failure item loop with exception
       shell: |
-        egrep "^.+\| node1 \| Exception: Test module failure exception fail-loop" job-output.txt
-        egrep "^.+\| node2 \| Exception: Test module failure exception fail-loop" job-output.txt
-
-    - name: Validate output - binary data
-      shell: |
-        egrep "^.*\| node1 \| \\\\x80abc" job-output.txt
-        egrep "^.*\| node2 \| \\\\x80abc" job-output.txt
+        egrep "^.+\| node1 \| Exception: Test module failure exception fail-loop" job-output-failure.txt
+        egrep "^.+\| node2 \| Exception: Test module failure exception fail-loop" job-output-failure.txt
diff --git a/playbooks/zuul-stream/post.yaml b/playbooks/zuul-stream/post.yaml
index 2c717a82e..a93771881 100644
--- a/playbooks/zuul-stream/post.yaml
+++ b/playbooks/zuul-stream/post.yaml
@@ -10,9 +10,6 @@
         state: directory
       delegate_to: localhost
 
-    - name: Rename job-output.txt
-      command: mv job-output.txt stream-job-output.txt
-
     - name: Fetch files
       synchronize:
         src: "{{ ansible_user_dir }}/{{ item }}"
@@ -21,5 +18,9 @@
       with_items:
         - logging.json
         - ansible.cfg
-        - stream-job-output.txt
-        - job-output.json
+        - job-output-success-19887.txt
+        - job-output-success-19887.json
+        - job-output-success-19885.txt
+        - job-output-success-19885.json
+        - job-output-failure.txt
+        - job-output-failure.json
diff --git a/playbooks/zuul-stream/validate.yaml b/playbooks/zuul-stream/validate.yaml
new file mode 100644
index 000000000..73ccd873a
--- /dev/null
+++ b/playbooks/zuul-stream/validate.yaml
@@ -0,0 +1,38 @@
+- name: Validate output - setupvar
+  shell: |
+    egrep "^.*\| node1 \|\s+\"setupvar\": {" {{ item }}
+    egrep "^.*\| node2 \|\s+\"setupvar\": {" {{ item }}
+
+- name: Validate output - shell task
+  shell: |
+    egrep "^.*\| node1 \| 1: lo:" {{ item }}
+    egrep "^.*\| node2 \| 1: lo:" {{ item }}
+
+- name: Validate output - loop with items
+  shell: |
+    egrep "^.+\| node1 \| ok: Item: item1" {{ item }}
+    egrep "^.+\| node1 \| ok: Item: item2" {{ item }}
+    egrep "^.+\| node1 \| ok: Item: item3" {{ item }}
+    egrep "^.+\| node2 \| ok: Item: item1" {{ item }}
+    egrep "^.+\| node2 \| ok: Item: item2" {{ item }}
+    egrep "^.+\| node2 \| ok: Item: item3" {{ item }}
+
+- name: Validate output - loop with complex items
+  shell: |
+    egrep "^.+\| node1 \| ok: Item: Runtime" {{ item }}
+    egrep "^.+\| node2 \| ok: Item: Runtime" {{ item }}
+
+- name: Validate output - failed shell task
+  shell: |
+    egrep "^.+\| node1 \| Exception: Test module failure exception task" {{ item }}
+    egrep "^.+\| node2 \| Exception: Test module failure exception task" {{ item }}
+
+- name: Validate output - item loop with exception
+  shell: |
+    egrep "^.+\| node1 \| Exception: Test module failure exception loop" {{ item }}
+    egrep "^.+\| node2 \| Exception: Test module failure exception loop" {{ item }}
+
+- name: Validate output - binary data
+  shell: |
+    egrep "^.*\| node1 \| \\\\x80abc" {{ item }}
+    egrep "^.*\| node2 \| \\\\x80abc" {{ item }}