diff options
author | Ian Wienand <iwienand@redhat.com> | 2022-07-19 13:46:53 +1000 |
---|---|---|
committer | Ian Wienand <iwienand@redhat.com> | 2022-07-21 10:33:29 +1000 |
commit | 0c0ec6e5802259bfc7da1b75a9e5d574c2227002 (patch) | |
tree | 8832562866e7a689a8423e87ce0585f35a92b426 | |
parent | 559602910ff57f90e6478e748d17e0e298430c73 (diff) | |
download | zuul-0c0ec6e5802259bfc7da1b75a9e5d574c2227002.tar.gz |
zuul-stream: start new console for stream tests
The zuul-stream-functional tests currently run Ansible against two
hosts (node1 and node2) started by the "infrastructure" Zuul. However
the tests are working with the zuul_console running on the default
port 19885 -- which is the zuul_console started by the setup jobs at
[1].
The result of this is that this test is only talking to a zuul_console
instance started from the current executor's code, not the Zuul
checkout uder test -- i.e. changes to zuul_console.py aren't tested by
this change.
This modifies the job playbook to have another step that can start its
own zuul_console service on the two hosts running at another port
(19887). This way we can test against the zuul_console code from the
Zuul checkout. A new step is added to run a playbook overriding to
this port.
We retain the existing test (against the already running port 19885)
as a backwards compatability test; although we don't exactly know what
this is running (as it comes from OpenDev's production executors) we
want changes to the console log/stream callback to be able to run
against it, as it may represent what is running a static node.
The test results are pulled apart a bit to be more explicit, logging
and testing each run separately.
[1] https://opendev.org/zuul/zuul-jobs/src/branch/master/roles/prepare-workspace/tasks/main.yaml#L2
Change-Id: Ib11b77cfdc6c59d12807c6d9684c3e653ccad863
-rw-r--r-- | playbooks/zuul-stream/fixtures/test-stream.yaml | 13 | ||||
-rw-r--r-- | playbooks/zuul-stream/functional.yaml | 84 | ||||
-rw-r--r-- | playbooks/zuul-stream/post.yaml | 11 | ||||
-rw-r--r-- | playbooks/zuul-stream/validate.yaml | 38 |
4 files changed, 100 insertions, 46 deletions
diff --git a/playbooks/zuul-stream/fixtures/test-stream.yaml b/playbooks/zuul-stream/fixtures/test-stream.yaml index 05b75daf9..0c10304b2 100644 --- a/playbooks/zuul-stream/fixtures/test-stream.yaml +++ b/playbooks/zuul-stream/fixtures/test-stream.yaml @@ -1,3 +1,16 @@ +- name: Start zuul stream daemon + hosts: node + tasks: + + # NOTE : when new_console is set, this playbook runs with + # ZUUL_CONSOLE_PORT=19887 so that we test with the zuul_console + # from the Zuul checkout, and not the one started by test + # infrastructure. + - name: Start zuul_console on non-default port + zuul_console: + port: 19887 + when: new_console | default(false) + - name: Run some commands to show that logging works hosts: node tasks: diff --git a/playbooks/zuul-stream/functional.yaml b/playbooks/zuul-stream/functional.yaml index ee1643fbc..3b79e7b40 100644 --- a/playbooks/zuul-stream/functional.yaml +++ b/playbooks/zuul-stream/functional.yaml @@ -7,71 +7,73 @@ # the python version of the platform is changed. python_path: "/usr/local/lib/python3.10/dist-packages" - - name: Run ansible that should succeed + - name: Run ansible that should succeed against testing console command: > /usr/lib/zuul/ansible/{{ zuul_ansible_version }}/bin/ansible-playbook + -e "new_console=true" src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/test-stream.yaml environment: + # Setup by test-stream.yaml so we start a new zuul_console + # from this checkout. + ZUUL_CONSOLE_PORT: 19887 ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json" ZUUL_JOBDIR: "{{ ansible_user_dir}}" PYTHONPATH: "{{ python_path }}" - - name: Run ansible playbook that should fail + - name: Save output + shell: | + mv job-output.txt job-output-success-19887.txt + mv job-output.json job-output-success-19887.json + + # NOTE(ianw) 2022-07 : we deliberatly have this second step to run + # against the console setup by the infrastructure executor in the + # job pre playbooks as a backwards compatability sanity check. + - name: Run ansible that should succeed against extant console command: > /usr/lib/zuul/ansible/{{ zuul_ansible_version }}/bin/ansible-playbook - src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/test-stream-failure.yaml - register: failed_results - failed_when: "failed_results.rc != 2" + -e "new_console=false" + src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/test-stream.yaml environment: ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json" ZUUL_JOBDIR: "{{ ansible_user_dir}}" PYTHONPATH: "{{ python_path }}" - - name: Validate output - setupvar - shell: | - egrep "^.*\| node1 \|\s+\"setupvar\": {" job-output.txt - egrep "^.*\| node2 \|\s+\"setupvar\": {" job-output.txt - - - name: Validate output - shell task + - name: Save output shell: | - egrep "^.*\| node1 \| 1: lo:" job-output.txt - egrep "^.*\| node2 \| 1: lo:" job-output.txt + mv job-output.txt job-output-success-19885.txt + mv job-output.json job-output-success-19885.json - - name: Validate output - loop with items - shell: | - egrep "^.+\| node1 \| ok: Item: item1" job-output.txt - egrep "^.+\| node1 \| ok: Item: item2" job-output.txt - egrep "^.+\| node1 \| ok: Item: item3" job-output.txt - egrep "^.+\| node2 \| ok: Item: item1" job-output.txt - egrep "^.+\| node2 \| ok: Item: item2" job-output.txt - egrep "^.+\| node2 \| ok: Item: item3" job-output.txt + - name: Validate outputs + include_tasks: validate.yaml + loop: + - job-output-success-19887.txt + - job-output-success-19885.txt - - name: Validate output - loop with complex items - shell: | - egrep "^.+\| node1 \| ok: Item: Runtime" job-output.txt - egrep "^.+\| node2 \| ok: Item: Runtime" job-output.txt + # failure case - - name: Validate output - failed shell task - shell: | - egrep "^.+\| node1 \| Exception: Test module failure exception task" job-output.txt - egrep "^.+\| node2 \| Exception: Test module failure exception task" job-output.txt + - name: Run ansible playbook that should fail + command: > + /usr/lib/zuul/ansible/{{ zuul_ansible_version }}/bin/ansible-playbook + src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/test-stream-failure.yaml + register: failed_results + failed_when: "failed_results.rc != 2" + environment: + ZUUL_CONSOLE_PORT: 19887 + ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json" + ZUUL_JOBDIR: "{{ ansible_user_dir}}" + PYTHONPATH: "{{ python_path }}" - - name: Validate output - item loop with exception + - name: Save output shell: | - egrep "^.+\| node1 \| Exception: Test module failure exception loop" job-output.txt - egrep "^.+\| node2 \| Exception: Test module failure exception loop" job-output.txt + mv job-output.txt job-output-failure.txt + mv job-output.json job-output-failure.json - name: Validate output - failure shell task with exception shell: | - egrep "^.+\| node1 \| Exception: Test module failure exception fail-task" job-output.txt - egrep "^.+\| node2 \| Exception: Test module failure exception fail-task" job-output.txt + egrep "^.+\| node1 \| Exception: Test module failure exception fail-task" job-output-failure.txt + egrep "^.+\| node2 \| Exception: Test module failure exception fail-task" job-output-failure.txt - name: Validate output - failure item loop with exception shell: | - egrep "^.+\| node1 \| Exception: Test module failure exception fail-loop" job-output.txt - egrep "^.+\| node2 \| Exception: Test module failure exception fail-loop" job-output.txt - - - name: Validate output - binary data - shell: | - egrep "^.*\| node1 \| \\\\x80abc" job-output.txt - egrep "^.*\| node2 \| \\\\x80abc" job-output.txt + egrep "^.+\| node1 \| Exception: Test module failure exception fail-loop" job-output-failure.txt + egrep "^.+\| node2 \| Exception: Test module failure exception fail-loop" job-output-failure.txt diff --git a/playbooks/zuul-stream/post.yaml b/playbooks/zuul-stream/post.yaml index 2c717a82e..a93771881 100644 --- a/playbooks/zuul-stream/post.yaml +++ b/playbooks/zuul-stream/post.yaml @@ -10,9 +10,6 @@ state: directory delegate_to: localhost - - name: Rename job-output.txt - command: mv job-output.txt stream-job-output.txt - - name: Fetch files synchronize: src: "{{ ansible_user_dir }}/{{ item }}" @@ -21,5 +18,9 @@ with_items: - logging.json - ansible.cfg - - stream-job-output.txt - - job-output.json + - job-output-success-19887.txt + - job-output-success-19887.json + - job-output-success-19885.txt + - job-output-success-19885.json + - job-output-failure.txt + - job-output-failure.json diff --git a/playbooks/zuul-stream/validate.yaml b/playbooks/zuul-stream/validate.yaml new file mode 100644 index 000000000..73ccd873a --- /dev/null +++ b/playbooks/zuul-stream/validate.yaml @@ -0,0 +1,38 @@ +- name: Validate output - setupvar + shell: | + egrep "^.*\| node1 \|\s+\"setupvar\": {" {{ item }} + egrep "^.*\| node2 \|\s+\"setupvar\": {" {{ item }} + +- name: Validate output - shell task + shell: | + egrep "^.*\| node1 \| 1: lo:" {{ item }} + egrep "^.*\| node2 \| 1: lo:" {{ item }} + +- name: Validate output - loop with items + shell: | + egrep "^.+\| node1 \| ok: Item: item1" {{ item }} + egrep "^.+\| node1 \| ok: Item: item2" {{ item }} + egrep "^.+\| node1 \| ok: Item: item3" {{ item }} + egrep "^.+\| node2 \| ok: Item: item1" {{ item }} + egrep "^.+\| node2 \| ok: Item: item2" {{ item }} + egrep "^.+\| node2 \| ok: Item: item3" {{ item }} + +- name: Validate output - loop with complex items + shell: | + egrep "^.+\| node1 \| ok: Item: Runtime" {{ item }} + egrep "^.+\| node2 \| ok: Item: Runtime" {{ item }} + +- name: Validate output - failed shell task + shell: | + egrep "^.+\| node1 \| Exception: Test module failure exception task" {{ item }} + egrep "^.+\| node2 \| Exception: Test module failure exception task" {{ item }} + +- name: Validate output - item loop with exception + shell: | + egrep "^.+\| node1 \| Exception: Test module failure exception loop" {{ item }} + egrep "^.+\| node2 \| Exception: Test module failure exception loop" {{ item }} + +- name: Validate output - binary data + shell: | + egrep "^.*\| node1 \| \\\\x80abc" {{ item }} + egrep "^.*\| node2 \| \\\\x80abc" {{ item }} |