zuul-stream : Test against a Python 2.7 container

Change Ief366c092e05fb88351782f6d9cd280bfae96237 intoduced a bug in
the streaming daemons because it was using Python 3.6 features.  The
streaming console needs to work on all Ansible managed nodes, which
includes back to Python 2.7 nodes (while Ansible supports that).

This introduces a regression test by building about the smallest
Python 2.7 container that can be managed by Ansbile.  We start this
container and modify the test inventory to include it, then run the
stream tests against it.

The existing testing runs against the "new" console but also tests
against the console OpenDev's Zuul starts to ensure
backwards-compatability.  Since this container wasn't started by Zuul
it doesn't have this, so that testing is skipped for this node.

It might be good to abstract all testing of the console daemons into
separate containers for each Ansible supported managed-node Python
version -- it's a bit more work than I want to take on right now.
This should ensure the lower-bound though and prevent regressions for
older platforms.

Change-Id: Ia78ad9e3ec51bc47bf68c9ff38c0fcd16ba2e728

5 files changed, 98 insertions, 6 deletions

diff --git a/playbooks/zuul-stream/2.7-container.yaml b/playbooks/zuul-stream/2.7-container.yaml
new file mode 100644
index 000000000..dfab08dfe
--- /dev/null
+++ b/playbooks/zuul-stream/2.7-container.yaml
@@ -0,0 +1,21 @@
+- name: Install docker
+  include_role:
+    name: ensure-docker
+
+- name: Build 2.7 container environment
+  shell: |
+    pushd {{ ansible_user_dir }}/src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/
+    cat ~/.ssh/id_rsa.pub > authorized_keys
+    docker build -f Dockerfile.py27 -t zuul_python27 .
+  args:
+    executable: /bin/bash
+
+- name: Run 2.7 container
+  shell: |
+    docker run -d -p 2022:22 -p 19887:19887 zuul_python27
+    docker ps
+
+- name: Accept host keys
+  shell: |
+    ssh-keyscan -p 2022 localhost >> ~/.ssh/known_hosts
+    ssh-keyscan -p 2022 127.0.0.1 >> ~/.ssh/known_hosts
diff --git a/playbooks/zuul-stream/create-inventory.yaml b/playbooks/zuul-stream/create-inventory.yaml
new file mode 100644
index 000000000..8ade2ab92
--- /dev/null
+++ b/playbooks/zuul-stream/create-inventory.yaml
@@ -0,0 +1,38 @@
+- name: Copy inventory
+  copy:
+    src: "{{ zuul.executor.log_root }}/zuul-info/inventory.yaml"
+    dest: "{{ ansible_user_dir }}/inventory.yaml"
+
+- name: Slurp inventory
+  slurp:
+    path: "{{ ansible_user_dir }}/inventory.yaml"
+  register: _inventory_yaml
+
+- name: Extract inventory
+  set_fact:
+    _new_inventory: "{{ _inventory_yaml['content'] | b64decode | from_yaml }}"
+
+- name: Setup new facts
+  set_fact:
+    _docker_inventory:
+      all:
+        children:
+          node:
+            hosts:
+              node3: null
+        hosts:
+          node3:
+            ansible_connection: ssh
+            ansible_host: 127.0.0.1
+            ansible_port: 2022
+            ansible_user: root
+            ansible_python_interpreter: /usr/local/bin/python2.7
+
+- name: Merge all facts
+  set_fact:
+    _new_inventory: '{{ _new_inventory | combine(_docker_inventory, recursive=True) }}'
+
+- name: Write out inventory
+  copy:
+    content: '{{ _new_inventory | to_nice_yaml }}'
+    dest: '{{ ansible_user_dir }}/inventory.yaml'
diff --git a/playbooks/zuul-stream/fixtures/Dockerfile.py27 b/playbooks/zuul-stream/fixtures/Dockerfile.py27
new file mode 100644
index 000000000..a30157b18
--- /dev/null
+++ b/playbooks/zuul-stream/fixtures/Dockerfile.py27
@@ -0,0 +1,24 @@
+FROM python:2.7.18-buster AS buster-2.7-ssh
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update \
+  && apt-get install -y dumb-init openssh-server \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir /var/run/sshd && chmod 0755 /var/run/sshd
+
+# This may or not be required to allow logins by preventing pam_loginuid
+# trying to write out audit level things that may not work in a container
+RUN sed -ri 's/session(\s+)required(\s+)pam_loginuid.so/session\1optional\2pam_loginuid.so/' /etc/pam.d/sshd
+
+RUN ssh-keygen -A -v
+
+RUN ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519
+
+COPY authorized_keys /root/.ssh/authorized_keys
+RUN chmod 0600 /root/.ssh/authorized_keys
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+CMD ["/usr/sbin/sshd", "-D", "-o", "ListenAddress=0.0.0.0" ]
diff --git a/playbooks/zuul-stream/functional.yaml b/playbooks/zuul-stream/functional.yaml
index cf60d2cf6..537f9b9d5 100644
--- a/playbooks/zuul-stream/functional.yaml
+++ b/playbooks/zuul-stream/functional.yaml
@@ -19,6 +19,7 @@
         ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
         ZUUL_JOBDIR: "{{ ansible_user_dir}}"
         PYTHONPATH: "{{ python_path }}"
+        ZUUL_CONSOLE_STREAM_LOCALHOST: 1
       register: _success_output
 
     - name: Save raw output to file
@@ -48,10 +49,13 @@
     # NOTE(ianw) 2022-07 : we deliberatly have this second step to run
     # against the console setup by the infrastructure executor in the
     # job pre playbooks as a backwards compatability sanity check.
+    # The py27 container job (node3) is not running an existing
+    # console streamer, so that will not output anything -- limit this
+    # out.
     - name: Run ansible that should succeed against extant console
       command: >
         /usr/lib/zuul/ansible/{{ zuul_ansible_version }}/bin/ansible-playbook
-        -e "new_console=false"
+        -e "new_console=false" --limit="node1,node2"
         src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/test-stream.yaml
       environment:
         ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
@@ -76,6 +80,8 @@
         - { node: 'node2', filename: 'job-output-success-19887.txt' }
         - { node: 'node1', filename: 'job-output-success-19885.txt' }
         - { node: 'node2', filename: 'job-output-success-19885.txt' }
+        # node3 only listen on 19887
+        - { node: 'node3', filename: 'job-output-success-19887.txt' }
 
     # failure case
 
@@ -100,8 +106,10 @@
       shell: |
         egrep "^.+\| node1 \| Exception: Test module failure exception fail-task" job-output-failure.txt
         egrep "^.+\| node2 \| Exception: Test module failure exception fail-task" job-output-failure.txt
+        egrep "^.+\| node3 \| Exception: Test module failure exception fail-task" job-output-failure.txt
 
     - name: Validate output - failure item loop with exception
       shell: |
         egrep "^.+\| node1 \| Exception: Test module failure exception fail-loop" job-output-failure.txt
         egrep "^.+\| node2 \| Exception: Test module failure exception fail-loop" job-output-failure.txt
+        egrep "^.+\| node3 \| Exception: Test module failure exception fail-loop" job-output-failure.txt
diff --git a/playbooks/zuul-stream/pre.yaml b/playbooks/zuul-stream/pre.yaml
index 23fae3549..9753fab85 100644
--- a/playbooks/zuul-stream/pre.yaml
+++ b/playbooks/zuul-stream/pre.yaml
@@ -9,6 +9,12 @@
 
   post_tasks:
 
+    - name: Setup 2.7 container environment
+      include_tasks: 2.7-container.yaml
+
+    - name: Setup inventory
+      include_tasks: create-inventory.yaml
+
     - name: Install pip
       shell: |+
         python3 -m pip install --upgrade pip setuptools wheel
@@ -36,11 +42,6 @@
         # venvs) and the installation fails due to conflicts.
         SETUPTOOLS_USE_DISTUTILS: stdlib
 
-    - name: Copy inventory
-      copy:
-        src: "{{ zuul.executor.log_root }}/zuul-info/inventory.yaml"
-        dest: "{{ ansible_user_dir }}/inventory.yaml"
-
     - name: Copy ansible.cfg
       template:
         src: templates/ansible.cfg.j2