Merge "zuul-stream : Test against a Python 2.7 container"

6 files changed, 104 insertions, 8 deletions

diff --git a/playbooks/zuul-stream/2.7-container.yaml b/playbooks/zuul-stream/2.7-container.yaml
new file mode 100644
index 000000000..dfab08dfe
--- /dev/null
+++ b/playbooks/zuul-stream/2.7-container.yaml
@@ -0,0 +1,21 @@
+- name: Install docker
+  include_role:
+    name: ensure-docker
+
+- name: Build 2.7 container environment
+  shell: |
+    pushd {{ ansible_user_dir }}/src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/
+    cat ~/.ssh/id_rsa.pub > authorized_keys
+    docker build -f Dockerfile.py27 -t zuul_python27 .
+  args:
+    executable: /bin/bash
+
+- name: Run 2.7 container
+  shell: |
+    docker run -d -p 2022:22 -p 19887:19887 zuul_python27
+    docker ps
+
+- name: Accept host keys
+  shell: |
+    ssh-keyscan -p 2022 localhost >> ~/.ssh/known_hosts
+    ssh-keyscan -p 2022 127.0.0.1 >> ~/.ssh/known_hosts
diff --git a/playbooks/zuul-stream/create-inventory.yaml b/playbooks/zuul-stream/create-inventory.yaml
new file mode 100644
index 000000000..8ade2ab92
--- /dev/null
+++ b/playbooks/zuul-stream/create-inventory.yaml
@@ -0,0 +1,38 @@
+- name: Copy inventory
+  copy:
+    src: "{{ zuul.executor.log_root }}/zuul-info/inventory.yaml"
+    dest: "{{ ansible_user_dir }}/inventory.yaml"
+
+- name: Slurp inventory
+  slurp:
+    path: "{{ ansible_user_dir }}/inventory.yaml"
+  register: _inventory_yaml
+
+- name: Extract inventory
+  set_fact:
+    _new_inventory: "{{ _inventory_yaml['content'] | b64decode | from_yaml }}"
+
+- name: Setup new facts
+  set_fact:
+    _docker_inventory:
+      all:
+        children:
+          node:
+            hosts:
+              node3: null
+        hosts:
+          node3:
+            ansible_connection: ssh
+            ansible_host: 127.0.0.1
+            ansible_port: 2022
+            ansible_user: root
+            ansible_python_interpreter: /usr/local/bin/python2.7
+
+- name: Merge all facts
+  set_fact:
+    _new_inventory: '{{ _new_inventory | combine(_docker_inventory, recursive=True) }}'
+
+- name: Write out inventory
+  copy:
+    content: '{{ _new_inventory | to_nice_yaml }}'
+    dest: '{{ ansible_user_dir }}/inventory.yaml'
diff --git a/playbooks/zuul-stream/fixtures/Dockerfile.py27 b/playbooks/zuul-stream/fixtures/Dockerfile.py27
new file mode 100644
index 000000000..a30157b18
--- /dev/null
+++ b/playbooks/zuul-stream/fixtures/Dockerfile.py27
@@ -0,0 +1,24 @@
+FROM python:2.7.18-buster AS buster-2.7-ssh
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update \
+  && apt-get install -y dumb-init openssh-server \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir /var/run/sshd && chmod 0755 /var/run/sshd
+
+# This may or not be required to allow logins by preventing pam_loginuid
+# trying to write out audit level things that may not work in a container
+RUN sed -ri 's/session(\s+)required(\s+)pam_loginuid.so/session\1optional\2pam_loginuid.so/' /etc/pam.d/sshd
+
+RUN ssh-keygen -A -v
+
+RUN ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519
+
+COPY authorized_keys /root/.ssh/authorized_keys
+RUN chmod 0600 /root/.ssh/authorized_keys
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+CMD ["/usr/sbin/sshd", "-D", "-o", "ListenAddress=0.0.0.0" ]
diff --git a/playbooks/zuul-stream/functional.yaml b/playbooks/zuul-stream/functional.yaml
index cf60d2cf6..537f9b9d5 100644
--- a/playbooks/zuul-stream/functional.yaml
+++ b/playbooks/zuul-stream/functional.yaml
@@ -19,6 +19,7 @@
         ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
         ZUUL_JOBDIR: "{{ ansible_user_dir}}"
         PYTHONPATH: "{{ python_path }}"
+        ZUUL_CONSOLE_STREAM_LOCALHOST: 1
       register: _success_output
 
     - name: Save raw output to file
@@ -48,10 +49,13 @@
     # NOTE(ianw) 2022-07 : we deliberatly have this second step to run
     # against the console setup by the infrastructure executor in the
     # job pre playbooks as a backwards compatability sanity check.
+    # The py27 container job (node3) is not running an existing
+    # console streamer, so that will not output anything -- limit this
+    # out.
     - name: Run ansible that should succeed against extant console
       command: >
         /usr/lib/zuul/ansible/{{ zuul_ansible_version }}/bin/ansible-playbook
-        -e "new_console=false"
+        -e "new_console=false" --limit="node1,node2"
         src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/test-stream.yaml
       environment:
         ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
@@ -76,6 +80,8 @@
         - { node: 'node2', filename: 'job-output-success-19887.txt' }
         - { node: 'node1', filename: 'job-output-success-19885.txt' }
         - { node: 'node2', filename: 'job-output-success-19885.txt' }
+        # node3 only listen on 19887
+        - { node: 'node3', filename: 'job-output-success-19887.txt' }
 
     # failure case
 
@@ -100,8 +106,10 @@
       shell: |
         egrep "^.+\| node1 \| Exception: Test module failure exception fail-task" job-output-failure.txt
         egrep "^.+\| node2 \| Exception: Test module failure exception fail-task" job-output-failure.txt
+        egrep "^.+\| node3 \| Exception: Test module failure exception fail-task" job-output-failure.txt
 
     - name: Validate output - failure item loop with exception
       shell: |
         egrep "^.+\| node1 \| Exception: Test module failure exception fail-loop" job-output-failure.txt
         egrep "^.+\| node2 \| Exception: Test module failure exception fail-loop" job-output-failure.txt
+        egrep "^.+\| node3 \| Exception: Test module failure exception fail-loop" job-output-failure.txt
diff --git a/playbooks/zuul-stream/pre.yaml b/playbooks/zuul-stream/pre.yaml
index 23fae3549..9753fab85 100644
--- a/playbooks/zuul-stream/pre.yaml
+++ b/playbooks/zuul-stream/pre.yaml
@@ -9,6 +9,12 @@
 
   post_tasks:
 
+    - name: Setup 2.7 container environment
+      include_tasks: 2.7-container.yaml
+
+    - name: Setup inventory
+      include_tasks: create-inventory.yaml
+
     - name: Install pip
       shell: |+
         python3 -m pip install --upgrade pip setuptools wheel
@@ -36,11 +42,6 @@
         # venvs) and the installation fails due to conflicts.
         SETUPTOOLS_USE_DISTUTILS: stdlib
 
-    - name: Copy inventory
-      copy:
-        src: "{{ zuul.executor.log_root }}/zuul-info/inventory.yaml"
-        dest: "{{ ansible_user_dir }}/inventory.yaml"
-
     - name: Copy ansible.cfg
       template:
         src: templates/ansible.cfg.j2
diff --git a/zuul/ansible/base/callback/zuul_stream.py b/zuul/ansible/base/callback/zuul_stream.py
index 664bf8417..fccfdc85d 100644
--- a/zuul/ansible/base/callback/zuul_stream.py
+++ b/zuul/ansible/base/callback/zuul_stream.py
@@ -50,6 +50,8 @@ from zuul.ansible import logconfig
 
 LOG_STREAM_PORT = int(os.environ.get("ZUUL_CONSOLE_PORT", 19885))
 LOG_STREAM_VERSION = 0
+#
+LOG_STREAM_LOCALHOST = int(os.environ.get("ZUUL_CONSOLE_STREAM_LOCALHOST", 0))
 
 
 def zuul_filter_result(result):
@@ -320,13 +322,15 @@ class CallbackModule(default.CallbackModule):
             hosts = self._get_task_hosts(task)
             for host, inventory_hostname in hosts:
                 port = LOG_STREAM_PORT
-                if host in ('localhost', '127.0.0.1'):
+                if (host in ('localhost', '127.0.0.1') and
+                    not LOG_STREAM_LOCALHOST):
                     # Don't try to stream from localhost
                     continue
                 ip = play_vars[host].get(
                     'ansible_host', play_vars[host].get(
                         'ansible_inventory_host'))
-                if ip in ('localhost', '127.0.0.1'):
+                if (ip in ('localhost', '127.0.0.1') and
+                    not LOG_STREAM_LOCALHOST):
                     # Don't try to stream from localhost
                     continue
                 if play_vars[host].get('ansible_connection') in ('winrm',):