diff options
-rw-r--r-- | playbooks/zuul-stream/2.7-container.yaml | 21 | ||||
-rw-r--r-- | playbooks/zuul-stream/create-inventory.yaml | 38 | ||||
-rw-r--r-- | playbooks/zuul-stream/fixtures/Dockerfile.py27 | 24 | ||||
-rw-r--r-- | playbooks/zuul-stream/functional.yaml | 10 | ||||
-rw-r--r-- | playbooks/zuul-stream/pre.yaml | 11 | ||||
-rw-r--r-- | zuul/ansible/base/callback/zuul_stream.py | 8 |
6 files changed, 104 insertions, 8 deletions
diff --git a/playbooks/zuul-stream/2.7-container.yaml b/playbooks/zuul-stream/2.7-container.yaml new file mode 100644 index 000000000..dfab08dfe --- /dev/null +++ b/playbooks/zuul-stream/2.7-container.yaml @@ -0,0 +1,21 @@ +- name: Install docker + include_role: + name: ensure-docker + +- name: Build 2.7 container environment + shell: | + pushd {{ ansible_user_dir }}/src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/ + cat ~/.ssh/id_rsa.pub > authorized_keys + docker build -f Dockerfile.py27 -t zuul_python27 . + args: + executable: /bin/bash + +- name: Run 2.7 container + shell: | + docker run -d -p 2022:22 -p 19887:19887 zuul_python27 + docker ps + +- name: Accept host keys + shell: | + ssh-keyscan -p 2022 localhost >> ~/.ssh/known_hosts + ssh-keyscan -p 2022 127.0.0.1 >> ~/.ssh/known_hosts diff --git a/playbooks/zuul-stream/create-inventory.yaml b/playbooks/zuul-stream/create-inventory.yaml new file mode 100644 index 000000000..8ade2ab92 --- /dev/null +++ b/playbooks/zuul-stream/create-inventory.yaml @@ -0,0 +1,38 @@ +- name: Copy inventory + copy: + src: "{{ zuul.executor.log_root }}/zuul-info/inventory.yaml" + dest: "{{ ansible_user_dir }}/inventory.yaml" + +- name: Slurp inventory + slurp: + path: "{{ ansible_user_dir }}/inventory.yaml" + register: _inventory_yaml + +- name: Extract inventory + set_fact: + _new_inventory: "{{ _inventory_yaml['content'] | b64decode | from_yaml }}" + +- name: Setup new facts + set_fact: + _docker_inventory: + all: + children: + node: + hosts: + node3: null + hosts: + node3: + ansible_connection: ssh + ansible_host: 127.0.0.1 + ansible_port: 2022 + ansible_user: root + ansible_python_interpreter: /usr/local/bin/python2.7 + +- name: Merge all facts + set_fact: + _new_inventory: '{{ _new_inventory | combine(_docker_inventory, recursive=True) }}' + +- name: Write out inventory + copy: + content: '{{ _new_inventory | to_nice_yaml }}' + dest: '{{ ansible_user_dir }}/inventory.yaml' diff --git a/playbooks/zuul-stream/fixtures/Dockerfile.py27 b/playbooks/zuul-stream/fixtures/Dockerfile.py27 new file mode 100644 index 000000000..a30157b18 --- /dev/null +++ b/playbooks/zuul-stream/fixtures/Dockerfile.py27 @@ -0,0 +1,24 @@ +FROM python:2.7.18-buster AS buster-2.7-ssh + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update \ + && apt-get install -y dumb-init openssh-server \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir /var/run/sshd && chmod 0755 /var/run/sshd + +# This may or not be required to allow logins by preventing pam_loginuid +# trying to write out audit level things that may not work in a container +RUN sed -ri 's/session(\s+)required(\s+)pam_loginuid.so/session\1optional\2pam_loginuid.so/' /etc/pam.d/sshd + +RUN ssh-keygen -A -v + +RUN ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519 + +COPY authorized_keys /root/.ssh/authorized_keys +RUN chmod 0600 /root/.ssh/authorized_keys + +ENTRYPOINT ["/usr/bin/dumb-init", "--"] +CMD ["/usr/sbin/sshd", "-D", "-o", "ListenAddress=0.0.0.0" ] diff --git a/playbooks/zuul-stream/functional.yaml b/playbooks/zuul-stream/functional.yaml index cf60d2cf6..537f9b9d5 100644 --- a/playbooks/zuul-stream/functional.yaml +++ b/playbooks/zuul-stream/functional.yaml @@ -19,6 +19,7 @@ ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json" ZUUL_JOBDIR: "{{ ansible_user_dir}}" PYTHONPATH: "{{ python_path }}" + ZUUL_CONSOLE_STREAM_LOCALHOST: 1 register: _success_output - name: Save raw output to file @@ -48,10 +49,13 @@ # NOTE(ianw) 2022-07 : we deliberatly have this second step to run # against the console setup by the infrastructure executor in the # job pre playbooks as a backwards compatability sanity check. + # The py27 container job (node3) is not running an existing + # console streamer, so that will not output anything -- limit this + # out. - name: Run ansible that should succeed against extant console command: > /usr/lib/zuul/ansible/{{ zuul_ansible_version }}/bin/ansible-playbook - -e "new_console=false" + -e "new_console=false" --limit="node1,node2" src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/test-stream.yaml environment: ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json" @@ -76,6 +80,8 @@ - { node: 'node2', filename: 'job-output-success-19887.txt' } - { node: 'node1', filename: 'job-output-success-19885.txt' } - { node: 'node2', filename: 'job-output-success-19885.txt' } + # node3 only listen on 19887 + - { node: 'node3', filename: 'job-output-success-19887.txt' } # failure case @@ -100,8 +106,10 @@ shell: | egrep "^.+\| node1 \| Exception: Test module failure exception fail-task" job-output-failure.txt egrep "^.+\| node2 \| Exception: Test module failure exception fail-task" job-output-failure.txt + egrep "^.+\| node3 \| Exception: Test module failure exception fail-task" job-output-failure.txt - name: Validate output - failure item loop with exception shell: | egrep "^.+\| node1 \| Exception: Test module failure exception fail-loop" job-output-failure.txt egrep "^.+\| node2 \| Exception: Test module failure exception fail-loop" job-output-failure.txt + egrep "^.+\| node3 \| Exception: Test module failure exception fail-loop" job-output-failure.txt diff --git a/playbooks/zuul-stream/pre.yaml b/playbooks/zuul-stream/pre.yaml index 23fae3549..9753fab85 100644 --- a/playbooks/zuul-stream/pre.yaml +++ b/playbooks/zuul-stream/pre.yaml @@ -9,6 +9,12 @@ post_tasks: + - name: Setup 2.7 container environment + include_tasks: 2.7-container.yaml + + - name: Setup inventory + include_tasks: create-inventory.yaml + - name: Install pip shell: |+ python3 -m pip install --upgrade pip setuptools wheel @@ -36,11 +42,6 @@ # venvs) and the installation fails due to conflicts. SETUPTOOLS_USE_DISTUTILS: stdlib - - name: Copy inventory - copy: - src: "{{ zuul.executor.log_root }}/zuul-info/inventory.yaml" - dest: "{{ ansible_user_dir }}/inventory.yaml" - - name: Copy ansible.cfg template: src: templates/ansible.cfg.j2 diff --git a/zuul/ansible/base/callback/zuul_stream.py b/zuul/ansible/base/callback/zuul_stream.py index 664bf8417..fccfdc85d 100644 --- a/zuul/ansible/base/callback/zuul_stream.py +++ b/zuul/ansible/base/callback/zuul_stream.py @@ -50,6 +50,8 @@ from zuul.ansible import logconfig LOG_STREAM_PORT = int(os.environ.get("ZUUL_CONSOLE_PORT", 19885)) LOG_STREAM_VERSION = 0 +# +LOG_STREAM_LOCALHOST = int(os.environ.get("ZUUL_CONSOLE_STREAM_LOCALHOST", 0)) def zuul_filter_result(result): @@ -320,13 +322,15 @@ class CallbackModule(default.CallbackModule): hosts = self._get_task_hosts(task) for host, inventory_hostname in hosts: port = LOG_STREAM_PORT - if host in ('localhost', '127.0.0.1'): + if (host in ('localhost', '127.0.0.1') and + not LOG_STREAM_LOCALHOST): # Don't try to stream from localhost continue ip = play_vars[host].get( 'ansible_host', play_vars[host].get( 'ansible_inventory_host')) - if ip in ('localhost', '127.0.0.1'): + if (ip in ('localhost', '127.0.0.1') and + not LOG_STREAM_LOCALHOST): # Don't try to stream from localhost continue if play_vars[host].get('ansible_connection') in ('winrm',): |