Merge "zuul-stream: Add variable to disable writing streaming files"

5 files changed, 56 insertions, 13 deletions

diff --git a/doc/source/howtos/nodepool_static.rst b/doc/source/howtos/nodepool_static.rst
index c10672e7b..163497bcf 100644
--- a/doc/source/howtos/nodepool_static.rst
+++ b/doc/source/howtos/nodepool_static.rst
@@ -69,14 +69,30 @@ Log streaming
 The log streaming service enables Zuul to show the live status of
 long-running ``shell`` or ``command`` tasks.  The server side is setup
 by the ``zuul_console:`` task built-in to Zuul's Ansible installation.
-The executor requires the ability to communicate with the job nodes on
-port 19885 for this to work.
+The executor requires the ability to communicate with this server on
+the job nodes via port ``19885`` for this to work.
 
-The log streaming service may leave files on the static node in the
-format ``/tmp/console-<uuid>-<task_id>-<host>.log`` if jobs are
-interrupted.  These may be safely removed after a short period of
-inactivity with a command such as
+The log streaming service spools command output via files on the job
+node in the format ``/tmp/console-<uuid>-<task_id>-<host>.log``.  By
+default, it will clean these files up automatically.
+
+Occasionally, a streaming file may be left if a job is interrupted.
+These may be safely removed after a short period of inactivity with a
+command such as
 
 .. code-block:: shell
 
    find /tmp -maxdepth 1 -name 'console-*-*-<host>.log' -mtime +2 -delete
+
+If the executor is unable to reach port ``19885`` (for example due to
+firewall rules), or the ``zuul_console`` daemon can not be run for
+some other reason, the command to clean these spool files will not be
+processed and they may be left behind; on an ephemeral node this is
+not usually a problem, but on a static node these files will persist.
+
+In this situation, , Zuul can be instructed to not to create any spool
+files for ``shell`` and ``command`` tasks via setting
+``zuul_console_disable: True`` (usually via a global host variable in
+inventory).  Live streaming of ``shell`` and ``command`` calls will of
+course be unavailable in this case, but no spool files will be
+created.
diff --git a/playbooks/zuul-stream/fixtures/test-stream.yaml b/playbooks/zuul-stream/fixtures/test-stream.yaml
index 0326ae54e..488f8cb2f 100644
--- a/playbooks/zuul-stream/fixtures/test-stream.yaml
+++ b/playbooks/zuul-stream/fixtures/test-stream.yaml
@@ -11,6 +11,14 @@
         port: 19887
       when: new_console | default(false)
 
+- name: Run command to show skipping works
+  vars:
+    zuul_console_disabled: true
+  hosts: node
+  tasks:
+    - name: Run quiet command
+      command: echo 'This command should not stream'
+
 - name: Run some commands to show that logging works
   hosts: node
   tasks:
diff --git a/playbooks/zuul-stream/functional.yaml b/playbooks/zuul-stream/functional.yaml
index 63e13e3f5..fa523f7c2 100644
--- a/playbooks/zuul-stream/functional.yaml
+++ b/playbooks/zuul-stream/functional.yaml
@@ -72,7 +72,7 @@
           mv job-output.txt job-output-success-19885.txt
           mv job-output.json job-output-success-19885.json
 
-    - name: Validate outputs
+    - name: Validate text outputs
       include_tasks: validate.yaml
       loop:
         - { node: 'node1', filename: 'job-output-success-19887.txt' }
@@ -82,6 +82,13 @@
         # node3 only listen on 19887
         - { node: 'node3', filename: 'job-output-success-19887.txt' }
 
+    # This shows that zuul_console_disabled has activated and set the
+    # UUID to "skip"
+    - name: Validate json output
+      shell: |
+        egrep 'zuul_log_id": "skip"' job-output-success-19885.json
+        egrep 'zuul_log_id": "skip"' job-output-success-19887.json
+
     # failure case
 
     - name: Run ansible playbook that should fail
diff --git a/zuul/ansible/base/action/command.py b/zuul/ansible/base/action/command.py
index 3bb88d8e5..a969a8b2b 100644
--- a/zuul/ansible/base/action/command.py
+++ b/zuul/ansible/base/action/command.py
@@ -13,8 +13,10 @@
 # You should have received a copy of the GNU General Public License
 # along with this software.  If not, see <http://www.gnu.org/licenses/>.
 
-
 from zuul.ansible import paths
+
+from ansible.module_utils.parsing.convert_bool import boolean
+
 command = paths._import_ansible_action_plugin("command")
 
 
@@ -25,10 +27,17 @@ class ActionModule(command.ActionModule):
         if self._task.action in (
                 'command', 'shell',
                 'ansible.builtin.command', 'ansible.builtin.shell'):
-            # This is a bit lame, but we do not log loops in the
-            # zuul_stream.py callback.  This allows us to not write
-            # out command.py output to files that will never be read.
-            if 'ansible_loop_var' in task_vars:
+            # Overloading the UUID is a bit lame, but it stops us
+            # having to modify the library command.py too much.  Both
+            # of these below stop the creation of the files on disk
+            # for situations where they won't be read and cleaned-up.
+            skip = boolean(
+                self._templar.template(
+                    "{{zuul_console_disabled|default(false)}}"))
+            if skip:
+                self._task.args['zuul_log_id'] = 'skip'
+            elif 'ansible_loop_var' in task_vars:
+                # we do not log loops in the zuul_stream.py callback.
                 self._task.args['zuul_log_id'] = 'in-loop-ignore'
             else:
                 # Get a unique key for ZUUL_LOG_ID_MAP.  ZUUL_LOG_ID_MAP
diff --git a/zuul/ansible/base/library/command.py b/zuul/ansible/base/library/command.py
index 3c22849ae..c52412edf 100755
--- a/zuul/ansible/base/library/command.py
+++ b/zuul/ansible/base/library/command.py
@@ -270,6 +270,8 @@ class Console(object):
         # special-casing for any of this path.
         if log_uuid == 'in-loop-ignore':
             self.logfile_name = os.devnull
+        elif log_uuid == 'skip':
+            self.logfile_name = os.devnull
         else:
             self.logfile_name = LOG_STREAM_FILE.format(log_uuid=log_uuid)
 
@@ -494,7 +496,8 @@ def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, ex
 
     try:
         if self._debug:
-            self.log('Executing: ' + self._clean_args(args))
+            self.log('Executing <%s>: %s',
+                     zuul_log_id, self._clean_args(args))
 
         # ZUUL: Replaced the execution loop with the zuul_runner run function