From 518cf7fe5eb82e1c850615f860010acd857be899 Mon Sep 17 00:00:00 2001 From: Albin Vass Date: Fri, 3 Apr 2020 10:59:00 +0200 Subject: Enables whitelisting and configuring callbacks Change-Id: Ida7b84795d922b85ec9cc6161ab1203fb82da825 --- doc/source/discussion/components.rst | 25 +++++++++++ .../notes/ansible-callbacks-c3bfce1a5cae6b15.yaml | 5 +++ .../git/common-config/playbooks/callback.yaml | 4 ++ .../playbooks/callback_plugins/test_callback.py | 35 ++++++++++++++++ .../ansible-callbacks/git/common-config/zuul.yaml | 21 ++++++++++ tests/fixtures/config/ansible-callbacks/main.yaml | 6 +++ tests/fixtures/zuul-executor-ansible-callback.conf | 48 ++++++++++++++++++++++ tests/unit/test_executor.py | 45 ++++++++++++++++++++ zuul/executor/server.py | 23 +++++++++++ 9 files changed, 212 insertions(+) create mode 100644 releasenotes/notes/ansible-callbacks-c3bfce1a5cae6b15.yaml create mode 100644 tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback.yaml create mode 100644 tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback_plugins/test_callback.py create mode 100644 tests/fixtures/config/ansible-callbacks/git/common-config/zuul.yaml create mode 100644 tests/fixtures/config/ansible-callbacks/main.yaml create mode 100644 tests/fixtures/zuul-executor-ansible-callback.conf diff --git a/doc/source/discussion/components.rst b/doc/source/discussion/components.rst index ba36c89c6..7cf715313 100644 --- a/doc/source/discussion/components.rst +++ b/doc/source/discussion/components.rst @@ -845,6 +845,31 @@ The following sections of ``zuul.conf`` are used by the executor: Value to pass to `git config user.name `_. +.. attr:: ansible_callback "" + + To whitelist ansible callback ````. Any attributes found is this section + will be added to the ``callback_`` section in ansible.cfg. + + An example of what configuring the builtin mail callback would look like. + The configuration in zuul.conf. + + .. code-block:: ini + + [ansible_callback "mail"] + to = user@example.org + sender = zuul@example.org + + Would generate the following in ansible.cfg: + + .. code-block:: ini + + [defaults] + callback_whitelist = mail + + [callback_mail] + to = user@example.org + sender = zuul@example.org + Operation ~~~~~~~~~ diff --git a/releasenotes/notes/ansible-callbacks-c3bfce1a5cae6b15.yaml b/releasenotes/notes/ansible-callbacks-c3bfce1a5cae6b15.yaml new file mode 100644 index 000000000..9d1c3b8da --- /dev/null +++ b/releasenotes/notes/ansible-callbacks-c3bfce1a5cae6b15.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Zuul now supports whitelisting and configuring ansible callbacks with + :attr:`ansible_callback ""`. diff --git a/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback.yaml b/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback.yaml new file mode 100644 index 000000000..50bbbbfc5 --- /dev/null +++ b/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback.yaml @@ -0,0 +1,4 @@ +- hosts: localhost + gather_facts: smart + tasks: + - command: echo test diff --git a/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback_plugins/test_callback.py b/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback_plugins/test_callback.py new file mode 100644 index 000000000..39ff7cd49 --- /dev/null +++ b/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback_plugins/test_callback.py @@ -0,0 +1,35 @@ +from ansible.plugins.callback import CallbackBase + +import os + +DOCUMENTATION = ''' + options: + file_name: + description: "" + ini: + - section: callback_test_callback + key: file_name + required: True + type: string +''' + + +class CallbackModule(CallbackBase): + CALLBACK_VERSION = 1.0 + CALLBACK_NEEDS_WHITELIST = True + + def __init__(self): + super(CallbackModule, self).__init__() + + def set_options(self, task_keys=None, var_options=None, direct=None): + super(CallbackModule, self).set_options(task_keys=task_keys, + var_options=var_options, + direct=direct) + + self.file_name = self.get_option('file_name') + + def v2_on_any(self, *args, **kwargs): + path = os.path.join(os.path.dirname(__file__), self.file_name) + self._display.display("Touching file: {}".format(path)) + with open(path, 'w'): + pass diff --git a/tests/fixtures/config/ansible-callbacks/git/common-config/zuul.yaml b/tests/fixtures/config/ansible-callbacks/git/common-config/zuul.yaml new file mode 100644 index 000000000..4acf6efb8 --- /dev/null +++ b/tests/fixtures/config/ansible-callbacks/git/common-config/zuul.yaml @@ -0,0 +1,21 @@ +- pipeline: + name: promote + manager: supercedent + post-review: true + trigger: + gerrit: + - event: change-merged + +- job: + name: callback-test + parent: null + run: playbooks/callback.yaml + nodeset: + nodes: + - name: ubuntu-xenial + label: ubuntu-xenial + +- project: + promote: + jobs: + - callback-test diff --git a/tests/fixtures/config/ansible-callbacks/main.yaml b/tests/fixtures/config/ansible-callbacks/main.yaml new file mode 100644 index 000000000..9d01f542f --- /dev/null +++ b/tests/fixtures/config/ansible-callbacks/main.yaml @@ -0,0 +1,6 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - common-config diff --git a/tests/fixtures/zuul-executor-ansible-callback.conf b/tests/fixtures/zuul-executor-ansible-callback.conf new file mode 100644 index 000000000..cf4592f83 --- /dev/null +++ b/tests/fixtures/zuul-executor-ansible-callback.conf @@ -0,0 +1,48 @@ +# Checks to make sure no key is configured in the +# [defaults] section of ansible.cfg, setting the +# same key twice would cause an error. + +# Equal sign in section name will not be treated as configuration field in ansible +[ansible_callback "nocows = True"] +[ansible_callback "nocows = False"] +# \n will not be treated as a newline character +[ansible_callback "\nnocows = True"] +[ansible_callback "\nnocows = False"] +# A single '%' sign would cause error if interpolation syntax is enabled +[ansible_callback "ansible_interpolation"] +test_field = test-%%-value + +[ansible_callback "test_callback"] +file_name = callback-success + +[gearman] +server=127.0.0.1 + +[statsd] +# note, use 127.0.0.1 rather than localhost to avoid getting ipv6 +# see: https://github.com/jsocol/pystatsd/issues/61 +server=127.0.0.1 + +[scheduler] +tenant_config=main.yaml + +[merger] +git_dir=/tmp/zuul-test/merger-git +git_user_email=zuul@example.com +git_user_name=zuul + +[executor] +git_dir=/tmp/zuul-test/executor-git + +[connection gerrit] +driver=gerrit +server=review.example.com +user=jenkins +sshkey=fake_id_rsa_path + +[connection smtp] +driver=smtp +server=localhost +port=25 +default_from=zuul@example.com +default_to=you@example.com diff --git a/tests/unit/test_executor.py b/tests/unit/test_executor.py index c0fbc5546..f16892035 100644 --- a/tests/unit/test_executor.py +++ b/tests/unit/test_executor.py @@ -15,6 +15,7 @@ import json import logging +import configparser import multiprocessing import os import time @@ -816,6 +817,50 @@ class TestExecutorFacts(AnsibleZuulTestCase): self.assertEqual(18, len(date_time)) +class TestAnsibleCallbackConfigs(AnsibleZuulTestCase): + + config_file = 'zuul-executor-ansible-callback.conf' + tenant_config_file = 'config/ansible-callbacks/main.yaml' + + def test_ansible_callback_config(self): + self.executor_server.keep_jobdir = True + A = self.fake_gerrit.addFakeChange('common-config', 'master', 'A') + self.fake_gerrit.addEvent(A.getChangeMergedEvent()) + self.waitUntilSettled() + + callbacks = [ + 'callback_test_callback', + 'callback_nocows = True', + 'callback_nocows = False', + 'callback_\\nnocows = True', + 'callback_\\nnocows = False', + 'callback_ansible_interpolation' + ] + + p = os.path.join(self.getJobFromHistory('callback-test').jobdir.root, + 'ansible/playbook_0/ansible.cfg') + self.assertEqual(self.getJobFromHistory('callback-test').result, + 'SUCCESS') + + c = configparser.ConfigParser(interpolation=None) + c.read(p) + for callback in callbacks: + self.assertIn(callback, c.sections()) + self.assertIn('test_field', c['callback_ansible_interpolation']) + self.assertIn('test-%-value', + c['callback_ansible_interpolation']['test_field']) + + self.assertIn('file_name', c['callback_test_callback']) + self.assertEqual('callback-success', + c['callback_test_callback']['file_name']) + callback_result_file = os.path.join( + self.getJobFromHistory('callback-test').jobdir.root, + 'trusted/project_0/review.example.com/', + 'common-config/playbooks/callback_plugins/', + c['callback_test_callback']['file_name']) + self.assertTrue(os.path.isfile(callback_result_file)) + + class TestExecutorEnvironment(AnsibleZuulTestCase): tenant_config_file = 'config/zuul-environment-filter/main.yaml' diff --git a/zuul/executor/server.py b/zuul/executor/server.py index 381dc53d3..4f73e20de 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -861,6 +861,7 @@ class AnsibleJob(object): self.callback_dir = os.path.join(plugin_dir, 'callback') self.lookup_dir = os.path.join(plugin_dir, 'lookup') self.filter_dir = os.path.join(plugin_dir, 'filter') + self.ansible_callbacks = self.executor_server.ansible_callbacks def run(self): self.running = True @@ -2076,6 +2077,11 @@ class AnsibleJob(object): # and reduces CPU load of the ansible process. config.write('internal_poll_interval = 0.01\n') + if self.ansible_callbacks: + config.write('callback_whitelist =\n') + for callback in self.ansible_callbacks.keys(): + config.write(' %s,\n' % callback) + config.write('[ssh_connection]\n') # NOTE(pabelanger): Try up to 3 times to run a task on a host, this # helps to mitigate UNREACHABLE host errors with SSH. @@ -2095,6 +2101,12 @@ class AnsibleJob(object): "-o UserKnownHostsFile=%s" % self.jobdir.known_hosts config.write('ssh_args = %s\n' % ssh_args) + if self.ansible_callbacks: + for cb_name, cb_config in self.ansible_callbacks.items(): + config.write("[callback_%s]\n" % cb_name) + for k, n in cb_config.items(): + config.write("%s = %s\n" % (k, n)) + def _ansibleTimeout(self, msg): self.log.warning(msg) self.abortRunningProc() @@ -2551,6 +2563,17 @@ class ExecutorServer(BaseMergeServer): 'ansible_setup_timeout', 60)) self.zone = get_default(self.config, 'executor', 'zone') + self.ansible_callbacks = {} + for section_name in self.config.sections(): + cb_match = re.match(r'^ansible_callback ([\'\"]?)(.*)(\1)$', + section_name, re.I) + if not cb_match: + continue + cb_name = cb_match.group(2) + self.ansible_callbacks[cb_name] = dict( + self.config.items(section_name) + ) + # TODO(tobiash): Take cgroups into account self.update_workers = multiprocessing.cpu_count() self.update_threads = [] -- cgit v1.2.1