diff options
Diffstat (limited to 'zuul/ansible/base')
194 files changed, 3126 insertions, 0 deletions
diff --git a/zuul/ansible/base/__init__.py b/zuul/ansible/base/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/__init__.py diff --git a/zuul/ansible/base/action/__init__.py b/zuul/ansible/base/action/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/__init__.py diff --git a/zuul/ansible/base/action/add_host.py b/zuul/ansible/base/action/add_host.py new file mode 100644 index 000000000..28e74b155 --- /dev/null +++ b/zuul/ansible/base/action/add_host.py @@ -0,0 +1,43 @@ +# Copyright 2018 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + +from zuul.ansible import paths +add_host = paths._import_ansible_action_plugin("add_host") + + +class ActionModule(add_host.ActionModule): + + def run(self, tmp=None, task_vars=None): + safe_args = set(( + 'ansible_connection', + 'ansible_host', + 'ansible_port', + 'ansible_user', + 'ansible_password', + 'ansible_ssh_host', + 'ansible_ssh_port', + 'ansible_ssh_user', + 'ansible_ssh_pass', + )) + args = set(filter( + lambda x: x.startswith('ansible_'), self._task.args.keys())) + conn = self._task.args.get('ansible_connection', 'ssh') + if args.issubset(safe_args) and conn in ('kubectl', 'ssh'): + return super(ActionModule, self).run(tmp, task_vars) + + return dict( + failed=True, + msg="Adding hosts %s with %s to the inventory is prohibited" % ( + conn, " ".join(args.difference(safe_args)))) diff --git a/zuul/ansible/base/action/add_host.pyi b/zuul/ansible/base/action/add_host.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/add_host.pyi diff --git a/zuul/ansible/base/action/aireos.py b/zuul/ansible/base/action/aireos.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/aireos.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/aireos.pyi b/zuul/ansible/base/action/aireos.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/aireos.pyi diff --git a/zuul/ansible/base/action/aireos_config.py b/zuul/ansible/base/action/aireos_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/aireos_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/aireos_config.pyi b/zuul/ansible/base/action/aireos_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/aireos_config.pyi diff --git a/zuul/ansible/base/action/aruba.py b/zuul/ansible/base/action/aruba.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/aruba.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/aruba.pyi b/zuul/ansible/base/action/aruba.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/aruba.pyi diff --git a/zuul/ansible/base/action/aruba_config.py b/zuul/ansible/base/action/aruba_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/aruba_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/aruba_config.pyi b/zuul/ansible/base/action/aruba_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/aruba_config.pyi diff --git a/zuul/ansible/base/action/asa.py b/zuul/ansible/base/action/asa.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/asa.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/asa.pyi b/zuul/ansible/base/action/asa.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/asa.pyi diff --git a/zuul/ansible/base/action/asa_config.py b/zuul/ansible/base/action/asa_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/asa_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/asa_config.pyi b/zuul/ansible/base/action/asa_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/asa_config.pyi diff --git a/zuul/ansible/base/action/asa_template.py b/zuul/ansible/base/action/asa_template.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/asa_template.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/asa_template.pyi b/zuul/ansible/base/action/asa_template.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/asa_template.pyi diff --git a/zuul/ansible/base/action/assemble.py b/zuul/ansible/base/action/assemble.py new file mode 100644 index 000000000..139ed7da4 --- /dev/null +++ b/zuul/ansible/base/action/assemble.py @@ -0,0 +1,35 @@ +# Copyright 2016 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + + +from zuul.ansible import paths +assemble = paths._import_ansible_action_plugin("assemble") + + +class ActionModule(assemble.ActionModule): + + def _find_needle(self, dirname, needle): + return paths._safe_find_needle( + super(ActionModule, self), dirname, needle) + + def run(self, tmp=None, task_vars=None): + + if not paths._is_official_module(self): + return paths._fail_module_dict(self._task.action) + + if paths._is_localhost_task(self): + paths._fail_if_unsafe(self._task.args['dest']) + + return super(ActionModule, self).run(tmp, task_vars) diff --git a/zuul/ansible/base/action/assemble.pyi b/zuul/ansible/base/action/assemble.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/assemble.pyi diff --git a/zuul/ansible/base/action/ce.py b/zuul/ansible/base/action/ce.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/ce.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/ce.pyi b/zuul/ansible/base/action/ce.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/ce.pyi diff --git a/zuul/ansible/base/action/ce_config.py b/zuul/ansible/base/action/ce_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/ce_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/ce_config.pyi b/zuul/ansible/base/action/ce_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/ce_config.pyi diff --git a/zuul/ansible/base/action/ce_template.py b/zuul/ansible/base/action/ce_template.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/ce_template.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/ce_template.pyi b/zuul/ansible/base/action/ce_template.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/ce_template.pyi diff --git a/zuul/ansible/base/action/copy.py b/zuul/ansible/base/action/copy.py new file mode 100644 index 000000000..e8927ce9e --- /dev/null +++ b/zuul/ansible/base/action/copy.py @@ -0,0 +1,35 @@ +# Copyright 2016 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + + +from zuul.ansible import paths +copy = paths._import_ansible_action_plugin("copy") + + +class ActionModule(copy.ActionModule): + + def _find_needle(self, dirname, needle): + return paths._safe_find_needle( + super(ActionModule, self), dirname, needle) + + def run(self, tmp=None, task_vars=None): + + if not paths._is_official_module(self): + return paths._fail_module_dict(self._task.action) + + if paths._is_localhost_task(self): + paths._fail_if_unsafe(self._task.args['dest']) + + return super(ActionModule, self).run(tmp, task_vars) diff --git a/zuul/ansible/base/action/copy.pyi b/zuul/ansible/base/action/copy.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/copy.pyi diff --git a/zuul/ansible/base/action/dellos10_config.py b/zuul/ansible/base/action/dellos10_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/dellos10_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/dellos10_config.pyi b/zuul/ansible/base/action/dellos10_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/dellos10_config.pyi diff --git a/zuul/ansible/base/action/dellos6_config.py b/zuul/ansible/base/action/dellos6_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/dellos6_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/dellos6_config.pyi b/zuul/ansible/base/action/dellos6_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/dellos6_config.pyi diff --git a/zuul/ansible/base/action/dellos9_config.py b/zuul/ansible/base/action/dellos9_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/dellos9_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/dellos9_config.pyi b/zuul/ansible/base/action/dellos9_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/dellos9_config.pyi diff --git a/zuul/ansible/base/action/eos_config.py b/zuul/ansible/base/action/eos_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/eos_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/eos_config.pyi b/zuul/ansible/base/action/eos_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/eos_config.pyi diff --git a/zuul/ansible/base/action/eos_template.py b/zuul/ansible/base/action/eos_template.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/eos_template.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/eos_template.pyi b/zuul/ansible/base/action/eos_template.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/eos_template.pyi diff --git a/zuul/ansible/base/action/fetch.py b/zuul/ansible/base/action/fetch.py new file mode 100644 index 000000000..0d35846e2 --- /dev/null +++ b/zuul/ansible/base/action/fetch.py @@ -0,0 +1,31 @@ +# Copyright 2016 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + + +from zuul.ansible import paths +fetch = paths._import_ansible_action_plugin("fetch") + + +class ActionModule(fetch.ActionModule): + + def run(self, tmp=None, task_vars=None): + if not paths._is_official_module(self): + return paths._fail_module_dict(self._task.action) + + dest = self._task.args.get('dest', None) + + if dest and not paths._is_safe_path(dest): + return paths._fail_dict(dest) + return super(ActionModule, self).run(tmp, task_vars) diff --git a/zuul/ansible/base/action/fetch.pyi b/zuul/ansible/base/action/fetch.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/fetch.pyi diff --git a/zuul/ansible/base/action/fortios_config.py b/zuul/ansible/base/action/fortios_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/fortios_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/fortios_config.pyi b/zuul/ansible/base/action/fortios_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/fortios_config.pyi diff --git a/zuul/ansible/base/action/include_vars.py b/zuul/ansible/base/action/include_vars.py new file mode 100644 index 000000000..930e54d50 --- /dev/null +++ b/zuul/ansible/base/action/include_vars.py @@ -0,0 +1,40 @@ +# Copyright 2016 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + + +from zuul.ansible import paths +include_vars = paths._import_ansible_action_plugin("include_vars") + + +class ActionModule(include_vars.ActionModule): + + def _find_needle(self, dirname, needle): + return paths._safe_find_needle( + super(ActionModule, self), dirname, needle) + + def run(self, tmp=None, task_vars=None): + if not paths._is_official_module(self): + return paths._fail_module_dict(self._task.action) + + source_dir = self._task.args.get('dir', None) + + # This is the handling for source_dir. The source_file is handled by + # the _find_needle override. + if source_dir: + self._set_args() + self._set_root_dir() + if not paths._is_safe_path(self.source_dir, allow_trusted=True): + return paths._fail_dict(self.source_dir) + return super(ActionModule, self).run(tmp, task_vars) diff --git a/zuul/ansible/base/action/include_vars.pyi b/zuul/ansible/base/action/include_vars.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/include_vars.pyi diff --git a/zuul/ansible/base/action/ios_config.py b/zuul/ansible/base/action/ios_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/ios_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/ios_config.pyi b/zuul/ansible/base/action/ios_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/ios_config.pyi diff --git a/zuul/ansible/base/action/ios_template.py b/zuul/ansible/base/action/ios_template.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/ios_template.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/ios_template.pyi b/zuul/ansible/base/action/ios_template.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/ios_template.pyi diff --git a/zuul/ansible/base/action/iosxr_config.py b/zuul/ansible/base/action/iosxr_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/iosxr_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/iosxr_config.pyi b/zuul/ansible/base/action/iosxr_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/iosxr_config.pyi diff --git a/zuul/ansible/base/action/iosxr_template.py b/zuul/ansible/base/action/iosxr_template.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/iosxr_template.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/iosxr_template.pyi b/zuul/ansible/base/action/iosxr_template.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/iosxr_template.pyi diff --git a/zuul/ansible/base/action/junos_config.py b/zuul/ansible/base/action/junos_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/junos_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/junos_config.pyi b/zuul/ansible/base/action/junos_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/junos_config.pyi diff --git a/zuul/ansible/base/action/junos_template.py b/zuul/ansible/base/action/junos_template.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/junos_template.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/junos_template.pyi b/zuul/ansible/base/action/junos_template.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/junos_template.pyi diff --git a/zuul/ansible/base/action/net_banner.py b/zuul/ansible/base/action/net_banner.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/net_banner.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/net_banner.pyi b/zuul/ansible/base/action/net_banner.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/net_banner.pyi diff --git a/zuul/ansible/base/action/net_base.py b/zuul/ansible/base/action/net_base.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/net_base.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/net_base.pyi b/zuul/ansible/base/action/net_base.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/net_base.pyi diff --git a/zuul/ansible/base/action/net_config.py b/zuul/ansible/base/action/net_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/net_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/net_config.pyi b/zuul/ansible/base/action/net_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/net_config.pyi diff --git a/zuul/ansible/base/action/net_interface.py b/zuul/ansible/base/action/net_interface.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/net_interface.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/net_interface.pyi b/zuul/ansible/base/action/net_interface.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/net_interface.pyi diff --git a/zuul/ansible/base/action/net_l2_interface.py b/zuul/ansible/base/action/net_l2_interface.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/net_l2_interface.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/net_l2_interface.pyi b/zuul/ansible/base/action/net_l2_interface.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/net_l2_interface.pyi diff --git a/zuul/ansible/base/action/net_l3_interface.py b/zuul/ansible/base/action/net_l3_interface.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/net_l3_interface.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/net_l3_interface.pyi b/zuul/ansible/base/action/net_l3_interface.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/net_l3_interface.pyi diff --git a/zuul/ansible/base/action/net_linkagg.py b/zuul/ansible/base/action/net_linkagg.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/net_linkagg.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/net_linkagg.pyi b/zuul/ansible/base/action/net_linkagg.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/net_linkagg.pyi diff --git a/zuul/ansible/base/action/net_lldp.py b/zuul/ansible/base/action/net_lldp.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/net_lldp.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/net_lldp.pyi b/zuul/ansible/base/action/net_lldp.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/net_lldp.pyi diff --git a/zuul/ansible/base/action/net_lldp_interface.py b/zuul/ansible/base/action/net_lldp_interface.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/net_lldp_interface.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/net_lldp_interface.pyi b/zuul/ansible/base/action/net_lldp_interface.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/net_lldp_interface.pyi diff --git a/zuul/ansible/base/action/net_logging.py b/zuul/ansible/base/action/net_logging.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/net_logging.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/net_logging.pyi b/zuul/ansible/base/action/net_logging.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/net_logging.pyi diff --git a/zuul/ansible/base/action/net_ping.py b/zuul/ansible/base/action/net_ping.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/net_ping.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/net_ping.pyi b/zuul/ansible/base/action/net_ping.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/net_ping.pyi diff --git a/zuul/ansible/base/action/net_static_route.py b/zuul/ansible/base/action/net_static_route.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/net_static_route.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/net_static_route.pyi b/zuul/ansible/base/action/net_static_route.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/net_static_route.pyi diff --git a/zuul/ansible/base/action/net_system.py b/zuul/ansible/base/action/net_system.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/net_system.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/net_system.pyi b/zuul/ansible/base/action/net_system.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/net_system.pyi diff --git a/zuul/ansible/base/action/net_template.py b/zuul/ansible/base/action/net_template.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/net_template.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/net_template.pyi b/zuul/ansible/base/action/net_template.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/net_template.pyi diff --git a/zuul/ansible/base/action/net_user.py b/zuul/ansible/base/action/net_user.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/net_user.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/net_user.pyi b/zuul/ansible/base/action/net_user.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/net_user.pyi diff --git a/zuul/ansible/base/action/net_vlan.py b/zuul/ansible/base/action/net_vlan.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/net_vlan.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/net_vlan.pyi b/zuul/ansible/base/action/net_vlan.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/net_vlan.pyi diff --git a/zuul/ansible/base/action/net_vrf.py b/zuul/ansible/base/action/net_vrf.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/net_vrf.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/net_vrf.pyi b/zuul/ansible/base/action/net_vrf.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/net_vrf.pyi diff --git a/zuul/ansible/base/action/netconf_config.py b/zuul/ansible/base/action/netconf_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/netconf_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/netconf_config.pyi b/zuul/ansible/base/action/netconf_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/netconf_config.pyi diff --git a/zuul/ansible/base/action/network.py b/zuul/ansible/base/action/network.py new file mode 100644 index 000000000..41fc56033 --- /dev/null +++ b/zuul/ansible/base/action/network.py @@ -0,0 +1,25 @@ +# Copyright 2016 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + + +from zuul.ansible import paths +network = paths._import_ansible_action_plugin("network") + + +class ActionModule(network.ActionModule): + + def run(self, tmp=None, task_vars=None): + + return dict(failed=True, msg='Use of network modules is prohibited') diff --git a/zuul/ansible/base/action/network.pyi b/zuul/ansible/base/action/network.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/network.pyi diff --git a/zuul/ansible/base/action/normal.py b/zuul/ansible/base/action/normal.py new file mode 100644 index 000000000..3230a708c --- /dev/null +++ b/zuul/ansible/base/action/normal.py @@ -0,0 +1,118 @@ +# Copyright 2017 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + +from ansible.module_utils.six.moves.urllib.parse import urlparse +from ansible.errors import AnsibleError + +from zuul.ansible import paths +normal = paths._import_ansible_action_plugin('normal') + +ALLOWED_URL_SCHEMES = ('https', 'http', 'ftp') + + +class ActionModule(normal.ActionModule): + '''Override the normal action plugin + + :py:class:`ansible.plugins.normal.ActionModule` is run for every + module that does not have a more specific matching action plugin. + + Our overridden version of it wraps the execution with checks to block + undesired actions on localhost. + ''' + + def run(self, tmp=None, task_vars=None): + '''Overridden primary method from the base class.''' + + if paths._is_localhost_task(self): + if not self.dispatch_handler(): + raise AnsibleError("Executing local code is prohibited") + return super(ActionModule, self).run(tmp, task_vars) + + def dispatch_handler(self): + '''Run per-action handler if one exists.''' + handler_name = 'handle_{action}'.format(action=self._task.action) + handler = getattr(self, handler_name, None) + if handler: + paths._fail_if_local_module(self) + handler() + return True + return False + + def handle_zuul_return(self): + '''Allow zuul_return module on localhost.''' + pass + + def handle_stat(self): + '''Allow stat module on localhost if it doesn't touch unsafe files. + + The :ansible:module:`stat` can be useful in jobs for manipulating logs + and artifacts. + + Block any access of files outside the zuul work dir. + ''' + if self._task.args.get('get_mime'): + raise AnsibleError("get_mime on localhost is forbidden") + paths._fail_if_unsafe(self._task.args['path']) + + def handle_file(self): + '''Allow file module on localhost if it doesn't touch unsafe files. + + The :ansible:module:`file` can be useful in jobs for manipulating logs + and artifacts. + + Block any access of files outside the zuul work dir. + ''' + for arg in ('path', 'dest', 'name'): + dest = self._task.args.get(arg) + if dest: + paths._fail_if_unsafe(dest) + + def handle_known_hosts(self): + '''Allow known_hosts on localhost + + The :ansible:module:`known_hosts` can be used to add SSH host keys of + a remote system. When run from a executor it can be used with the + add_host task to access remote servers. This is needed because ansible + on the executor is configured to check host keys by default. + + Block any access of files outside the zuul work dir. + ''' + if paths._is_localhost_task(self): + path = self._task.args.get('path') + if path: + paths._fail_if_unsafe(path) + + def handle_uri(self): + '''Allow uri module on localhost if it doesn't touch unsafe files. + + The :ansible:module:`uri` can be used from the executor to do + things like pinging readthedocs.org that otherwise don't need a node. + However, it can also download content to a local file, or be used to + read from file:/// urls. + + Block any use of url schemes other than https, http and ftp. Further, + block any local file interaction that falls outside of the zuul + work dir. + ''' + # uri takes all the file arguments, so just let handle_file validate + # them for us. + self.handle_file() + scheme = urlparse(self._task.args['url']).scheme + if scheme not in ALLOWED_URL_SCHEMES: + raise AnsibleError( + "{scheme} urls are not allowed from localhost." + " Only {allowed_schemes} are allowed".format( + scheme=scheme, + allowed_schemes=ALLOWED_URL_SCHEMES)) diff --git a/zuul/ansible/base/action/normal.pyi b/zuul/ansible/base/action/normal.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/normal.pyi diff --git a/zuul/ansible/base/action/nxos_config.py b/zuul/ansible/base/action/nxos_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/nxos_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/nxos_config.pyi b/zuul/ansible/base/action/nxos_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/nxos_config.pyi diff --git a/zuul/ansible/base/action/nxos_template.py b/zuul/ansible/base/action/nxos_template.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/nxos_template.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/nxos_template.pyi b/zuul/ansible/base/action/nxos_template.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/nxos_template.pyi diff --git a/zuul/ansible/base/action/ops_config.py b/zuul/ansible/base/action/ops_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/ops_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/ops_config.pyi b/zuul/ansible/base/action/ops_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/ops_config.pyi diff --git a/zuul/ansible/base/action/ops_template.py b/zuul/ansible/base/action/ops_template.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/ops_template.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/ops_template.pyi b/zuul/ansible/base/action/ops_template.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/ops_template.pyi diff --git a/zuul/ansible/base/action/patch.py b/zuul/ansible/base/action/patch.py new file mode 100644 index 000000000..098d4819c --- /dev/null +++ b/zuul/ansible/base/action/patch.py @@ -0,0 +1,43 @@ +# Copyright 2016 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + +from zuul.ansible import paths +patch = paths._import_ansible_action_plugin("patch") + + +class ActionModule(patch.ActionModule): + + def _find_needle(self, dirname, needle): + return paths._safe_find_needle( + super(ActionModule, self), dirname, needle) + + def run(self, tmp=None, task_vars=None): + if not paths._is_official_module(self): + return paths._fail_module_dict(self._task.action) + + if paths._is_localhost_task(self): + # The patch module has two possibilities of describing where to + # operate, basedir and dest. We need to perform the safe path check + # for both. + dirs_to_check = [ + self._task.args.get('basedir'), + self._task.args.get('dest'), + ] + + for directory in dirs_to_check: + if directory is not None: + paths._fail_if_unsafe(directory) + + return super(ActionModule, self).run(tmp, task_vars) diff --git a/zuul/ansible/base/action/patch.pyi b/zuul/ansible/base/action/patch.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/patch.pyi diff --git a/zuul/ansible/base/action/raw.py b/zuul/ansible/base/action/raw.py new file mode 100644 index 000000000..fb1e1a6e3 --- /dev/null +++ b/zuul/ansible/base/action/raw.py @@ -0,0 +1,32 @@ +# Copyright 2019 BMW Group +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + + +from ansible.errors import AnsibleError +from zuul.ansible import paths +raw = paths._import_ansible_action_plugin("raw") + + +class ActionModule(raw.ActionModule): + + def run(self, tmp=None, task_vars=None): + + if not paths._is_official_module(self): + return paths._fail_module_dict(self._task.action) + + if paths._is_localhost_task(self): + raise AnsibleError("Executing local code is prohibited") + + return super(ActionModule, self).run(tmp, task_vars) diff --git a/zuul/ansible/base/action/raw.pyi b/zuul/ansible/base/action/raw.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/raw.pyi diff --git a/zuul/ansible/base/action/script.py b/zuul/ansible/base/action/script.py new file mode 100644 index 000000000..9fc38cc96 --- /dev/null +++ b/zuul/ansible/base/action/script.py @@ -0,0 +1,36 @@ +# Copyright 2016 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + + +from ansible.errors import AnsibleError +from zuul.ansible import paths +script = paths._import_ansible_action_plugin("script") + + +class ActionModule(script.ActionModule): + + def _find_needle(self, dirname, needle): + return paths._safe_find_needle( + super(ActionModule, self), dirname, needle) + + def run(self, tmp=None, task_vars=None): + + if not paths._is_official_module(self): + return paths._fail_module_dict(self._task.action) + + if paths._is_localhost_task(self): + raise AnsibleError("Executing local code is prohibited") + + return super(ActionModule, self).run(tmp, task_vars) diff --git a/zuul/ansible/base/action/script.pyi b/zuul/ansible/base/action/script.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/script.pyi diff --git a/zuul/ansible/base/action/sros_config.py b/zuul/ansible/base/action/sros_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/sros_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/sros_config.pyi b/zuul/ansible/base/action/sros_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/sros_config.pyi diff --git a/zuul/ansible/base/action/synchronize.py b/zuul/ansible/base/action/synchronize.py new file mode 100644 index 000000000..b07ba79be --- /dev/null +++ b/zuul/ansible/base/action/synchronize.py @@ -0,0 +1,49 @@ +# Copyright 2016 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + + +from zuul.ansible import paths +synchronize = paths._import_ansible_action_plugin("synchronize") + + +class ActionModule(synchronize.ActionModule): + + def run(self, tmp=None, task_vars=None): + if not paths._is_official_module(self): + return paths._fail_module_dict(self._task.action) + + try: + delegate_to = self._task.delegate_to + except (AttributeError, KeyError): + delegate_to = None + + if delegate_to and not paths._is_localhost_task(self): + return super(ActionModule, self).run(tmp, task_vars) + + source = self._task.args.get('src', None) + dest = self._task.args.get('dest', None) + mode = self._task.args.get('mode', 'push') + + if 'rsync_opts' not in self._task.args: + self._task.args['rsync_opts'] = [] + if '--safe-links' not in self._task.args['rsync_opts']: + self._task.args['rsync_opts'].append('--safe-links') + + if mode == 'push' and not paths._is_safe_path( + source, allow_trusted=True): + return paths._fail_dict(source, prefix='Syncing files from') + if mode == 'pull' and not paths._is_safe_path(dest): + return paths._fail_dict(dest, prefix='Syncing files to') + return super(ActionModule, self).run(tmp, task_vars) diff --git a/zuul/ansible/base/action/synchronize.pyi b/zuul/ansible/base/action/synchronize.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/synchronize.pyi diff --git a/zuul/ansible/base/action/template.py b/zuul/ansible/base/action/template.py new file mode 100644 index 000000000..5f0e5602c --- /dev/null +++ b/zuul/ansible/base/action/template.py @@ -0,0 +1,31 @@ +# Copyright 2016 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + + +from zuul.ansible import paths +template = paths._import_ansible_action_plugin("template") + + +class ActionModule(template.ActionModule): + + def _find_needle(self, dirname, needle): + return paths._safe_find_needle( + super(ActionModule, self), dirname, needle) + + def run(self, tmp=None, task_vars=None): + if not paths._is_official_module(self): + return paths._fail_module_dict(self._task.action) + + return super(ActionModule, self).run(tmp, task_vars) diff --git a/zuul/ansible/base/action/template.pyi b/zuul/ansible/base/action/template.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/template.pyi diff --git a/zuul/ansible/base/action/unarchive.py b/zuul/ansible/base/action/unarchive.py new file mode 100644 index 000000000..9eb9bb544 --- /dev/null +++ b/zuul/ansible/base/action/unarchive.py @@ -0,0 +1,36 @@ +# Copyright 2016 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + + +from zuul.ansible import paths +unarchive = paths._import_ansible_action_plugin("unarchive") + + +class ActionModule(unarchive.ActionModule): + + def _find_needle(self, dirname, needle): + return paths._safe_find_needle( + super(ActionModule, self), dirname, needle) + + def run(self, tmp=None, task_vars=None): + if not paths._is_official_module(self): + return paths._fail_module_dict(self._task.action) + + # Note: The unarchive module reuses the copy module to copy the archive + # to the remote. Thus we don't need to check the dest here if we run + # against localhost. We also have tests that would break if this + # changes in the future. + + return super(ActionModule, self).run(tmp, task_vars) diff --git a/zuul/ansible/base/action/unarchive.pyi b/zuul/ansible/base/action/unarchive.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/unarchive.pyi diff --git a/zuul/ansible/base/action/vyos_config.py b/zuul/ansible/base/action/vyos_config.py new file mode 120000 index 000000000..7a739baa2 --- /dev/null +++ b/zuul/ansible/base/action/vyos_config.py @@ -0,0 +1 @@ +network.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/vyos_config.pyi b/zuul/ansible/base/action/vyos_config.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/vyos_config.pyi diff --git a/zuul/ansible/base/action/win_copy.py b/zuul/ansible/base/action/win_copy.py new file mode 100644 index 000000000..d9dbe4dc8 --- /dev/null +++ b/zuul/ansible/base/action/win_copy.py @@ -0,0 +1,31 @@ +# Copyright 2016 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + + +from zuul.ansible import paths +win_copy = paths._import_ansible_action_plugin("win_copy") + + +class ActionModule(win_copy.ActionModule): + + def _find_needle(self, dirname, needle): + return paths._safe_find_needle( + super(ActionModule, self), dirname, needle) + + def run(self, tmp=None, task_vars=None): + if not paths._is_official_module(self): + return paths._fail_module_dict(self._task.action) + + return super(ActionModule, self).run(tmp, task_vars) diff --git a/zuul/ansible/base/action/win_copy.pyi b/zuul/ansible/base/action/win_copy.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/win_copy.pyi diff --git a/zuul/ansible/base/action/win_template.py b/zuul/ansible/base/action/win_template.py new file mode 100644 index 000000000..36b475aea --- /dev/null +++ b/zuul/ansible/base/action/win_template.py @@ -0,0 +1,31 @@ +# Copyright 2016 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + + +from zuul.ansible import paths +win_template = paths._import_ansible_action_plugin("win_template") + + +class ActionModule(win_template.ActionModule): + + def _find_needle(self, dirname, needle): + return paths._safe_find_needle( + super(ActionModule, self), dirname, needle) + + def run(self, tmp=None, task_vars=None): + if not paths._is_official_module(self): + return paths._fail_module_dict(self._task.action) + + return super(ActionModule, self).run(tmp, task_vars) diff --git a/zuul/ansible/base/action/win_template.pyi b/zuul/ansible/base/action/win_template.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/action/win_template.pyi diff --git a/zuul/ansible/base/actiongeneral/__init__.py b/zuul/ansible/base/actiongeneral/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/actiongeneral/__init__.py diff --git a/zuul/ansible/base/actiongeneral/command.py b/zuul/ansible/base/actiongeneral/command.py new file mode 100644 index 000000000..f9b976ca0 --- /dev/null +++ b/zuul/ansible/base/actiongeneral/command.py @@ -0,0 +1,29 @@ +# Copyright 2018 BMW Car IT GmbH +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + + +from zuul.ansible import paths +command = paths._import_ansible_action_plugin("command") + + +class ActionModule(command.ActionModule): + + def run(self, tmp=None, task_vars=None): + # we need the zuul_log_id on shell and command tasks + host = paths._sanitize_filename(task_vars.get('inventory_hostname')) + if self._task.action in ('command', 'shell'): + self._task.args['zuul_log_id'] = "%s-%s" % (self._task._uuid, host) + + return super(ActionModule, self).run(tmp, task_vars) diff --git a/zuul/ansible/base/actiongeneral/command.pyi b/zuul/ansible/base/actiongeneral/command.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/actiongeneral/command.pyi diff --git a/zuul/ansible/base/actiongeneral/zuul_return.py b/zuul/ansible/base/actiongeneral/zuul_return.py new file mode 100644 index 000000000..badd03bb1 --- /dev/null +++ b/zuul/ansible/base/actiongeneral/zuul_return.py @@ -0,0 +1,106 @@ +#!/usr/bin/python + +# Copyright (c) 2017 Red Hat +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + +import os +import json +import tempfile + +from ansible.plugins.action import ActionBase + +from zuul.ansible import paths + + +def merge_dict(dict_a, dict_b): + """ + Add dict_a into dict_b + Merge values if possible else dict_a value replace dict_b value + """ + for key in dict_a: + if key in dict_b: + if isinstance(dict_a[key], dict) and isinstance(dict_b[key], dict): + merge_dict(dict_a[key], dict_b[key]) + else: + dict_b[key] = dict_a[key] + else: + dict_b[key] = dict_a[key] + return dict_b + + +def merge_data(dict_a, dict_b): + """ + Merge dict_a into dict_b, handling any special cases for zuul variables + """ + artifacts_a = dict_a.get('zuul', {}).get('artifacts', []) + if not isinstance(artifacts_a, list): + artifacts_a = [] + artifacts_b = dict_b.get('zuul', {}).get('artifacts', []) + if not isinstance(artifacts_b, list): + artifacts_b = [] + artifacts = artifacts_a + artifacts_b + merge_dict(dict_a, dict_b) + if artifacts: + dict_b.setdefault('zuul', {})['artifacts'] = artifacts + return dict_b + + +def set_value(path, new_data, new_file): + workdir = os.path.dirname(path) + data = None + if os.path.exists(path): + with open(path, 'r') as f: + data = f.read() + if data: + data = json.loads(data) + else: + data = {} + + if new_file: + with open(new_file, 'r') as f: + merge_data(json.load(f), data) + if new_data: + merge_data(new_data, data) + + (f, tmp_path) = tempfile.mkstemp(dir=workdir) + try: + f = os.fdopen(f, 'w') + json.dump(data, f) + f.close() + os.rename(tmp_path, path) + except Exception: + os.unlink(tmp_path) + raise + + +class ActionModule(ActionBase): + def run(self, tmp=None, task_vars=None): + if task_vars is None: + task_vars = dict() + results = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + path = self._task.args.get('path') + if not path: + path = os.path.join(os.environ['ZUUL_JOBDIR'], 'work', + 'results.json') + + if not paths._is_safe_path(path, allow_trusted=False): + return paths._fail_dict(path) + + set_value( + path, self._task.args.get('data'), self._task.args.get('file')) + + return results diff --git a/zuul/ansible/base/callback/__init__.py b/zuul/ansible/base/callback/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/callback/__init__.py diff --git a/zuul/ansible/base/callback/zuul_json.py b/zuul/ansible/base/callback/zuul_json.py new file mode 100644 index 000000000..e52e2e24d --- /dev/null +++ b/zuul/ansible/base/callback/zuul_json.py @@ -0,0 +1,186 @@ +# (c) 2016, Matt Martz <matt@sivel.net> +# (c) 2017, Red Hat, Inc. +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# Copy of github.com/ansible/ansible/lib/ansible/plugins/callback/json.py +# We need to run as a secondary callback not a stdout and we need to control +# the output file location via a zuul environment variable similar to how we +# do in zuul_stream. +# Subclassing wreaks havoc on the module loader and namepsaces +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import copy +import datetime +import json +import os + +from ansible.plugins.callback import CallbackBase +try: + # It's here in 2.3 + from ansible.vars import strip_internal_keys +except ImportError: + try: + # It's here in 2.4 + from ansible.vars.manager import strip_internal_keys + except ImportError: + # It's here in 2.5 + from ansible.vars.clean import strip_internal_keys + +from zuul.ansible import logconfig + + +def current_time(): + return '%sZ' % datetime.datetime.utcnow().isoformat() + + +class CallbackModule(CallbackBase): + CALLBACK_VERSION = 2.0 + # aggregate means we can be loaded and not be the stdout plugin + CALLBACK_TYPE = 'aggregate' + CALLBACK_NAME = 'zuul_json' + + def __init__(self, display=None): + super(CallbackModule, self).__init__(display) + self.results = [] + self.playbook = {} + logging_config = logconfig.load_job_config( + os.environ['ZUUL_JOB_LOG_CONFIG']) + + self.output_path = os.path.splitext( + logging_config.job_output_file)[0] + '.json' + + self._playbook_name = None + + def _new_playbook(self, play): + # Get the hostvars from just one host - the vars we're looking for will + # be identical on all of them + hostvars = next(iter(play._variable_manager._hostvars.values())) + self._playbook_name = None + + # TODO(mordred) For now, protect specific variable lookups to make it + # not absurdly strange to run local tests with the callback plugin + # enabled. Remove once we have a "run playbook like zuul runs playbook" + # tool. + phase = hostvars.get('zuul_execution_phase') + index = hostvars.get('zuul_execution_phase_index') + playbook = hostvars.get('zuul_execution_canonical_name_and_path') + trusted = hostvars.get('zuul_execution_trusted') + trusted = True if trusted == "True" else False + branch = hostvars.get('zuul_execution_branch') + + self.playbook['playbook'] = playbook + self.playbook['phase'] = phase + self.playbook['index'] = index + self.playbook['trusted'] = trusted + self.playbook['branch'] = branch + + def _new_play(self, play): + return { + 'play': { + 'name': play.name, + 'id': str(play._uuid), + 'duration': { + 'start': current_time() + } + }, + 'tasks': [] + } + + def _new_task(self, task): + data = { + 'task': { + 'name': task.name, + 'id': str(task._uuid), + 'duration': { + 'start': current_time() + } + }, + 'hosts': {} + } + if task._role: + data['role'] = { + 'name': task._role.get_name(), + 'id': str(task._role._uuid), + 'path': task._role._role_path, + } + return data + + def v2_playbook_on_start(self, playbook): + self._playbook_name = os.path.splitext(playbook._file_name)[0] + + def v2_playbook_on_play_start(self, play): + if self._playbook_name: + self._new_playbook(play) + + self.results.append(self._new_play(play)) + + def v2_playbook_on_task_start(self, task, is_conditional): + self.results[-1]['tasks'].append(self._new_task(task)) + + def v2_runner_on_ok(self, result, **kwargs): + host = result._host + action = result._task.action + if result._result.get('_ansible_no_log', False) or result._task.no_log: + self.results[-1]['tasks'][-1]['hosts'][host.name] = dict( + censored="the output has been hidden due to the fact that" + " 'no_log: true' was specified for this result") + else: + # strip_internal_keys makes a deep copy of dict items, but + # not lists, so we need to create our own complete deep + # copy first so we don't modify the original. + myresult = copy.deepcopy(result._result) + clean_result = strip_internal_keys(myresult) + + for index, item_result in enumerate( + clean_result.get('results', [])): + if not item_result.get('_ansible_no_log', False): + continue + clean_result['results'][index] = dict( + censored="the output has been hidden due to the fact that" + " 'no_log: true' was specified for this result") + + self.results[-1]['tasks'][-1]['hosts'][host.name] = clean_result + end_time = current_time() + self.results[-1]['tasks'][-1]['task']['duration']['end'] = end_time + self.results[-1]['play']['duration']['end'] = end_time + self.results[-1]['tasks'][-1]['hosts'][host.name]['action'] = action + + def v2_playbook_on_stats(self, stats): + """Display info about playbook statistics""" + hosts = sorted(stats.processed.keys()) + + summary = {} + for h in hosts: + s = stats.summarize(h) + summary[h] = s + + self.playbook['plays'] = self.results + self.playbook['stats'] = summary + + # For now, just read in the old file and write it all out again + # This may well not scale from a memory perspective- but let's see how + # it goes. + output = [] + if os.path.exists(self.output_path): + output = json.load(open(self.output_path, 'r')) + output.append(self.playbook) + + json.dump(output, open(self.output_path, 'w'), + indent=4, sort_keys=True, separators=(',', ': ')) + + v2_runner_on_failed = v2_runner_on_ok + v2_runner_on_unreachable = v2_runner_on_ok + v2_runner_on_skipped = v2_runner_on_ok diff --git a/zuul/ansible/base/callback/zuul_stream.py b/zuul/ansible/base/callback/zuul_stream.py new file mode 100644 index 000000000..966c9008c --- /dev/null +++ b/zuul/ansible/base/callback/zuul_stream.py @@ -0,0 +1,633 @@ +# Copyright 2017 Red Hat, Inc. +# +# Zuul is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Zuul is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# This is not needed in python3 - but it is needed in python2 because there +# is a json module in ansible.plugins.callback and python2 gets confused. +# Easy local testing with ansible-playbook is handy when hacking on zuul_stream +# so just put in the __future__ statement. +from __future__ import absolute_import + +import datetime +import logging +import logging.config +import json +import os +import socket +import threading +import time + +from ansible.plugins.callback import default +from zuul.ansible import paths + +from zuul.ansible import logconfig + +LOG_STREAM_PORT = 19885 + + +def zuul_filter_result(result): + """Remove keys from shell/command output. + + Zuul streams stdout into the log above, so including stdout and stderr + in the result dict that ansible displays in the logs is duplicate + noise. We keep stdout in the result dict so that other callback plugins + like ARA could also have access to it. But drop them here. + + Remove changed so that we don't show a bunch of "changed" titles + on successful shell tasks, since that doesn't make sense from a Zuul + POV. The super class treats missing "changed" key as False. + + Remove cmd because most of the script content where people want to + see the script run is run with -x. It's possible we may want to revist + this to be smarter about when we remove it - like, only remove it + if it has an embedded newline - so that for normal 'simple' uses + of cmd it'll echo what the command was for folks. + """ + + stdout = result.pop('stdout', '') + stdout_lines = result.pop('stdout_lines', []) + if not stdout_lines and stdout: + stdout_lines = stdout.split('\n') + + for key in ('changed', 'cmd', 'zuul_log_id', 'invocation', + 'stderr', 'stderr_lines'): + result.pop(key, None) + return stdout_lines + + +class CallbackModule(default.CallbackModule): + + ''' + This is the Zuul streaming callback. It's based on the default + callback plugin, but streams results from shell commands. + ''' + + CALLBACK_VERSION = 2.0 + CALLBACK_TYPE = 'stdout' + CALLBACK_NAME = 'zuul_stream' + + def __init__(self): + + super(CallbackModule, self).__init__() + self._task = None + self._daemon_running = False + self._play = None + self._streamers = [] + self._streamers_stop = False + self.configure_logger() + self._items_done = False + self._deferred_result = None + self._playbook_name = None + + def configure_logger(self): + # ansible appends timestamp, user and pid to the log lines emitted + # to the log file. We're doing other things though, so we don't want + # this. + logging_config = logconfig.load_job_config( + os.environ['ZUUL_JOB_LOG_CONFIG']) + + if self._display.verbosity > 2: + logging_config.setDebug() + + logging_config.apply() + + self._logger = logging.getLogger('zuul.executor.ansible') + + def _log(self, msg, ts=None, job=True, executor=False, debug=False): + msg = msg.rstrip() + if job: + now = ts or datetime.datetime.now() + self._logger.info("{now} | {msg}".format(now=now, msg=msg)) + if executor: + if debug: + self._display.vvv(msg) + else: + self._display.display(msg) + + def _read_log(self, host, ip, log_id, task_name, hosts): + self._log("[%s] Starting to log %s for task %s" + % (host, log_id, task_name), job=False, executor=True) + while True: + try: + s = socket.create_connection((ip, LOG_STREAM_PORT), 5) + # Disable the socket timeout after we have successfully + # connected to accomodate the fact that jobs may not be writing + # logs continously. Without this we can easily trip the 5 + # second timeout. + s.settimeout(None) + except socket.timeout: + self._log( + "Timeout exception waiting for the logger. " + "Please check connectivity to [%s:%s]" + % (ip, LOG_STREAM_PORT), executor=True) + self._log_streamline( + "localhost", + "Timeout exception waiting for the logger. " + "Please check connectivity to [%s:%s]" + % (ip, LOG_STREAM_PORT)) + return + except Exception: + self._log("[%s] Waiting on logger" % host, + executor=True, debug=True) + time.sleep(0.1) + continue + msg = "%s\n" % log_id + s.send(msg.encode("utf-8")) + buff = s.recv(4096) + buffering = True + while buffering: + if b'\n' in buff: + (line, buff) = buff.split(b'\n', 1) + # We can potentially get binary data here. In order to + # being able to handle that use the backslashreplace + # error handling method. This decodes unknown utf-8 + # code points to escape sequences which exactly represent + # the correct data without throwing a decoding exception. + done = self._log_streamline( + host, line.decode("utf-8", "backslashreplace")) + if done: + return + else: + more = s.recv(4096) + if not more: + buffering = False + else: + buff += more + if buff: + self._log_streamline( + host, buff.decode("utf-8", "backslashreplace")) + + def _log_streamline(self, host, line): + if "[Zuul] Task exit code" in line: + return True + elif self._streamers_stop and "[Zuul] Log not found" in line: + return True + elif "[Zuul] Log not found" in line: + # don't output this line + return False + else: + ts, ln = line.split(' | ', 1) + + self._log("%s | %s " % (host, ln), ts=ts) + return False + + def _log_module_failure(self, result, result_dict): + if 'module_stdout' in result_dict and result_dict['module_stdout']: + self._log_message( + result, status='MODULE FAILURE', + msg=result_dict['module_stdout']) + elif 'exception' in result_dict and result_dict['exception']: + self._log_message( + result, status='MODULE FAILURE', + msg=result_dict['exception']) + elif 'module_stderr' in result_dict: + self._log_message( + result, status='MODULE FAILURE', + msg=result_dict['module_stderr']) + + def v2_playbook_on_start(self, playbook): + self._playbook_name = os.path.splitext(playbook._file_name)[0] + + def v2_playbook_on_include(self, included_file): + for host in included_file._hosts: + self._log("{host} | included: {filename}".format( + host=host.name, + filename=included_file._filename)) + + def v2_playbook_on_play_start(self, play): + self._play = play + # Log an extra blank line to get space before each play + self._log("") + + # the name of a play defaults to the hosts string + name = play.get_name().strip() + msg = u"PLAY [{name}]".format(name=name) + + self._log(msg) + + def v2_playbook_on_task_start(self, task, is_conditional): + # Log an extra blank line to get space before each task + self._log("") + + self._task = task + + if self._play.strategy != 'free': + task_name = self._print_task_banner(task) + else: + task_name = task.get_name().strip() + + if task.action in ('command', 'shell'): + play_vars = self._play._variable_manager._hostvars + + hosts = self._get_task_hosts(task) + for host, inventory_hostname in hosts: + if host in ('localhost', '127.0.0.1'): + # Don't try to stream from localhost + continue + ip = play_vars[host].get( + 'ansible_host', play_vars[host].get( + 'ansible_inventory_host')) + if ip in ('localhost', '127.0.0.1'): + # Don't try to stream from localhost + continue + if task.loop: + # Don't try to stream from loops + continue + if play_vars[host].get('ansible_connection') in ('kubectl', ): + # Don't try to stream from kubectl connection + continue + + log_id = "%s-%s" % ( + task._uuid, paths._sanitize_filename(inventory_hostname)) + streamer = threading.Thread( + target=self._read_log, args=( + host, ip, log_id, task_name, hosts)) + streamer.daemon = True + streamer.start() + self._streamers.append(streamer) + + def v2_playbook_on_handler_task_start(self, task): + self.v2_playbook_on_task_start(task, False) + + def _stop_streamers(self): + self._streamers_stop = True + while True: + if not self._streamers: + break + streamer = self._streamers.pop() + streamer.join(30) + if streamer.is_alive(): + msg = "[Zuul] Log Stream did not terminate" + self._log(msg, job=True, executor=True) + self._streamers_stop = False + + def _process_result_for_localhost(self, result, is_task=True): + result_dict = dict(result._result) + localhost_names = ('localhost', '127.0.0.1', '::1') + is_localhost = False + task_host = result._host.get_name() + delegated_vars = result_dict.get('_ansible_delegated_vars', None) + if delegated_vars: + delegated_host = delegated_vars['ansible_host'] + if delegated_host in localhost_names: + is_localhost = True + elif result._task._variable_manager is None: + # Handle fact gathering which doens't have a variable manager + if task_host == 'localhost': + is_localhost = True + else: + task_hostvars = result._task._variable_manager._hostvars[task_host] + # Normally hosts in the inventory will have ansible_host + # or ansible_inventory host defined. The implied + # inventory record for 'localhost' will have neither, so + # default to that if none are supplied. + if task_hostvars.get('ansible_host', task_hostvars.get( + 'ansible_inventory_host', 'localhost')) in localhost_names: + is_localhost = True + + if not is_localhost and is_task: + self._stop_streamers() + if result._task.action in ('command', 'shell', + 'win_command', 'win_shell'): + stdout_lines = zuul_filter_result(result_dict) + # We don't have streaming for localhost and windows modules so get + # standard out after the fact. + if is_localhost or result._task.action in ( + 'win_command', 'win_shell'): + for line in stdout_lines: + hostname = self._get_hostname(result) + self._log("%s | %s " % (hostname, line)) + + def v2_runner_on_failed(self, result, ignore_errors=False): + result_dict = dict(result._result) + + self._handle_exception(result_dict) + + if result_dict.get('msg') == 'All items completed': + result_dict['status'] = 'ERROR' + self._deferred_result = result_dict + return + + self._process_result_for_localhost(result) + + if result._task.loop and 'results' in result_dict: + # items have their own events + pass + elif (result_dict.get('msg') == 'MODULE FAILURE'): + self._log_module_failure(result, result_dict) + else: + self._log_message( + result=result, status='ERROR', result_dict=result_dict) + if ignore_errors: + self._log_message(result, "Ignoring Errors", status="ERROR") + + def v2_runner_on_skipped(self, result): + if result._task.loop: + self._items_done = False + self._deferred_result = dict(result._result) + else: + reason = result._result.get('skip_reason') + if reason: + # No reason means it's an item, which we'll log differently + self._log_message(result, status='skipping', msg=reason) + + def v2_runner_item_on_skipped(self, result): + reason = result._result.get('skip_reason') + if reason: + self._log_message(result, status='skipping', msg=reason) + else: + self._log_message(result, status='skipping') + + if self._deferred_result: + self._process_deferred(result) + + def v2_runner_on_ok(self, result): + if (self._play.strategy == 'free' + and self._last_task_banner != result._task._uuid): + self._print_task_banner(result._task) + + result_dict = dict(result._result) + + self._clean_results(result_dict, result._task.action) + if '_zuul_nolog_return' in result_dict: + # We have a custom zuul module that doesn't want the parameters + # from its returned splatted to stdout. This is typically for + # modules that are collecting data to be displayed some other way. + for key in list(result_dict.keys()): + if key != 'changed': + result_dict.pop(key) + + if result_dict.get('changed', False): + status = 'changed' + else: + status = 'ok' + + if (result_dict.get('msg') == 'All items completed' + and not self._items_done): + result_dict['status'] = status + self._deferred_result = result_dict + return + + if not result._task.loop: + self._process_result_for_localhost(result) + else: + self._items_done = False + + self._handle_warnings(result_dict) + + if result._task.loop and 'results' in result_dict: + # items have their own events + pass + + elif (result_dict.get('msg') == 'MODULE FAILURE'): + self._log_module_failure(result, result_dict) + elif result._task.action == 'debug': + # this is a debug statement, handle it special + for key in [k for k in result_dict + if k.startswith('_ansible')]: + del result_dict[key] + if 'changed' in result_dict: + del result_dict['changed'] + keyname = next(iter(result_dict.keys())) + # If it has msg, that means it was like: + # + # debug: + # msg: Some debug text the user was looking for + # + # So we log it with self._log to get just the raw string the + # user provided. Note that msg may be a multi line block quote + # so we handle that here as well. + if keyname == 'msg': + msg_lines = result_dict['msg'].rstrip().split('\n') + for msg_line in msg_lines: + self._log(msg=msg_line) + else: + self._log_message( + msg=json.dumps(result_dict, indent=2, sort_keys=True), + status=status, result=result) + elif result._task.action not in ('command', 'shell'): + if 'msg' in result_dict: + self._log_message(msg=result_dict['msg'], + result=result, status=status) + else: + self._log_message( + result=result, + status=status) + elif 'results' in result_dict: + for res in result_dict['results']: + self._log_message( + result, + "Runtime: {delta}".format(**res)) + elif result_dict.get('msg') == 'All items completed': + self._log_message(result, result_dict['msg']) + else: + self._log_message( + result, + "Runtime: {delta}".format( + **result_dict)) + + def v2_runner_item_on_ok(self, result): + result_dict = dict(result._result) + self._process_result_for_localhost(result, is_task=False) + + if result_dict.get('changed', False): + status = 'changed' + else: + status = 'ok' + + if result_dict.get('msg') == 'MODULE FAILURE': + self._log_module_failure(result, result_dict) + elif result._task.action not in ('command', 'shell', + 'win_command', 'win_shell'): + if 'msg' in result_dict: + self._log_message( + result=result, msg=result_dict['msg'], status=status) + else: + self._log_message( + result=result, + msg=json.dumps(result_dict['item'], + indent=2, sort_keys=True), + status=status) + else: + stdout_lines = zuul_filter_result(result_dict) + for line in stdout_lines: + hostname = self._get_hostname(result) + self._log("%s | %s " % (hostname, line)) + + if isinstance(result_dict['item'], str): + self._log_message( + result, + "Item: {item} Runtime: {delta}".format(**result_dict)) + else: + self._log_message( + result, + "Item: Runtime: {delta}".format( + **result_dict)) + + if self._deferred_result: + self._process_deferred(result) + + def v2_runner_item_on_failed(self, result): + result_dict = dict(result._result) + self._process_result_for_localhost(result, is_task=False) + + if result_dict.get('msg') == 'MODULE FAILURE': + self._log_module_failure(result, result_dict) + elif result._task.action not in ('command', 'shell', + 'win_command', 'win_shell'): + self._log_message( + result=result, + msg="Item: {item}".format(item=result_dict['item']), + status='ERROR', + result_dict=result_dict) + else: + stdout_lines = zuul_filter_result(result_dict) + for line in stdout_lines: + hostname = self._get_hostname(result) + self._log("%s | %s " % (hostname, line)) + + # self._log("Result: %s" % (result_dict)) + self._log_message( + result, "Item: {item} Result: {rc}".format(**result_dict)) + + if self._deferred_result: + self._process_deferred(result) + + def v2_playbook_on_stats(self, stats): + # Add a spacer line before the stats so that there will be a line + # between the last task and the recap + self._log("") + + self._log("PLAY RECAP") + + hosts = sorted(stats.processed.keys()) + for host in hosts: + t = stats.summarize(host) + self._log( + "{host} |" + " ok: {ok}" + " changed: {changed}" + " unreachable: {unreachable}" + " failed: {failures}".format(host=host, **t)) + + # Add a spacer line after the stats so that there will be a line + # between each playbook + self._log("") + + def _process_deferred(self, result): + self._items_done = True + result_dict = self._deferred_result + self._deferred_result = None + status = result_dict.get('status') + + if status: + self._log_message(result, "All items complete", status=status) + + # Log an extra blank line to get space after each task + self._log("") + + def _print_task_banner(self, task): + + task_name = task.get_name().strip() + + if task.loop: + task_type = 'LOOP' + else: + task_type = 'TASK' + + # TODO(mordred) With the removal of printing task args, do we really + # want to keep doing this section? + task_args = task.args.copy() + is_shell = task_args.pop('_uses_shell', False) + if is_shell and task_name == 'command': + task_name = 'shell' + raw_params = task_args.pop('_raw_params', '').split('\n') + # If there's just a single line, go ahead and print it + if len(raw_params) == 1 and task_name in ('shell', 'command'): + task_name = '{name}: {command}'.format( + name=task_name, command=raw_params[0]) + + msg = "{task_type} [{task}]".format( + task_type=task_type, + task=task_name) + self._log(msg) + return task + + def _get_task_hosts(self, task): + result = [] + + # _restriction returns the parsed/compiled list of hosts after + # applying subsets/limits + hosts = self._play._variable_manager._inventory._restriction + for inventory_host in hosts: + # If this task has as delegate to, we don't care about the play + # hosts, we care about the task's delegate target. + if task.delegate_to: + host = task.delegate_to + else: + host = inventory_host + result.append((host, inventory_host)) + + return result + + def _dump_result_dict(self, result_dict): + result_dict = result_dict.copy() + for key in list(result_dict.keys()): + if key.startswith('_ansible'): + del result_dict[key] + zuul_filter_result(result_dict) + return result_dict + + def _log_message(self, result, msg=None, status="ok", result_dict=None): + hostname = self._get_hostname(result) + if result_dict: + result_dict = self._dump_result_dict(result_dict) + if result._task.no_log: + self._log("{host} | {msg}".format( + host=hostname, + msg="Output suppressed because no_log was given")) + return + if (not msg and result_dict + and set(result_dict.keys()) == set(['msg', 'failed'])): + msg = result_dict['msg'] + result_dict = None + if msg: + msg_lines = msg.rstrip().split('\n') + if len(msg_lines) > 1: + self._log("{host} | {status}:".format( + host=hostname, status=status)) + for msg_line in msg_lines: + self._log("{host} | {msg_line}".format( + host=hostname, msg_line=msg_line)) + else: + self._log("{host} | {status}: {msg}".format( + host=hostname, status=status, msg=msg)) + else: + self._log("{host} | {status}".format( + host=hostname, status=status, msg=msg)) + if result_dict: + result_string = json.dumps(result_dict, indent=2, sort_keys=True) + for line in result_string.split('\n'): + self._log("{host} | {line}".format(host=hostname, line=line)) + + def _get_hostname(self, result): + delegated_vars = result._result.get('_ansible_delegated_vars', None) + if delegated_vars: + return "{host} -> {delegated_host}".format( + host=result._host.get_name(), + delegated_host=delegated_vars['ansible_host']) + else: + return result._host.get_name() + + v2_runner_on_unreachable = v2_runner_on_failed diff --git a/zuul/ansible/base/callback/zuul_unreachable.py b/zuul/ansible/base/callback/zuul_unreachable.py new file mode 100644 index 000000000..8c7b85f43 --- /dev/null +++ b/zuul/ansible/base/callback/zuul_unreachable.py @@ -0,0 +1,45 @@ +# Copyright 2018 BMW Carit GmbH +# +# Zuul is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Zuul is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see <http://www.gnu.org/licenses/>. + +# This is not needed in python3 - but it is needed in python2 because there +# is a json module in ansible.plugins.callback and python2 gets confused. +# Easy local testing with ansible-playbook is handy when hacking on zuul_stream +# so just put in the __future__ statement. +from __future__ import absolute_import + +import os + +from ansible.plugins.callback import default + + +class CallbackModule(default.CallbackModule): + + CALLBACK_VERSION = 2.0 + # aggregate means we can be loaded and not be the stdout plugin + CALLBACK_TYPE = 'aggregate' + CALLBACK_NAME = 'zuul_unreachable' + + def __init__(self): + super(CallbackModule, self).__init__() + self.output_path = os.path.join( + os.environ['ZUUL_JOBDIR'], '.ansible', 'nodes.unreachable') + self.unreachable_hosts = set() + + def v2_runner_on_unreachable(self, result): + host = result._host.get_name() + if host not in self.unreachable_hosts: + self.unreachable_hosts.add(host) + with open(self.output_path, 'a') as f: + f.write('%s\n' % host) diff --git a/zuul/ansible/base/filter/__init__.py b/zuul/ansible/base/filter/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/filter/__init__.py diff --git a/zuul/ansible/base/filter/zuul_filters.py b/zuul/ansible/base/filter/zuul_filters.py new file mode 100644 index 000000000..fa21f6bd5 --- /dev/null +++ b/zuul/ansible/base/filter/zuul_filters.py @@ -0,0 +1,74 @@ +# Copyright 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def zuul_legacy_vars(zuul): + # intentionally omitted: + # BASE_LOG_PATH + # JOB_TAGS + # LOG_PATH + # ZUUL_COMMIT + # ZUUL_REF + # ZUUL_URL + # + # newly added to all builds: + # ZUUL_SHORT_PROJECT_NAME + # + # existing in most builds but newly added for periodic: + # ZUUL_BRANCH + + short_name = zuul['project']['name'].split('/')[-1] + params = dict(ZUUL_UUID=zuul['build'], + ZUUL_PROJECT=zuul['project']['name'], + ZUUL_SHORT_PROJECT_NAME=short_name, + ZUUL_PIPELINE=zuul['pipeline'], + ZUUL_VOTING=zuul['voting'], + WORKSPACE='/home/zuul/workspace') + if 'timeout' in zuul and zuul['timeout'] is not None: + params['BUILD_TIMEOUT'] = str(int(zuul['timeout']) * 1000) + if 'branch' in zuul: + params['ZUUL_BRANCH'] = zuul['branch'] + + if 'change' in zuul: + changes_str = '^'.join( + ['%s:%s:refs/changes/%s/%s/%s' % ( + i['project']['name'], + i['branch'], + str(i['change'])[-2:], + i['change'], + i['patchset']) + for i in zuul['items']]) + params['ZUUL_CHANGES'] = changes_str + + change_ids = ' '.join(['%s,%s' % (i['change'], i['patchset']) + for i in zuul['items']]) + params['ZUUL_CHANGE_IDS'] = change_ids + params['ZUUL_CHANGE'] = str(zuul['change']) + params['ZUUL_PATCHSET'] = str(zuul['patchset']) + + if 'newrev' in zuul or 'oldrev' in zuul: + params['ZUUL_REFNAME'] = zuul['ref'] + params['ZUUL_OLDREV'] = zuul.get('oldrev', '0' * 40) + params['ZUUL_NEWREV'] = zuul.get('newrev', '0' * 40) + + params['TOX_TESTENV_PASSENV'] = ' '.join(params.keys()) + return params + + +class FilterModule(object): + + def filters(self): + return { + 'zuul_legacy_vars': zuul_legacy_vars, + } diff --git a/zuul/ansible/base/library/__init__.py b/zuul/ansible/base/library/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/library/__init__.py diff --git a/zuul/ansible/base/library/command.py b/zuul/ansible/base/library/command.py new file mode 100755 index 000000000..db186788c --- /dev/null +++ b/zuul/ansible/base/library/command.py @@ -0,0 +1,652 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright: (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>, and others +# Copyright: (c) 2016, Toshio Kuratomi <tkuratomi@ansible.com> +# +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['stableinterface'], + 'supported_by': 'core'} + + +# flake8: noqa +# This file shares a significant chunk of code with an upstream ansible +# function, run_command. The goal is to not have to fork quite so much +# of that function, and discussing that design with upstream means we +# should keep the changes to substantive ones only. For that reason, this +# file is purposely not enforcing pep8, as making the function pep8 clean +# would remove our ability to easily have a discussion with our friends +# upstream + +DOCUMENTATION = ''' +--- +module: command +short_description: Executes a command on a remote node +version_added: historical +description: + - The C(command) module takes the command name followed by a list of space-delimited arguments. + - The given command will be executed on all selected nodes. It will not be + processed through the shell, so variables like C($HOME) and operations + like C("<"), C(">"), C("|"), C(";") and C("&") will not work (use the M(shell) + module if you need these features). + - For Windows targets, use the M(win_command) module instead. +options: + free_form: + description: + - The command module takes a free form command to run. There is no parameter actually named 'free form'. + See the examples! + required: yes + creates: + description: + - A filename or (since 2.0) glob pattern, when it already exists, this step will B(not) be run. + removes: + description: + - A filename or (since 2.0) glob pattern, when it does not exist, this step will B(not) be run. + version_added: "0.8" + chdir: + description: + - Change into this directory before running the command. + version_added: "0.6" + warn: + description: + - If command_warnings are on in ansible.cfg, do not warn about this particular line if set to C(no). + type: bool + default: 'yes' + version_added: "1.8" + stdin: + version_added: "2.4" + description: + - Set the stdin of the command directly to the specified value. + required: false + default: null +notes: + - If you want to run a command through the shell (say you are using C(<), C(>), C(|), etc), you actually want the M(shell) module instead. + Parsing shell metacharacters can lead to unexpected commands being executed if quoting is not done correctly so it is more secure to + use the C(command) module when possible. + - " C(creates), C(removes), and C(chdir) can be specified after the command. + For instance, if you only want to run a command if a certain file does not exist, use this." + - The C(executable) parameter is removed since version 2.4. If you have a need for this parameter, use the M(shell) module instead. + - For Windows targets, use the M(win_command) module instead. +author: + - Ansible Core Team + - Michael DeHaan +''' + +EXAMPLES = ''' +- name: return motd to registered var + command: cat /etc/motd + register: mymotd + +- name: Run the command if the specified file does not exist. + command: /usr/bin/make_database.sh arg1 arg2 creates=/path/to/database + +# You can also use the 'args' form to provide the options. +- name: This command will change the working directory to somedir/ and will only run when /path/to/database doesn't exist. + command: /usr/bin/make_database.sh arg1 arg2 + args: + chdir: somedir/ + creates: /path/to/database + +- name: safely use templated variable to run command. Always use the quote filter to avoid injection issues. + command: cat {{ myfile|quote }} + register: myoutput +''' + +RETURN = ''' +cmd: + description: the cmd that was run on the remote machine + returned: always + type: list + sample: + - echo + - hello +delta: + description: cmd end time - cmd start time + returned: always + type: string + sample: 0:00:00.001529 +end: + description: cmd end time + returned: always + type: string + sample: '2017-09-29 22:03:48.084657' +start: + description: cmd start time + returned: always + type: string + sample: '2017-09-29 22:03:48.083128' +''' + +import datetime +import glob +import os +import shlex + +from ansible.module_utils.basic import AnsibleModule + +# Imports needed for Zuul things +import re +import subprocess +import traceback +import threading +from ansible.module_utils.basic import heuristic_log_sanitize +from ansible.module_utils.six import ( + PY2, + PY3, + b, + binary_type, + string_types, + text_type, +) +from ansible.module_utils.six.moves import shlex_quote +from ansible.module_utils._text import to_native, to_bytes, to_text + + +LOG_STREAM_FILE = '/tmp/console-{log_uuid}.log' +PASSWD_ARG_RE = re.compile(r'^[-]{0,2}pass[-]?(word|wd)?') +# List to save stdout log lines in as we collect them +_log_lines = [] + + +class Console(object): + def __init__(self, log_uuid): + self.logfile_name = LOG_STREAM_FILE.format(log_uuid=log_uuid) + + def __enter__(self): + self.logfile = open(self.logfile_name, 'ab', buffering=0) + return self + + def __exit__(self, etype, value, tb): + self.logfile.close() + + def addLine(self, ln): + # Note this format with deliminator is "inspired" by the old + # Jenkins format but with microsecond resolution instead of + # millisecond. It is kept so log parsing/formatting remains + # consistent. + ts = str(datetime.datetime.now()).encode('utf-8') + if not isinstance(ln, bytes): + try: + ln = ln.encode('utf-8') + except Exception: + ln = repr(ln).encode('utf-8') + b'\n' + outln = b'%s | %s' % (ts, ln) + self.logfile.write(outln) + + +def follow(fd, log_uuid): + newline_warning = False + with Console(log_uuid) as console: + while True: + line = fd.readline() + if not line: + break + _log_lines.append(line) + if not line.endswith(b'\n'): + line += b'\n' + newline_warning = True + console.addLine(line) + if newline_warning: + console.addLine('[Zuul] No trailing newline\n') + + +# Taken from ansible/module_utils/basic.py ... forking the method for now +# so that we can dive in and figure out how to make appropriate hook points +def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, executable=None, data=None, binary_data=False, path_prefix=None, cwd=None, + use_unsafe_shell=False, prompt_regex=None, environ_update=None, umask=None, encoding='utf-8', errors='surrogate_or_strict'): + ''' + Execute a command, returns rc, stdout, and stderr. + + :arg args: is the command to run + * If args is a list, the command will be run with shell=False. + * If args is a string and use_unsafe_shell=False it will split args to a list and run with shell=False + * If args is a string and use_unsafe_shell=True it runs with shell=True. + :kw check_rc: Whether to call fail_json in case of non zero RC. + Default False + :kw close_fds: See documentation for subprocess.Popen(). Default True + :kw executable: See documentation for subprocess.Popen(). Default None + :kw data: If given, information to write to the stdin of the command + :kw binary_data: If False, append a newline to the data. Default False + :kw path_prefix: If given, additional path to find the command in. + This adds to the PATH environment vairable so helper commands in + the same directory can also be found + :kw cwd: If given, working directory to run the command inside + :kw use_unsafe_shell: See `args` parameter. Default False + :kw prompt_regex: Regex string (not a compiled regex) which can be + used to detect prompts in the stdout which would otherwise cause + the execution to hang (especially if no input data is specified) + :kw environ_update: dictionary to *update* os.environ with + :kw umask: Umask to be used when running the command. Default None + :kw encoding: Since we return native strings, on python3 we need to + know the encoding to use to transform from bytes to text. If you + want to always get bytes back, use encoding=None. The default is + "utf-8". This does not affect transformation of strings given as + args. + :kw errors: Since we return native strings, on python3 we need to + transform stdout and stderr from bytes to text. If the bytes are + undecodable in the ``encoding`` specified, then use this error + handler to deal with them. The default is ``surrogate_or_strict`` + which means that the bytes will be decoded using the + surrogateescape error handler if available (available on all + python3 versions we support) otherwise a UnicodeError traceback + will be raised. This does not affect transformations of strings + given as args. + :returns: A 3-tuple of return code (integer), stdout (native string), + and stderr (native string). On python2, stdout and stderr are both + byte strings. On python3, stdout and stderr are text strings converted + according to the encoding and errors parameters. If you want byte + strings on python3, use encoding=None to turn decoding to text off. + ''' + + if not isinstance(args, (list, binary_type, text_type)): + msg = "Argument 'args' to run_command must be list or string" + self.fail_json(rc=257, cmd=args, msg=msg) + + shell = False + if use_unsafe_shell: + + # stringify args for unsafe/direct shell usage + if isinstance(args, list): + args = " ".join([shlex_quote(x) for x in args]) + + # not set explicitly, check if set by controller + if executable: + args = [executable, '-c', args] + elif self._shell not in (None, '/bin/sh'): + args = [self._shell, '-c', args] + else: + shell = True + else: + # ensure args are a list + if isinstance(args, (binary_type, text_type)): + # On python2.6 and below, shlex has problems with text type + # On python3, shlex needs a text type. + if PY2: + args = to_bytes(args, errors='surrogate_or_strict') + elif PY3: + args = to_text(args, errors='surrogateescape') + args = shlex.split(args) + + # expand shellisms + args = [os.path.expanduser(os.path.expandvars(x)) for x in args if x is not None] + + prompt_re = None + if prompt_regex: + if isinstance(prompt_regex, text_type): + if PY3: + prompt_regex = to_bytes(prompt_regex, errors='surrogateescape') + elif PY2: + prompt_regex = to_bytes(prompt_regex, errors='surrogate_or_strict') + try: + prompt_re = re.compile(prompt_regex, re.MULTILINE) + except re.error: + self.fail_json(msg="invalid prompt regular expression given to run_command") + + rc = 0 + msg = None + st_in = None + + # Manipulate the environ we'll send to the new process + old_env_vals = {} + # We can set this from both an attribute and per call + for key, val in self.run_command_environ_update.items(): + old_env_vals[key] = os.environ.get(key, None) + os.environ[key] = val + if environ_update: + for key, val in environ_update.items(): + old_env_vals[key] = os.environ.get(key, None) + os.environ[key] = val + if path_prefix: + old_env_vals['PATH'] = os.environ['PATH'] + os.environ['PATH'] = "%s:%s" % (path_prefix, os.environ['PATH']) + + # If using test-module and explode, the remote lib path will resemble ... + # /tmp/test_module_scratch/debug_dir/ansible/module_utils/basic.py + # If using ansible or ansible-playbook with a remote system ... + # /tmp/ansible_vmweLQ/ansible_modlib.zip/ansible/module_utils/basic.py + + # Clean out python paths set by ansiballz + if 'PYTHONPATH' in os.environ: + pypaths = os.environ['PYTHONPATH'].split(':') + pypaths = [x for x in pypaths + if not x.endswith('/ansible_modlib.zip') and + not x.endswith('/debug_dir')] + os.environ['PYTHONPATH'] = ':'.join(pypaths) + if not os.environ['PYTHONPATH']: + del os.environ['PYTHONPATH'] + + # create a printable version of the command for use + # in reporting later, which strips out things like + # passwords from the args list + to_clean_args = args + if PY2: + if isinstance(args, text_type): + to_clean_args = to_bytes(args) + else: + if isinstance(args, binary_type): + to_clean_args = to_text(args) + if isinstance(args, (text_type, binary_type)): + to_clean_args = shlex.split(to_clean_args) + + clean_args = [] + is_passwd = False + for arg in (to_native(a) for a in to_clean_args): + if is_passwd: + is_passwd = False + clean_args.append('********') + continue + if PASSWD_ARG_RE.match(arg): + sep_idx = arg.find('=') + if sep_idx > -1: + clean_args.append('%s=********' % arg[:sep_idx]) + continue + else: + is_passwd = True + arg = heuristic_log_sanitize(arg, self.no_log_values) + clean_args.append(arg) + clean_args = ' '.join(shlex_quote(arg) for arg in clean_args) + + if data: + st_in = subprocess.PIPE + + # ZUUL: changed stderr to follow stdout + kwargs = dict( + executable=executable, + shell=shell, + close_fds=close_fds, + stdin=st_in, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + + # store the pwd + prev_dir = os.getcwd() + + # make sure we're in the right working directory + if cwd and os.path.isdir(cwd): + cwd = os.path.abspath(os.path.expanduser(cwd)) + kwargs['cwd'] = cwd + try: + os.chdir(cwd) + except (OSError, IOError) as e: + self.fail_json(rc=e.errno, msg="Could not open %s, %s" % (cwd, to_native(e)), + exception=traceback.format_exc()) + + old_umask = None + if umask: + old_umask = os.umask(umask) + + t = None + fail_json_kwargs = None + + try: + if self._debug: + self.log('Executing: ' + clean_args) + + # ZUUL: Replaced the execution loop with the zuul_runner run function + + cmd = subprocess.Popen(args, **kwargs) + if self.no_log: + t = None + else: + t = threading.Thread(target=follow, args=(cmd.stdout, zuul_log_id)) + t.daemon = True + t.start() + + # ZUUL: Our log thread will catch the output so don't do that here. + + # # the communication logic here is essentially taken from that + # # of the _communicate() function in ssh.py + # + # stdout = b('') + # stderr = b('') + # + # # ZUUL: stderr follows stdout + # rpipes = [cmd.stdout] + # + # if data: + # if not binary_data: + # data += '\n' + # if isinstance(data, text_type): + # data = to_bytes(data) + # cmd.stdin.write(data) + # cmd.stdin.close() + # + # while True: + # rfds, wfds, efds = select.select(rpipes, [], rpipes, 1) + # stdout += self._read_from_pipes(rpipes, rfds, cmd.stdout) + # + # # ZUUL: stderr follows stdout + # # stderr += self._read_from_pipes(rpipes, rfds, cmd.stderr) + # + # # if we're checking for prompts, do it now + # if prompt_re: + # if prompt_re.search(stdout) and not data: + # if encoding: + # stdout = to_native(stdout, encoding=encoding, errors=errors) + # else: + # stdout = stdout + # return (257, stdout, "A prompt was encountered while running a command, but no input data was specified") + # # only break out if no pipes are left to read or + # # the pipes are completely read and + # # the process is terminated + # if (not rpipes or not rfds) and cmd.poll() is not None: + # break + # # No pipes are left to read but process is not yet terminated + # # Only then it is safe to wait for the process to be finished + # # NOTE: Actually cmd.poll() is always None here if rpipes is empty + # elif not rpipes and cmd.poll() is None: + # cmd.wait() + # # The process is terminated. Since no pipes to read from are + # # left, there is no need to call select() again. + # break + + # ZUUL: If the console log follow thread *is* stuck in readline, + # we can't close stdout (attempting to do so raises an + # exception) , so this is disabled. + # cmd.stdout.close() + # cmd.stderr.close() + + rc = cmd.wait() + + # Give the thread that is writing the console log up to 10 seconds + # to catch up and exit. If it hasn't done so by then, it is very + # likely stuck in readline() because it spawed a child that is + # holding stdout or stderr open. + if t: + t.join(10) + with Console(zuul_log_id) as console: + if t.isAlive(): + console.addLine("[Zuul] standard output/error still open " + "after child exited") + # ZUUL: stdout and stderr are in the console log file + # ZUUL: return the saved log lines so we can ship them back + stdout = b('').join(_log_lines) + else: + stdout = b('') + stderr = b('') + + except (OSError, IOError) as e: + self.log("Error Executing CMD:%s Exception:%s" % (clean_args, to_native(e))) + # ZUUL: store fail_json_kwargs and fail later in finally + fail_json_kwargs = dict(rc=e.errno, msg=to_native(e), cmd=clean_args) + except Exception as e: + self.log("Error Executing CMD:%s Exception:%s" % (clean_args, to_native(traceback.format_exc()))) + # ZUUL: store fail_json_kwargs and fail later in finally + fail_json_kwargs = dict(rc=257, msg=to_native(e), exception=traceback.format_exc(), cmd=clean_args) + finally: + if t: + with Console(zuul_log_id) as console: + if t.isAlive(): + console.addLine("[Zuul] standard output/error still open " + "after child exited") + if fail_json_kwargs: + # we hit an exception and need to use the rc from + # fail_json_kwargs + rc = fail_json_kwargs['rc'] + + console.addLine("[Zuul] Task exit code: %s\n" % rc) + + if fail_json_kwargs: + self.fail_json(**fail_json_kwargs) + + # Restore env settings + for key, val in old_env_vals.items(): + if val is None: + del os.environ[key] + else: + os.environ[key] = val + + if old_umask: + os.umask(old_umask) + + if rc != 0 and check_rc: + msg = heuristic_log_sanitize(stderr.rstrip(), self.no_log_values) + self.fail_json(cmd=clean_args, rc=rc, stdout=stdout, stderr=stderr, msg=msg) + + # reset the pwd + os.chdir(prev_dir) + + if encoding is not None: + return (rc, to_native(stdout, encoding=encoding, errors=errors), + to_native(stderr, encoding=encoding, errors=errors)) + return (rc, stdout, stderr) + + +def check_command(module, commandline): + arguments = {'chown': 'owner', 'chmod': 'mode', 'chgrp': 'group', + 'ln': 'state=link', 'mkdir': 'state=directory', + 'rmdir': 'state=absent', 'rm': 'state=absent', 'touch': 'state=touch'} + commands = {'curl': 'get_url or uri', 'wget': 'get_url or uri', + 'svn': 'subversion', 'service': 'service', + 'mount': 'mount', 'rpm': 'yum, dnf or zypper', 'yum': 'yum', 'apt-get': 'apt', + 'tar': 'unarchive', 'unzip': 'unarchive', 'sed': 'replace, lineinfile or template', + 'dnf': 'dnf', 'zypper': 'zypper'} + become = ['sudo', 'su', 'pbrun', 'pfexec', 'runas', 'pmrun'] + command = os.path.basename(commandline.split()[0]) + + disable_suffix = "If you need to use command because {mod} is insufficient you can add" \ + " warn=False to this command task or set command_warnings=False in" \ + " ansible.cfg to get rid of this message." + substitutions = {'mod': None, 'cmd': command} + + if command in arguments: + msg = "Consider using the {mod} module with {subcmd} rather than running {cmd}. " + disable_suffix + substitutions['mod'] = 'file' + substitutions['subcmd'] = arguments[command] + module.warn(msg.format(**substitutions)) + + if command in commands: + msg = "Consider using the {mod} module rather than running {cmd}. " + disable_suffix + substitutions['mod'] = commands[command] + module.warn(msg.format(**substitutions)) + + if command in become: + module.warn("Consider using 'become', 'become_method', and 'become_user' rather than running %s" % (command,)) + + +def main(): + + # the command module is the one ansible module that does not take key=value args + # hence don't copy this one if you are looking to build others! + module = AnsibleModule( + argument_spec=dict( + _raw_params=dict(), + _uses_shell=dict(type='bool', default=False), + chdir=dict(type='path'), + executable=dict(), + creates=dict(type='path'), + removes=dict(type='path'), + # The default for this really comes from the action plugin + warn=dict(type='bool', default=True), + stdin=dict(required=False), + zuul_log_id=dict(type='str'), + ) + ) + + shell = module.params['_uses_shell'] + chdir = module.params['chdir'] + executable = module.params['executable'] + args = module.params['_raw_params'] + creates = module.params['creates'] + removes = module.params['removes'] + warn = module.params['warn'] + stdin = module.params['stdin'] + zuul_log_id = module.params['zuul_log_id'] + + if not shell and executable: + module.warn("As of Ansible 2.4, the parameter 'executable' is no longer supported with the 'command' module. Not using '%s'." % executable) + executable = None + + if not zuul_log_id: + module.fail_json(rc=256, msg="zuul_log_id missing: %s" % module.params) + + if not args or args.strip() == '': + module.fail_json(rc=256, msg="no command given") + + if chdir: + chdir = os.path.abspath(chdir) + os.chdir(chdir) + + if creates: + # do not run the command if the line contains creates=filename + # and the filename already exists. This allows idempotence + # of command executions. + if glob.glob(creates): + module.exit_json( + cmd=args, + stdout="skipped, since %s exists" % creates, + changed=False, + rc=0 + ) + + if removes: + # do not run the command if the line contains removes=filename + # and the filename does not exist. This allows idempotence + # of command executions. + if not glob.glob(removes): + module.exit_json( + cmd=args, + stdout="skipped, since %s does not exist" % removes, + changed=False, + rc=0 + ) + + if warn: + check_command(module, args) + + if not shell: + args = shlex.split(args) + startd = datetime.datetime.now() + + rc, out, err = zuul_run_command(module, args, zuul_log_id, executable=executable, use_unsafe_shell=shell, encoding=None, data=stdin) + + endd = datetime.datetime.now() + delta = endd - startd + + result = dict( + cmd=args, + stdout=out.rstrip(b"\r\n"), + stderr=err.rstrip(b"\r\n"), + rc=rc, + start=str(startd), + end=str(endd), + delta=str(delta), + changed=True, + zuul_log_id=zuul_log_id + ) + + if rc != 0: + module.fail_json(msg='non-zero return code', **result) + + module.exit_json(**result) + + +if __name__ == '__main__': + main() diff --git a/zuul/ansible/base/library/zuul_console.py b/zuul/ansible/base/library/zuul_console.py new file mode 100644 index 000000000..d119c5259 --- /dev/null +++ b/zuul/ansible/base/library/zuul_console.py @@ -0,0 +1,312 @@ +#!/usr/bin/python + +# Copyright (c) 2016 IBM Corp. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + +import glob +import os +import sys +import select +import socket +import subprocess +import threading +import time + +LOG_STREAM_FILE = '/tmp/console-{log_uuid}.log' +LOG_STREAM_PORT = 19885 + + +def daemonize(): + # A really basic daemonize method that should work well enough for + # now in this circumstance. Based on the public domain code at: + # http://web.archive.org/web/20131017130434/http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/ + + pid = os.fork() + if pid > 0: + return True + + os.chdir('/') + os.setsid() + os.umask(0) + + pid = os.fork() + if pid > 0: + sys.exit(0) + + sys.stdout.flush() + sys.stderr.flush() + i = open('/dev/null', 'r') + o = open('/dev/null', 'a+') + e = open('/dev/null', 'ab+', 0) + os.dup2(i.fileno(), sys.stdin.fileno()) + os.dup2(o.fileno(), sys.stdout.fileno()) + os.dup2(e.fileno(), sys.stderr.fileno()) + return False + + +class Console(object): + def __init__(self, path): + self.path = path + self.file = open(path, 'rb') + self.stat = os.stat(path) + self.size = self.stat.st_size + + +class Server(object): + + MAX_REQUEST_LEN = 1024 + REQUEST_TIMEOUT = 10 + + def __init__(self, path, port): + self.path = path + + s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + s.setsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR, 1) + s.bind(('::', port)) + s.listen(1) + + self.socket = s + + def accept(self): + conn, addr = self.socket.accept() + return conn + + def run(self): + while True: + conn = self.accept() + t = threading.Thread(target=self.handleOneConnection, args=(conn,)) + t.daemon = True + t.start() + + def chunkConsole(self, conn, log_uuid): + try: + console = Console(self.path.format(log_uuid=log_uuid)) + except Exception: + return + while True: + chunk = console.file.read(4096) + if not chunk: + break + conn.send(chunk) + return console + + def followConsole(self, console, conn): + while True: + # As long as we have unread data, keep reading/sending + while True: + chunk = console.file.read(4096) + if chunk: + conn.send(chunk) + else: + break + + # At this point, we are waiting for more data to be written + time.sleep(0.5) + + # Check to see if the remote end has sent any data, if so, + # discard + r, w, e = select.select([conn], [], [conn], 0) + if conn in e: + return False + if conn in r: + ret = conn.recv(1024) + # Discard anything read, if input is eof, it has + # disconnected. + if not ret: + return False + + # See if the file has been truncated + try: + st = os.stat(console.path) + if (st.st_ino != console.stat.st_ino or + st.st_size < console.size): + return True + except Exception: + return True + console.size = st.st_size + + def get_command(self, conn): + poll = select.poll() + bitmask = (select.POLLIN | select.POLLERR | + select.POLLHUP | select.POLLNVAL) + poll.register(conn, bitmask) + buffer = b'' + ret = None + start = time.time() + while True: + elapsed = time.time() - start + timeout = max(self.REQUEST_TIMEOUT - elapsed, 0) + if not timeout: + raise Exception("Timeout while waiting for input") + for fd, event in poll.poll(timeout): + if event & select.POLLIN: + buffer += conn.recv(self.MAX_REQUEST_LEN) + else: + raise Exception("Received error event") + if len(buffer) >= self.MAX_REQUEST_LEN: + raise Exception("Request too long") + try: + ret = buffer.decode('utf-8') + x = ret.find('\n') + if x > 0: + return ret[:x] + except UnicodeDecodeError: + pass + + def handleOneConnection(self, conn): + log_uuid = self.get_command(conn) + # use path split to make use the input isn't trying to be clever + # and construct some path like /tmp/console-/../../something + log_uuid = os.path.split(log_uuid.rstrip())[-1] + + # FIXME: this won't notice disconnects until it tries to send + console = None + try: + while True: + if console is not None: + try: + console.file.close() + except Exception: + pass + while True: + console = self.chunkConsole(conn, log_uuid) + if console: + break + conn.send('[Zuul] Log not found\n') + time.sleep(0.5) + while True: + if self.followConsole(console, conn): + break + else: + return + finally: + try: + conn.close() + except Exception: + pass + + +def get_inode(port_number=19885): + for netfile in ('/proc/net/tcp6', '/proc/net/tcp'): + if not os.path.exists(netfile): + continue + with open(netfile) as f: + # discard header line + f.readline() + for line in f: + # sl local_address rem_address st tx_queue:rx_queue tr:tm->when + # retrnsmt uid timeout inode + fields = line.split() + # Format is localaddr:localport in hex + port = int(fields[1].split(':')[1], base=16) + if port == port_number: + return fields[9] + + +def get_pid_from_inode(inode): + my_euid = os.geteuid() + exceptions = [] + for d in os.listdir('/proc'): + try: + try: + int(d) + except Exception: + continue + d_abs_path = os.path.join('/proc', d) + if os.stat(d_abs_path).st_uid != my_euid: + continue + fd_dir = os.path.join(d_abs_path, 'fd') + if os.path.exists(fd_dir): + if os.stat(fd_dir).st_uid != my_euid: + continue + for fd in os.listdir(fd_dir): + try: + fd_path = os.path.join(fd_dir, fd) + if os.path.islink(fd_path): + target = os.readlink(fd_path) + if '[' + inode + ']' in target: + return d, exceptions + except Exception as e: + exceptions.append(e) + except Exception as e: + exceptions.append(e) + return None, exceptions + + +def test(): + s = Server(LOG_STREAM_FILE, LOG_STREAM_PORT) + s.run() + + +def main(): + module = AnsibleModule( + argument_spec=dict( + path=dict(default=LOG_STREAM_FILE), + port=dict(default=LOG_STREAM_PORT, type='int'), + state=dict(default='present', choices=['absent', 'present']), + ) + ) + + p = module.params + path = p['path'] + port = p['port'] + state = p['state'] + + if state == 'present': + if daemonize(): + module.exit_json() + + s = Server(path, port) + s.run() + else: + pid = None + exceptions = [] + inode = get_inode() + if not inode: + module.fail_json( + msg="Could not find inode for port", + exceptions=[]) + + pid, exceptions = get_pid_from_inode(inode) + if not pid: + except_strings = [str(e) for e in exceptions] + module.fail_json( + msg="Could not find zuul_console process for inode", + exceptions=except_strings) + + try: + subprocess.check_output(['kill', pid]) + except subprocess.CalledProcessError as e: + module.fail_json( + msg="Could not kill zuul_console pid", + exceptions=[str(e)]) + + for fn in glob.glob(LOG_STREAM_FILE.format(log_uuid='*')): + try: + os.unlink(fn) + except Exception as e: + module.fail_json( + msg="Could not remove logfile {fn}".format(fn=fn), + exceptions=[str(e)]) + + module.exit_json() + +from ansible.module_utils.basic import * # noqa +from ansible.module_utils.basic import AnsibleModule + +if __name__ == '__main__': + main() +# test() diff --git a/zuul/ansible/base/lookup/__init__.py b/zuul/ansible/base/lookup/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/__init__.py diff --git a/zuul/ansible/base/lookup/_banned.py b/zuul/ansible/base/lookup/_banned.py new file mode 100644 index 000000000..65708f80d --- /dev/null +++ b/zuul/ansible/base/lookup/_banned.py @@ -0,0 +1,25 @@ +# Copyright 2017 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase + + +class LookupModule(LookupBase): + + def run(self, *args, **kwargs): + raise AnsibleError( + "Use of lookup modules that perform local actions on the executor" + " is forbidden.") diff --git a/zuul/ansible/base/lookup/_banned.pyi b/zuul/ansible/base/lookup/_banned.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/_banned.pyi diff --git a/zuul/ansible/base/lookup/chef_databag.py b/zuul/ansible/base/lookup/chef_databag.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/chef_databag.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/chef_databag.pyi b/zuul/ansible/base/lookup/chef_databag.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/chef_databag.pyi diff --git a/zuul/ansible/base/lookup/consul_kv.py b/zuul/ansible/base/lookup/consul_kv.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/consul_kv.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/consul_kv.pyi b/zuul/ansible/base/lookup/consul_kv.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/consul_kv.pyi diff --git a/zuul/ansible/base/lookup/credstash.py b/zuul/ansible/base/lookup/credstash.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/credstash.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/credstash.pyi b/zuul/ansible/base/lookup/credstash.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/credstash.pyi diff --git a/zuul/ansible/base/lookup/csvfile.py b/zuul/ansible/base/lookup/csvfile.py new file mode 100644 index 000000000..0d5136931 --- /dev/null +++ b/zuul/ansible/base/lookup/csvfile.py @@ -0,0 +1,44 @@ +# Copyright 2017 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + +import csv + +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_native + +from zuul.ansible import paths +csvfile = paths._import_ansible_lookup_plugin("csvfile") + + +class LookupModule(csvfile.LookupModule): + + def read_csv( + self, filename, key, delimiter, encoding='utf-8', + dflt=None, col=1): + paths._fail_if_unsafe(filename, allow_trusted=True) + + # upstream csvfile read_csv does not work with python3 so + # carry our own version. + try: + f = open(filename, 'r') + creader = csv.reader(f, dialect=csv.excel, delimiter=delimiter) + + for row in creader: + if row[0] == key: + return row[int(col)] + except Exception as e: + raise AnsibleError("csvfile: %s" % to_native(e)) + + return dflt diff --git a/zuul/ansible/base/lookup/csvfile.pyi b/zuul/ansible/base/lookup/csvfile.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/csvfile.pyi diff --git a/zuul/ansible/base/lookup/cyberarkpassword.py b/zuul/ansible/base/lookup/cyberarkpassword.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/cyberarkpassword.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/cyberarkpassword.pyi b/zuul/ansible/base/lookup/cyberarkpassword.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/cyberarkpassword.pyi diff --git a/zuul/ansible/base/lookup/dig.py b/zuul/ansible/base/lookup/dig.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/dig.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/dig.pyi b/zuul/ansible/base/lookup/dig.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/dig.pyi diff --git a/zuul/ansible/base/lookup/dnstxt.py b/zuul/ansible/base/lookup/dnstxt.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/dnstxt.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/dnstxt.pyi b/zuul/ansible/base/lookup/dnstxt.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/dnstxt.pyi diff --git a/zuul/ansible/base/lookup/env.py b/zuul/ansible/base/lookup/env.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/env.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/env.pyi b/zuul/ansible/base/lookup/env.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/env.pyi diff --git a/zuul/ansible/base/lookup/etcd.py b/zuul/ansible/base/lookup/etcd.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/etcd.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/etcd.pyi b/zuul/ansible/base/lookup/etcd.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/etcd.pyi diff --git a/zuul/ansible/base/lookup/file.py b/zuul/ansible/base/lookup/file.py new file mode 100644 index 000000000..904837003 --- /dev/null +++ b/zuul/ansible/base/lookup/file.py @@ -0,0 +1,28 @@ +# Copyright 2017 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + + +from zuul.ansible import paths +file_mod = paths._import_ansible_lookup_plugin("file") + + +class LookupModule(file_mod.LookupModule): + + def run(self, terms, variables=None, **kwargs): + for term in terms: + lookupfile = self.find_file_in_search_path( + variables, 'files', term) + paths._fail_if_unsafe(lookupfile, allow_trusted=True) + return super(LookupModule, self).run(terms, variables, **kwargs) diff --git a/zuul/ansible/base/lookup/file.pyi b/zuul/ansible/base/lookup/file.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/file.pyi diff --git a/zuul/ansible/base/lookup/fileglob.py b/zuul/ansible/base/lookup/fileglob.py new file mode 100644 index 000000000..8d6b88120 --- /dev/null +++ b/zuul/ansible/base/lookup/fileglob.py @@ -0,0 +1,45 @@ +# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com> +# Copyright 2017 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + +# Forked from lib/ansible/plugins/lookup/fileglob.py in ansible + +import os +import glob + +from zuul.ansible import paths + +from ansible.plugins.lookup import LookupBase +from ansible.module_utils._text import to_bytes, to_text + + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + + ret = [] + for term in terms: + term_file = os.path.basename(term) + dwimmed_path = self.find_file_in_search_path( + variables, 'files', os.path.dirname(term)) + if dwimmed_path: + paths._fail_if_unsafe(dwimmed_path, allow_trusted=True) + globbed = glob.glob(to_bytes( + os.path.join(dwimmed_path, term_file), + errors='surrogate_or_strict')) + ret.extend( + to_text(g, errors='surrogate_or_strict') + for g in globbed if os.path.isfile(g)) + return ret diff --git a/zuul/ansible/base/lookup/fileglob.pyi b/zuul/ansible/base/lookup/fileglob.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/fileglob.pyi diff --git a/zuul/ansible/base/lookup/filetree.py b/zuul/ansible/base/lookup/filetree.py new file mode 100644 index 000000000..490dc384c --- /dev/null +++ b/zuul/ansible/base/lookup/filetree.py @@ -0,0 +1,32 @@ +# Copyright 2017 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + +import os + +from zuul.ansible import paths +filetree = paths._import_ansible_lookup_plugin("filetree") + + +class LookupModule(filetree.LookupModule): + + def run(self, terms, variables=None, **kwargs): + basedir = self.get_basedir(variables) + for term in terms: + term_file = os.path.basename(term) + dwimmed_path = self._loader.path_dwim_relative( + basedir, 'files', os.path.dirname(term)) + path = os.path.join(dwimmed_path, term_file) + paths._fail_if_unsafe(path, allow_trusted=True) + return super(LookupModule, self).run(terms, variables, **kwargs) diff --git a/zuul/ansible/base/lookup/filetree.pyi b/zuul/ansible/base/lookup/filetree.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/filetree.pyi diff --git a/zuul/ansible/base/lookup/first_found.py b/zuul/ansible/base/lookup/first_found.py new file mode 100644 index 000000000..264296ebf --- /dev/null +++ b/zuul/ansible/base/lookup/first_found.py @@ -0,0 +1,201 @@ +# (c) 2013, seth vidal <skvidal@fedoraproject.org> red hat, inc +# Copyright 2017 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + +# take a list of files and (optionally) a list of paths +# return the first existing file found in the paths +# [file1, file2, file3], [path1, path2, path3] +# search order is: +# path1/file1 +# path1/file2 +# path1/file3 +# path2/file1 +# path2/file2 +# path2/file3 +# path3/file1 +# path3/file2 +# path3/file3 + +# first file found with os.path.exists() is returned +# no file matches raises ansibleerror +# EXAMPLES +# - name: copy first existing file found to /some/file +# action: copy src=$item dest=/some/file +# with_first_found: +# - files: foo ${inventory_hostname} bar +# paths: /tmp/production /tmp/staging + +# that will look for files in this order: +# /tmp/production/foo +# ${inventory_hostname} +# bar +# /tmp/staging/foo +# ${inventory_hostname} +# bar + +# - name: copy first existing file found to /some/file +# action: copy src=$item dest=/some/file +# with_first_found: +# - files: /some/place/foo ${inventory_hostname} /some/place/else + +# that will look for files in this order: +# /some/place/foo +# $relative_path/${inventory_hostname} +# /some/place/else + +# example - including tasks: +# tasks: +# - include: $item +# with_first_found: +# - files: generic +# paths: tasks/staging tasks/production +# this will include the tasks in the file generic where it is found first +# (staging or production) + +# example simple file lists +# tasks: +# - name: first found file +# action: copy src=$item dest=/etc/file.cfg +# with_first_found: +# - files: foo.${inventory_hostname} foo + + +# example skipping if no matched files +# First_found also offers the ability to control whether or not failing +# to find a file returns an error or not +# +# - name: first found file - or skip +# action: copy src=$item dest=/etc/file.cfg +# with_first_found: +# - files: foo.${inventory_hostname} +# skip: true + +# example a role with default configuration and configuration per host +# you can set multiple terms with their own files and paths to look through. +# consider a role that sets some configuration per host falling back on a +# default config. +# +# - name: some configuration template +# template: src={{ item }} dest=/etc/file.cfg mode=0444 owner=root group=root +# with_first_found: +# - files: +# - ${inventory_hostname}/etc/file.cfg +# paths: +# - ../../../templates.overwrites +# - ../../../templates +# - files: +# - etc/file.cfg +# paths: +# - templates + +# the above will return an empty list if the files cannot be found at all +# if skip is unspecificed or if it is set to false then it will return a list +# error which can be caught bye ignore_errors: true for that action. + +# finally - if you want you can use it, in place to replace +# first_available_file: +# you simply cannot use the - files, path or skip options. simply replace +# first_available_file with with_first_found and leave the file listing in +# place +# +# +# - name: with_first_found like first_available_file +# action: copy src=$item dest=/tmp/faftest +# with_first_found: +# - ../files/foo +# - ../files/bar +# - ../files/baz +# ignore_errors: true + +import os + +from jinja2.exceptions import UndefinedError + +from ansible.constants import mk_boolean as boolean +from ansible.errors import AnsibleLookupError +from ansible.errors import AnsibleUndefinedVariable +from ansible.module_utils.six import string_types +from ansible.plugins.lookup import LookupBase + +from zuul.ansible import paths as zuul_paths + + +class LookupModule(LookupBase): + + def run(self, terms, variables, **kwargs): + + anydict = False + skip = False + + for term in terms: + if isinstance(term, dict): + anydict = True + + total_search = [] + if anydict: + for term in terms: + if isinstance(term, dict): + files = term.get('files', []) + paths = term.get('paths', []) + skip = boolean(term.get('skip', False)) + + filelist = files + if isinstance(files, string_types): + files = files.replace(',', ' ') + files = files.replace(';', ' ') + filelist = files.split(' ') + + pathlist = paths + if paths: + if isinstance(paths, string_types): + paths = paths.replace(',', ' ') + paths = paths.replace(':', ' ') + paths = paths.replace(';', ' ') + pathlist = paths.split(' ') + + if not pathlist: + total_search = filelist + else: + for path in pathlist: + for fn in filelist: + f = os.path.join(path, fn) + total_search.append(f) + else: + total_search.append(term) + else: + total_search = self._flatten(terms) + + for fn in total_search: + zuul_paths._fail_if_unsafe(fn, allow_trusted=True) + try: + fn = self._templar.template(fn) + except (AnsibleUndefinedVariable, UndefinedError): + continue + + # get subdir if set by task executor, default to files otherwise + subdir = getattr(self, '_subdir', 'files') + path = None + path = self.find_file_in_search_path( + variables, subdir, fn, ignore_missing=True) + if path is not None: + return [path] + else: + if skip: + return [] + else: + raise AnsibleLookupError( + "No file was found when using with_first_found. Use the" + " 'skip: true' option to allow this task to be skipped if" + " no files are found") diff --git a/zuul/ansible/base/lookup/first_found.pyi b/zuul/ansible/base/lookup/first_found.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/first_found.pyi diff --git a/zuul/ansible/base/lookup/hashi_valut.py b/zuul/ansible/base/lookup/hashi_valut.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/hashi_valut.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/hashi_valut.pyi b/zuul/ansible/base/lookup/hashi_valut.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/hashi_valut.pyi diff --git a/zuul/ansible/base/lookup/hiera.py b/zuul/ansible/base/lookup/hiera.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/hiera.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/hiera.pyi b/zuul/ansible/base/lookup/hiera.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/hiera.pyi diff --git a/zuul/ansible/base/lookup/ini.py b/zuul/ansible/base/lookup/ini.py new file mode 100644 index 000000000..83d7b144a --- /dev/null +++ b/zuul/ansible/base/lookup/ini.py @@ -0,0 +1,31 @@ +# Copyright 2017 Red Hat, Inc. +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see <http://www.gnu.org/licenses/>. + + +from zuul.ansible import paths +ini = paths._import_ansible_lookup_plugin("ini") + + +class LookupModule(ini.LookupModule): + + def read_properties(self, filename, *args, **kwargs): + paths._fail_if_unsafe(filename, allow_trusted=True) + return super(LookupModule, self).read_properties( + filename, *args, **kwargs) + + def read_ini(self, filename, *args, **kwargs): + paths._fail_if_unsafe(filename, allow_trusted=True) + return super(LookupModule, self).read_ini( + filename, *args, **kwargs) diff --git a/zuul/ansible/base/lookup/ini.pyi b/zuul/ansible/base/lookup/ini.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/ini.pyi diff --git a/zuul/ansible/base/lookup/keyring.py b/zuul/ansible/base/lookup/keyring.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/keyring.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/keyring.pyi b/zuul/ansible/base/lookup/keyring.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/keyring.pyi diff --git a/zuul/ansible/base/lookup/lastpass.py b/zuul/ansible/base/lookup/lastpass.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/lastpass.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/lastpass.pyi b/zuul/ansible/base/lookup/lastpass.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/lastpass.pyi diff --git a/zuul/ansible/base/lookup/lines.py b/zuul/ansible/base/lookup/lines.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/lines.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/lines.pyi b/zuul/ansible/base/lookup/lines.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/lines.pyi diff --git a/zuul/ansible/base/lookup/mongodb.py b/zuul/ansible/base/lookup/mongodb.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/mongodb.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/mongodb.pyi b/zuul/ansible/base/lookup/mongodb.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/mongodb.pyi diff --git a/zuul/ansible/base/lookup/password.py b/zuul/ansible/base/lookup/password.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/password.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/password.pyi b/zuul/ansible/base/lookup/password.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/password.pyi diff --git a/zuul/ansible/base/lookup/passwordstore.py b/zuul/ansible/base/lookup/passwordstore.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/passwordstore.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/passwordstore.pyi b/zuul/ansible/base/lookup/passwordstore.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/passwordstore.pyi diff --git a/zuul/ansible/base/lookup/pipe.py b/zuul/ansible/base/lookup/pipe.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/pipe.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/pipe.pyi b/zuul/ansible/base/lookup/pipe.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/pipe.pyi diff --git a/zuul/ansible/base/lookup/redis_kv.py b/zuul/ansible/base/lookup/redis_kv.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/redis_kv.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/redis_kv.pyi b/zuul/ansible/base/lookup/redis_kv.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/redis_kv.pyi diff --git a/zuul/ansible/base/lookup/shelvefile.py b/zuul/ansible/base/lookup/shelvefile.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/shelvefile.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/shelvefile.pyi b/zuul/ansible/base/lookup/shelvefile.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/shelvefile.pyi diff --git a/zuul/ansible/base/lookup/template.py b/zuul/ansible/base/lookup/template.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/template.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/template.pyi b/zuul/ansible/base/lookup/template.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/template.pyi diff --git a/zuul/ansible/base/lookup/url.py b/zuul/ansible/base/lookup/url.py new file mode 120000 index 000000000..d45b9c405 --- /dev/null +++ b/zuul/ansible/base/lookup/url.py @@ -0,0 +1 @@ +_banned.py
\ No newline at end of file diff --git a/zuul/ansible/base/lookup/url.pyi b/zuul/ansible/base/lookup/url.pyi new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/zuul/ansible/base/lookup/url.pyi |