diff options
70 files changed, 300 insertions, 1726 deletions
diff --git a/.zuul.yaml b/.zuul.yaml index 9d9000756..06c167e9b 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -34,21 +34,6 @@ - playbooks/zuul-stream/.* - job: - name: zuul-stream-functional-2.8 - parent: zuul-stream-functional - # Force executor to use same Ansible version as "controller" node so - # that the inventory.yaml file will be correct for that version. - ansible-version: 2.8 - vars: - zuul_ansible_version: 2.8 - -- job: - name: zuul-stream-functional-2.9 - parent: zuul-stream-functional - vars: - zuul_ansible_version: 2.9 - -- job: name: zuul-stream-functional-5 parent: zuul-stream-functional vars: @@ -323,8 +308,6 @@ files: - web/.* nodeset: ubuntu-jammy - - zuul-stream-functional-2.8 - - zuul-stream-functional-2.9 - zuul-stream-functional-5 - zuul-stream-functional-6 - zuul-tox-remote @@ -354,8 +337,6 @@ files: - web/.* nodeset: ubuntu-jammy - - zuul-stream-functional-2.8 - - zuul-stream-functional-2.9 - zuul-stream-functional-5 - zuul-stream-functional-6 - zuul-tox-remote diff --git a/doc/source/examples/docker-compose.yaml b/doc/source/examples/docker-compose.yaml index ee0093b3c..af5c970de 100644 --- a/doc/source/examples/docker-compose.yaml +++ b/doc/source/examples/docker-compose.yaml @@ -26,7 +26,7 @@ services: - "../../../tools/:/var/zuul-tools/:z" # NOTE(pabelanger): Be sure to update this line each time we change the # default version of ansible for Zuul. - command: "/usr/local/lib/zuul/ansible/2.9/bin/ansible-playbook /var/playbooks/setup.yaml" + command: "/usr/local/lib/zuul/ansible/5/bin/ansible-playbook /var/playbooks/setup.yaml" networks: - zuul zk: diff --git a/doc/source/howtos/nodepool_static.rst b/doc/source/howtos/nodepool_static.rst index c10672e7b..163497bcf 100644 --- a/doc/source/howtos/nodepool_static.rst +++ b/doc/source/howtos/nodepool_static.rst @@ -69,14 +69,30 @@ Log streaming The log streaming service enables Zuul to show the live status of long-running ``shell`` or ``command`` tasks. The server side is setup by the ``zuul_console:`` task built-in to Zuul's Ansible installation. -The executor requires the ability to communicate with the job nodes on -port 19885 for this to work. +The executor requires the ability to communicate with this server on +the job nodes via port ``19885`` for this to work. -The log streaming service may leave files on the static node in the -format ``/tmp/console-<uuid>-<task_id>-<host>.log`` if jobs are -interrupted. These may be safely removed after a short period of -inactivity with a command such as +The log streaming service spools command output via files on the job +node in the format ``/tmp/console-<uuid>-<task_id>-<host>.log``. By +default, it will clean these files up automatically. + +Occasionally, a streaming file may be left if a job is interrupted. +These may be safely removed after a short period of inactivity with a +command such as .. code-block:: shell find /tmp -maxdepth 1 -name 'console-*-*-<host>.log' -mtime +2 -delete + +If the executor is unable to reach port ``19885`` (for example due to +firewall rules), or the ``zuul_console`` daemon can not be run for +some other reason, the command to clean these spool files will not be +processed and they may be left behind; on an ephemeral node this is +not usually a problem, but on a static node these files will persist. + +In this situation, , Zuul can be instructed to not to create any spool +files for ``shell`` and ``command`` tasks via setting +``zuul_console_disable: True`` (usually via a global host variable in +inventory). Live streaming of ``shell`` and ``command`` calls will of +course be unavailable in this case, but no spool files will be +created. diff --git a/doc/source/monitoring.rst b/doc/source/monitoring.rst index 1cb61ee01..7f99c7f7f 100644 --- a/doc/source/monitoring.rst +++ b/doc/source/monitoring.rst @@ -520,6 +520,8 @@ These metrics are emitted by the Zuul :ref:`scheduler`: requests is better. Ideally, there will be no `failed` requests. +.. stat:: zuul.nodepool + .. stat:: current_requests :type: gauge @@ -819,18 +821,18 @@ Liveness Probes The Prometheus server also supports liveness and ready probes at the following URIS: -.. path:: /health/live +.. path:: health/live Returns 200 as long as the process is running. -.. path:: /health/ready +.. path:: health/ready Returns 200 if the process is in `RUNNING` or `PAUSED` states. Otherwise, returns 503. Note that 503 is returned for `INITIALIZED`, so this may be used to determine when a component has completely finished loading configuration. -.. path:: /health/status +.. path:: health/status This always returns 200, but includes the component status as the text body of the response. diff --git a/playbooks/zuul-stream/fixtures/test-stream.yaml b/playbooks/zuul-stream/fixtures/test-stream.yaml index 0326ae54e..488f8cb2f 100644 --- a/playbooks/zuul-stream/fixtures/test-stream.yaml +++ b/playbooks/zuul-stream/fixtures/test-stream.yaml @@ -11,6 +11,14 @@ port: 19887 when: new_console | default(false) +- name: Run command to show skipping works + vars: + zuul_console_disabled: true + hosts: node + tasks: + - name: Run quiet command + command: echo 'This command should not stream' + - name: Run some commands to show that logging works hosts: node tasks: diff --git a/playbooks/zuul-stream/functional.yaml b/playbooks/zuul-stream/functional.yaml index 63e13e3f5..fa523f7c2 100644 --- a/playbooks/zuul-stream/functional.yaml +++ b/playbooks/zuul-stream/functional.yaml @@ -72,7 +72,7 @@ mv job-output.txt job-output-success-19885.txt mv job-output.json job-output-success-19885.json - - name: Validate outputs + - name: Validate text outputs include_tasks: validate.yaml loop: - { node: 'node1', filename: 'job-output-success-19887.txt' } @@ -82,6 +82,13 @@ # node3 only listen on 19887 - { node: 'node3', filename: 'job-output-success-19887.txt' } + # This shows that zuul_console_disabled has activated and set the + # UUID to "skip" + - name: Validate json output + shell: | + egrep 'zuul_log_id": "skip"' job-output-success-19885.json + egrep 'zuul_log_id": "skip"' job-output-success-19887.json + # failure case - name: Run ansible playbook that should fail diff --git a/releasenotes/notes/ansible-2-removed-482fa9e5b944b7d4.yaml b/releasenotes/notes/ansible-2-removed-482fa9e5b944b7d4.yaml new file mode 100644 index 000000000..26c1e59ee --- /dev/null +++ b/releasenotes/notes/ansible-2-removed-482fa9e5b944b7d4.yaml @@ -0,0 +1,6 @@ +--- +upgrade: + - | + Support for Ansible versions 2.8 and 2.9 has been removed. + Migrate any existing jobs which rely on these versions to Ansible + version 5 or 6 before upgrading. diff --git a/tests/base.py b/tests/base.py index 317d9d175..b85bce504 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1033,6 +1033,11 @@ class GerritWebServer(object): self.send_response(500) self.end_headers() + def _403(self, msg): + self.send_response(403) + self.end_headers() + self.wfile.write(msg.encode('utf8')) + def _404(self): self.send_response(404) self.end_headers() @@ -1075,6 +1080,9 @@ class GerritWebServer(object): if not change: return self._404() + if not fake_gerrit._fake_submit_permission: + return self._403('submit not permitted') + candidate = self._get_change(change_id) sr = candidate.getSubmitRecords() if sr[0]['status'] != 'OK': @@ -1308,6 +1316,8 @@ class FakeGerritConnection(gerritconnection.GerritConnection): self._poller_event = poller_event self._ref_watcher_event = ref_watcher_event self._fake_submit_whole_topic = False + self._fake_submit_permission = True + self.submit_retry_backoff = 0 def onStop(self): super().onStop() diff --git a/tests/fixtures/config/ansible-callbacks/main28.yaml b/tests/fixtures/config/ansible-callbacks/main28.yaml deleted file mode 100644 index 371710b4f..000000000 --- a/tests/fixtures/config/ansible-callbacks/main28.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- tenant: - name: tenant-one - default-ansible-version: '2.8' - source: - gerrit: - config-projects: - - common-config diff --git a/tests/fixtures/config/ansible-callbacks/main29.yaml b/tests/fixtures/config/ansible-callbacks/main29.yaml deleted file mode 100644 index b127139a9..000000000 --- a/tests/fixtures/config/ansible-callbacks/main29.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- tenant: - name: tenant-one - default-ansible-version: '2.9' - source: - gerrit: - config-projects: - - common-config diff --git a/tests/fixtures/config/ansible-versions/git/common-config/zuul.yaml b/tests/fixtures/config/ansible-versions/git/common-config/zuul.yaml index e1bac5e01..497053c90 100644 --- a/tests/fixtures/config/ansible-versions/git/common-config/zuul.yaml +++ b/tests/fixtures/config/ansible-versions/git/common-config/zuul.yaml @@ -34,23 +34,7 @@ parent: ansible-version vars: test_ansible_version_major: 2 - test_ansible_version_minor: 8 - -- job: - name: ansible-28 - parent: ansible-version - ansible-version: 2.8 - vars: - test_ansible_version_major: 2 - test_ansible_version_minor: 8 - -- job: - name: ansible-29 - parent: ansible-version - ansible-version: 2.9 - vars: - test_ansible_version_major: 2 - test_ansible_version_minor: 9 + test_ansible_version_minor: 13 - job: name: ansible-5 @@ -73,8 +57,6 @@ check: jobs: - ansible-default - - ansible-28 - - ansible-29 - ansible-5 - ansible-6 @@ -83,7 +65,5 @@ check: jobs: - ansible-default-zuul-conf - - ansible-28 - - ansible-29 - ansible-5 - ansible-6 diff --git a/tests/fixtures/config/ansible/main28.yaml b/tests/fixtures/config/ansible/main28.yaml deleted file mode 100644 index f2add49c7..000000000 --- a/tests/fixtures/config/ansible/main28.yaml +++ /dev/null @@ -1,11 +0,0 @@ -- tenant: - name: tenant-one - default-ansible-version: '2.8' - source: - gerrit: - config-projects: - - common-config - untrusted-projects: - - org/project - - bare-role - - org/ansible diff --git a/tests/fixtures/config/ansible/main29.yaml b/tests/fixtures/config/ansible/main29.yaml deleted file mode 100644 index 758292950..000000000 --- a/tests/fixtures/config/ansible/main29.yaml +++ /dev/null @@ -1,11 +0,0 @@ -- tenant: - name: tenant-one - default-ansible-version: '2.9' - source: - gerrit: - config-projects: - - common-config - untrusted-projects: - - org/project - - bare-role - - org/ansible diff --git a/tests/fixtures/config/executor-facts/main28.yaml b/tests/fixtures/config/executor-facts/main28.yaml deleted file mode 100644 index 686899bf8..000000000 --- a/tests/fixtures/config/executor-facts/main28.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- tenant: - name: tenant-one - default-ansible-version: '2.8' - source: - gerrit: - config-projects: - - common-config - untrusted-projects: - - org/project diff --git a/tests/fixtures/config/executor-facts/main29.yaml b/tests/fixtures/config/executor-facts/main29.yaml deleted file mode 100644 index df934ff22..000000000 --- a/tests/fixtures/config/executor-facts/main29.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- tenant: - name: tenant-one - default-ansible-version: '2.9' - source: - gerrit: - config-projects: - - common-config - untrusted-projects: - - org/project diff --git a/tests/fixtures/config/inventory/git/common-config/zuul.yaml b/tests/fixtures/config/inventory/git/common-config/zuul.yaml index ca687139d..97ee54fb9 100644 --- a/tests/fixtures/config/inventory/git/common-config/zuul.yaml +++ b/tests/fixtures/config/inventory/git/common-config/zuul.yaml @@ -115,24 +115,6 @@ run: playbooks/network.yaml - job: - name: ansible-version28-inventory - nodeset: - nodes: - - name: ubuntu-xenial - label: ubuntu-xenial - ansible-version: '2.8' - run: playbooks/ansible-version.yaml - -- job: - name: ansible-version29-inventory - nodeset: - nodes: - - name: ubuntu-xenial - label: ubuntu-xenial - ansible-version: '2.9' - run: playbooks/ansible-version.yaml - -- job: name: ansible-version5-inventory nodeset: nodes: diff --git a/tests/fixtures/config/inventory/git/org_project/.zuul.yaml b/tests/fixtures/config/inventory/git/org_project/.zuul.yaml index 69ec3127e..78f65ea83 100644 --- a/tests/fixtures/config/inventory/git/org_project/.zuul.yaml +++ b/tests/fixtures/config/inventory/git/org_project/.zuul.yaml @@ -7,7 +7,5 @@ - executor-only-inventory - group-inventory - hostvars-inventory - - ansible-version28-inventory - - ansible-version29-inventory - ansible-version5-inventory - ansible-version6-inventory diff --git a/tests/fixtures/config/inventory/git/org_project3/.zuul.yaml b/tests/fixtures/config/inventory/git/org_project3/.zuul.yaml index 6cf9faf88..0f11335da 100644 --- a/tests/fixtures/config/inventory/git/org_project3/.zuul.yaml +++ b/tests/fixtures/config/inventory/git/org_project3/.zuul.yaml @@ -2,8 +2,3 @@ check: jobs: - single-inventory - - single-inventory-list - - executor-only-inventory - - group-inventory - - hostvars-inventory - - ansible-version28-inventory diff --git a/tests/fixtures/zuul-default-ansible-version.conf b/tests/fixtures/zuul-default-ansible-version.conf index 6ec8e38df..000f27723 100644 --- a/tests/fixtures/zuul-default-ansible-version.conf +++ b/tests/fixtures/zuul-default-ansible-version.conf @@ -6,7 +6,8 @@ server=127.0.0.1 [scheduler] tenant_config=main.yaml relative_priority=true -default_ansible_version=2.8 +# Used by ansible-default-zuul-conf job +default_ansible_version=6 [merger] git_dir=/tmp/zuul-test/merger-git diff --git a/tests/remote/test_remote_action_modules.py b/tests/remote/test_remote_action_modules.py index 30e430b74..923b877c2 100644 --- a/tests/remote/test_remote_action_modules.py +++ b/tests/remote/test_remote_action_modules.py @@ -21,7 +21,7 @@ from tests.base import AnsibleZuulTestCase class FunctionalActionModulesMixIn: tenant_config_file = 'config/remote-action-modules/main.yaml' # This should be overriden in child classes. - ansible_version = '2.9' + ansible_version = '5' wait_timeout = 120 def _setUp(self): @@ -87,22 +87,6 @@ class FunctionalActionModulesMixIn: self._run_job('shell-good', 'SUCCESS') -class TestActionModules28(AnsibleZuulTestCase, FunctionalActionModulesMixIn): - ansible_version = '2.8' - - def setUp(self): - super().setUp() - self._setUp() - - -class TestActionModules29(AnsibleZuulTestCase, FunctionalActionModulesMixIn): - ansible_version = '2.9' - - def setUp(self): - super().setUp() - self._setUp() - - class TestActionModules5(AnsibleZuulTestCase, FunctionalActionModulesMixIn): ansible_version = '5' diff --git a/tests/remote/test_remote_zuul_json.py b/tests/remote/test_remote_zuul_json.py index e4510e7d1..4451c3630 100644 --- a/tests/remote/test_remote_zuul_json.py +++ b/tests/remote/test_remote_zuul_json.py @@ -144,22 +144,6 @@ class FunctionalZuulJSONMixIn: dateutil.parser.parse(play_end_time) -class TestZuulJSON28(AnsibleZuulTestCase, FunctionalZuulJSONMixIn): - ansible_version = '2.8' - - def setUp(self): - super().setUp() - self._setUp() - - -class TestZuulJSON29(AnsibleZuulTestCase, FunctionalZuulJSONMixIn): - ansible_version = '2.9' - - def setUp(self): - super().setUp() - self._setUp() - - class TestZuulJSON5(AnsibleZuulTestCase, FunctionalZuulJSONMixIn): ansible_version = '5' diff --git a/tests/remote/test_remote_zuul_stream.py b/tests/remote/test_remote_zuul_stream.py index b84c4b0d8..7a7fef08d 100644 --- a/tests/remote/test_remote_zuul_stream.py +++ b/tests/remote/test_remote_zuul_stream.py @@ -25,7 +25,7 @@ from tests.base import AnsibleZuulTestCase class FunctionalZuulStreamMixIn: tenant_config_file = 'config/remote-zuul-stream/main.yaml' # This should be overriden in child classes. - ansible_version = '2.9' + ansible_version = '5' def _setUp(self): self.log_console_port = 19000 + int( @@ -201,8 +201,6 @@ class FunctionalZuulStreamMixIn: self.assertLogLine( r'controller \| ok: Runtime: \d:\d\d:\d\d\.\d\d\d\d\d\d', text) self.assertLogLine('PLAY RECAP', text) - # NOTE(pabelanger): Ansible 2.8 added new stats - # skipped, rescued, ignored. self.assertLogLine( r'controller \| ok: \d+ changed: \d+ unreachable: 0 failed: 0 ' 'skipped: 0 rescued: 1 ignored: 0', text) @@ -255,24 +253,6 @@ class FunctionalZuulStreamMixIn: self.assertLogLine(regex, text) -class TestZuulStream28(AnsibleZuulTestCase, FunctionalZuulStreamMixIn): - ansible_version = '2.8' - ansible_core_version = '2.8' - - def setUp(self): - super().setUp() - self._setUp() - - -class TestZuulStream29(AnsibleZuulTestCase, FunctionalZuulStreamMixIn): - ansible_version = '2.9' - ansible_core_version = '2.9' - - def setUp(self): - super().setUp() - self._setUp() - - class TestZuulStream5(AnsibleZuulTestCase, FunctionalZuulStreamMixIn): ansible_version = '5' ansible_core_version = '2.12' diff --git a/tests/unit/test_executor.py b/tests/unit/test_executor.py index 27ca75531..31e3b74c5 100644 --- a/tests/unit/test_executor.py +++ b/tests/unit/test_executor.py @@ -870,16 +870,6 @@ class ExecutorFactsMixin: output) -class TestExecutorFacts28(AnsibleZuulTestCase, ExecutorFactsMixin): - tenant_config_file = 'config/executor-facts/main28.yaml' - ansible_major_minor = '2.8' - - -class TestExecutorFacts29(AnsibleZuulTestCase, ExecutorFactsMixin): - tenant_config_file = 'config/executor-facts/main29.yaml' - ansible_major_minor = '2.9' - - class TestExecutorFacts5(AnsibleZuulTestCase, ExecutorFactsMixin): tenant_config_file = 'config/executor-facts/main5.yaml' ansible_major_minor = '2.12' @@ -941,20 +931,6 @@ class AnsibleCallbackConfigsMixin: output) -class TestAnsibleCallbackConfigs28(AnsibleZuulTestCase, - AnsibleCallbackConfigsMixin): - config_file = 'zuul-executor-ansible-callback.conf' - tenant_config_file = 'config/ansible-callbacks/main28.yaml' - ansible_major_minor = '2.8' - - -class TestAnsibleCallbackConfigs29(AnsibleZuulTestCase, - AnsibleCallbackConfigsMixin): - config_file = 'zuul-executor-ansible-callback.conf' - tenant_config_file = 'config/ansible-callbacks/main29.yaml' - ansible_major_minor = '2.9' - - class TestAnsibleCallbackConfigs5(AnsibleZuulTestCase, AnsibleCallbackConfigsMixin): config_file = 'zuul-executor-ansible-callback.conf' diff --git a/tests/unit/test_gerrit.py b/tests/unit/test_gerrit.py index 47545b9be..e98cb80b5 100644 --- a/tests/unit/test_gerrit.py +++ b/tests/unit/test_gerrit.py @@ -359,6 +359,25 @@ class TestGerritWeb(ZuulTestCase): changes="1,1 2,1 3,1"), ], ordered=False) + def test_submit_failure(self): + # Test that we log the reason for a submit failure (403 error) + self.fake_gerrit._fake_submit_permission = False + A = self.fake_gerrit.addFakeChange('org/project1', "master", "A") + A.addApproval('Code-Review', 2) + with self.assertLogs('zuul.test.FakeGerritConnection', level='INFO' + ) as full_logs: + self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) + self.waitUntilSettled() + self.log.debug("Full logs:") + for x in full_logs.output: + self.log.debug(x) + self.assertRegexInList( + r'Error submitting data to gerrit on attempt 3: ' + 'Received response 403: submit not permitted', + full_logs.output) + + self.assertEqual(A.data['status'], 'NEW') + class TestFileComments(AnsibleZuulTestCase): config_file = 'zuul-gerrit-web.conf' diff --git a/tests/unit/test_inventory.py b/tests/unit/test_inventory.py index 83a62a0e7..acfe1a3ae 100644 --- a/tests/unit/test_inventory.py +++ b/tests/unit/test_inventory.py @@ -180,56 +180,6 @@ class TestInventoryShellType(TestInventoryBase): class TestInventoryAutoPython(TestInventoryBase): - def test_auto_python_ansible28_inventory(self): - inventory = self._get_build_inventory('ansible-version28-inventory') - - all_nodes = ('ubuntu-xenial',) - self.assertIn('all', inventory) - self.assertIn('hosts', inventory['all']) - self.assertIn('vars', inventory['all']) - for node_name in all_nodes: - self.assertIn(node_name, inventory['all']['hosts']) - node_vars = inventory['all']['hosts'][node_name] - self.assertEqual( - 'auto', node_vars['ansible_python_interpreter']) - - self.assertIn('zuul', inventory['all']['vars']) - z_vars = inventory['all']['vars']['zuul'] - self.assertIn('executor', z_vars) - self.assertIn('src_root', z_vars['executor']) - self.assertIn('job', z_vars) - self.assertEqual(z_vars['ansible_version'], '2.8') - self.assertEqual(z_vars['job'], 'ansible-version28-inventory') - self.assertEqual(z_vars['message'], 'QQ==') - - self.executor_server.release() - self.waitUntilSettled() - - def test_auto_python_ansible29_inventory(self): - inventory = self._get_build_inventory('ansible-version29-inventory') - - all_nodes = ('ubuntu-xenial',) - self.assertIn('all', inventory) - self.assertIn('hosts', inventory['all']) - self.assertIn('vars', inventory['all']) - for node_name in all_nodes: - self.assertIn(node_name, inventory['all']['hosts']) - node_vars = inventory['all']['hosts'][node_name] - self.assertEqual( - 'auto', node_vars['ansible_python_interpreter']) - - self.assertIn('zuul', inventory['all']['vars']) - z_vars = inventory['all']['vars']['zuul'] - self.assertIn('executor', z_vars) - self.assertIn('src_root', z_vars['executor']) - self.assertIn('job', z_vars) - self.assertEqual(z_vars['ansible_version'], '2.9') - self.assertEqual(z_vars['job'], 'ansible-version29-inventory') - self.assertEqual(z_vars['message'], 'QQ==') - - self.executor_server.release() - self.waitUntilSettled() - def test_auto_python_ansible5_inventory(self): inventory = self._get_build_inventory('ansible-version5-inventory') diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py index 3d41d74c6..fe5b46e0c 100644 --- a/tests/unit/test_v3.py +++ b/tests/unit/test_v3.py @@ -3906,16 +3906,6 @@ class FunctionalAnsibleMixIn(object): output) -class TestAnsible28(AnsibleZuulTestCase, FunctionalAnsibleMixIn): - tenant_config_file = 'config/ansible/main28.yaml' - ansible_major_minor = '2.8' - - -class TestAnsible29(AnsibleZuulTestCase, FunctionalAnsibleMixIn): - tenant_config_file = 'config/ansible/main29.yaml' - ansible_major_minor = '2.9' - - class TestAnsible5(AnsibleZuulTestCase, FunctionalAnsibleMixIn): tenant_config_file = 'config/ansible/main5.yaml' ansible_major_minor = '2.12' @@ -8016,8 +8006,6 @@ class TestAnsibleVersion(AnsibleZuulTestCase): self.assertHistory([ dict(name='ansible-default', result='SUCCESS', changes='1,1'), - dict(name='ansible-28', result='SUCCESS', changes='1,1'), - dict(name='ansible-29', result='SUCCESS', changes='1,1'), dict(name='ansible-5', result='SUCCESS', changes='1,1'), dict(name='ansible-6', result='SUCCESS', changes='1,1'), ], ordered=False) @@ -8038,8 +8026,6 @@ class TestDefaultAnsibleVersion(AnsibleZuulTestCase): self.assertHistory([ dict(name='ansible-default-zuul-conf', result='SUCCESS', changes='1,1'), - dict(name='ansible-28', result='SUCCESS', changes='1,1'), - dict(name='ansible-29', result='SUCCESS', changes='1,1'), dict(name='ansible-5', result='SUCCESS', changes='1,1'), dict(name='ansible-6', result='SUCCESS', changes='1,1'), ], ordered=False) diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py index f9de11a9b..d8901fabb 100644 --- a/tests/unit/test_web.py +++ b/tests/unit/test_web.py @@ -1454,6 +1454,69 @@ class TestWebMultiTenant(BaseTestWeb): sorted(["tenant-one", "tenant-two", "tenant-four"])) +class TestWebGlobalSemaphores(BaseTestWeb): + tenant_config_file = 'config/global-semaphores-config/main.yaml' + + def test_web_semaphores(self): + self.executor_server.hold_jobs_in_build = True + A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') + self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertBuilds([ + dict(name='test-global-semaphore', changes='1,1'), + dict(name='test-common-semaphore', changes='1,1'), + dict(name='test-project1-semaphore', changes='1,1'), + dict(name='test-global-semaphore', changes='2,1'), + dict(name='test-common-semaphore', changes='2,1'), + dict(name='test-project2-semaphore', changes='2,1'), + ]) + + tenant1_buildset_uuid = self.builds[0].parameters['zuul']['buildset'] + data = self.get_url('api/tenant/tenant-one/semaphores').json() + + expected = [ + {'name': 'common-semaphore', + 'global': False, + 'max': 10, + 'holders': { + 'count': 1, + 'this_tenant': [ + {'buildset_uuid': tenant1_buildset_uuid, + 'job_name': 'test-common-semaphore'} + ], + 'other_tenants': 0 + }}, + {'name': 'global-semaphore', + 'global': True, + 'max': 100, + 'holders': { + 'count': 2, + 'this_tenant': [ + {'buildset_uuid': tenant1_buildset_uuid, + 'job_name': 'test-global-semaphore'} + ], + 'other_tenants': 1 + }}, + {'name': 'project1-semaphore', + 'global': False, + 'max': 11, + 'holders': { + 'count': 1, + 'this_tenant': [ + {'buildset_uuid': tenant1_buildset_uuid, + 'job_name': 'test-project1-semaphore'} + ], + 'other_tenants': 0 + }} + ] + self.assertEqual(expected, data) + + class TestEmptyConfig(BaseTestWeb): tenant_config_file = 'config/empty-config/main.yaml' diff --git a/tests/unit/test_zk.py b/tests/unit/test_zk.py index b1f393e47..fc6a47b93 100644 --- a/tests/unit/test_zk.py +++ b/tests/unit/test_zk.py @@ -1352,7 +1352,7 @@ class TestSystemConfigCache(ZooKeeperBaseTestCase): "use_relative_priority": True, "max_hold_expiration": 7200, "default_hold_expiration": 3600, - "default_ansible_version": "2.9", + "default_ansible_version": "5", "web_root": "/web/root", "web_status_url": "/web/status", "websocket_url": "/web/socket", diff --git a/web/src/containers/build/Artifact.jsx b/web/src/containers/build/Artifact.jsx index 4d4259440..3793222d9 100644 --- a/web/src/containers/build/Artifact.jsx +++ b/web/src/containers/build/Artifact.jsx @@ -17,6 +17,7 @@ import PropTypes from 'prop-types' import { TreeView, } from 'patternfly-react' +import ReactJson from 'react-json-view' class Artifact extends React.Component { @@ -32,7 +33,17 @@ class Artifact extends React.Component { {Object.keys(artifact.metadata).map(key => ( <tr key={key}> <td>{key}</td> - <td style={{width:'100%'}}>{artifact.metadata[key]}</td> + <td style={{width:'100%'}}> + {typeof(artifact.metadata[key]) === 'object'? + <ReactJson + src={artifact.metadata[key]} + name={null} + collapsed={true} + sortKeys={true} + enableClipboard={false} + displayDataTypes={false}/> + :artifact.metadata[key].toString()} + </td> </tr> ))} </tbody> diff --git a/web/src/containers/build/Console.jsx b/web/src/containers/build/Console.jsx index f677dfbd2..8fa79a125 100644 --- a/web/src/containers/build/Console.jsx +++ b/web/src/containers/build/Console.jsx @@ -227,26 +227,6 @@ class HostTask extends React.Component { name = task.role.name + ': ' + name } - // NOTE(ianw) 2022-08-26 since we have some rows that expand and - // others that don't, the expansion button pushes things out of - // alignment. This tries to emulate the button and then - // hide it. See also: - // https://github.com/patternfly/patternfly/issues/5055 - // We might want to think about other ways to present this? - if (!interestingKeys) { - dataListCells.push( - <DataListCell key='padding-icon' isIcon={true} - class='pf-c-data-list__item-control'> - <div className='pf-c-data-list__toggle' - style={{visibility: 'hidden'}}> - <Button disabled> - <AngleRightIcon /> - </Button> - </div> - </DataListCell> - ) - } - dataListCells.push( <DataListCell key='name' width={4}>{name}</DataListCell> ) @@ -323,11 +303,23 @@ class HostTask extends React.Component { </DataListContent> </DataListItem> } else { - item = <DataListItem> - <DataListItemRow> + // We currently have to build the data-list item/row/control manually + // as we don't have a way to hide the toggle. Hopefully PF will + // add a prop that does this so we can get rid of this, see: + // https://github.com/patternfly/patternfly/issues/5055 + item = <li className="pf-c-data-list__item"> + <div className="pf-c-data-list__item-row"> + <div className="pf-c-data-list__item-control" + style={{visibility: 'hidden'}}> + <div className="pf-c-data-list__toggle"> + <Button disabled> + <AngleRightIcon /> + </Button> + </div> + </div> <DataListItemCells dataListCells={ dataListCells } /> - </DataListItemRow> - </DataListItem> + </div> + </li> } const modalDescription = <Flex> diff --git a/zuul/ansible/2.8/action/__init__.py b/zuul/ansible/2.8/action/__init__.py deleted file mode 120000 index 4048e7ac1..000000000 --- a/zuul/ansible/2.8/action/__init__.py +++ /dev/null @@ -1 +0,0 @@ -../../base/action/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/action/command.py b/zuul/ansible/2.8/action/command.py deleted file mode 120000 index 56c6b636f..000000000 --- a/zuul/ansible/2.8/action/command.py +++ /dev/null @@ -1 +0,0 @@ -../../base/action/command.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/action/command.pyi b/zuul/ansible/2.8/action/command.pyi deleted file mode 120000 index a003281ca..000000000 --- a/zuul/ansible/2.8/action/command.pyi +++ /dev/null @@ -1 +0,0 @@ -../../base/action/command.pyi
\ No newline at end of file diff --git a/zuul/ansible/2.8/action/zuul_return.py b/zuul/ansible/2.8/action/zuul_return.py deleted file mode 120000 index 83c2fc619..000000000 --- a/zuul/ansible/2.8/action/zuul_return.py +++ /dev/null @@ -1 +0,0 @@ -../../base/action/zuul_return.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/callback/__init__.py b/zuul/ansible/2.8/callback/__init__.py deleted file mode 120000 index 00b974388..000000000 --- a/zuul/ansible/2.8/callback/__init__.py +++ /dev/null @@ -1 +0,0 @@ -../../base/callback/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/callback/zuul_json.py b/zuul/ansible/2.8/callback/zuul_json.py deleted file mode 120000 index b0a07779b..000000000 --- a/zuul/ansible/2.8/callback/zuul_json.py +++ /dev/null @@ -1 +0,0 @@ -../../base/callback/zuul_json.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/callback/zuul_stream.py b/zuul/ansible/2.8/callback/zuul_stream.py deleted file mode 120000 index f75561bf4..000000000 --- a/zuul/ansible/2.8/callback/zuul_stream.py +++ /dev/null @@ -1 +0,0 @@ -../../base/callback/zuul_stream.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/callback/zuul_unreachable.py b/zuul/ansible/2.8/callback/zuul_unreachable.py deleted file mode 120000 index 205baca6f..000000000 --- a/zuul/ansible/2.8/callback/zuul_unreachable.py +++ /dev/null @@ -1 +0,0 @@ -../../base/callback/zuul_unreachable.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/filter/__init__.py b/zuul/ansible/2.8/filter/__init__.py deleted file mode 120000 index f80a4da61..000000000 --- a/zuul/ansible/2.8/filter/__init__.py +++ /dev/null @@ -1 +0,0 @@ -../../base/filter/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/filter/zuul_filters.py b/zuul/ansible/2.8/filter/zuul_filters.py deleted file mode 120000 index d406e5fe6..000000000 --- a/zuul/ansible/2.8/filter/zuul_filters.py +++ /dev/null @@ -1 +0,0 @@ -../../base/filter/zuul_filters.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/library/__init__.py b/zuul/ansible/2.8/library/__init__.py deleted file mode 120000 index 0b68ce0f4..000000000 --- a/zuul/ansible/2.8/library/__init__.py +++ /dev/null @@ -1 +0,0 @@ -../../base/library/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/library/command.py b/zuul/ansible/2.8/library/command.py deleted file mode 100755 index 5f7f31947..000000000 --- a/zuul/ansible/2.8/library/command.py +++ /dev/null @@ -1,682 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -# Copyright: (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>, and others -# Copyright: (c) 2016, Toshio Kuratomi <tkuratomi@ansible.com> -# -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - -from __future__ import absolute_import, division, print_function -__metaclass__ = type - - -ANSIBLE_METADATA = {'metadata_version': '1.1', - 'status': ['stableinterface'], - 'supported_by': 'core'} - - -# flake8: noqa -# This file shares a significant chunk of code with an upstream ansible -# function, run_command. The goal is to not have to fork quite so much -# of that function, and discussing that design with upstream means we -# should keep the changes to substantive ones only. For that reason, this -# file is purposely not enforcing pep8, as making the function pep8 clean -# would remove our ability to easily have a discussion with our friends -# upstream - -DOCUMENTATION = ''' ---- -module: command -short_description: Executes a command on a remote node -version_added: historical -description: - - The C(command) module takes the command name followed by a list of space-delimited arguments. - - The given command will be executed on all selected nodes. It will not be - processed through the shell, so variables like C($HOME) and operations - like C("<"), C(">"), C("|"), C(";") and C("&") will not work (use the M(shell) - module if you need these features). - - For Windows targets, use the M(win_command) module instead. -options: - free_form: - description: - - The command module takes a free form command to run. There is no parameter actually named 'free form'. - See the examples! - required: yes - argv: - description: - - Allows the user to provide the command as a list vs. a string. Only the string or the list form can be - provided, not both. One or the other must be provided. - version_added: "2.6" - creates: - description: - - A filename or (since 2.0) glob pattern, when it already exists, this step will B(not) be run. - removes: - description: - - A filename or (since 2.0) glob pattern, when it does not exist, this step will B(not) be run. - version_added: "0.8" - chdir: - description: - - Change into this directory before running the command. - version_added: "0.6" - warn: - description: - - If command_warnings are on in ansible.cfg, do not warn about this particular line if set to C(no). - type: bool - default: 'yes' - version_added: "1.8" - stdin: - version_added: "2.4" - description: - - Set the stdin of the command directly to the specified value. -notes: - - If you want to run a command through the shell (say you are using C(<), C(>), C(|), etc), you actually want the M(shell) module instead. - Parsing shell metacharacters can lead to unexpected commands being executed if quoting is not done correctly so it is more secure to - use the C(command) module when possible. - - " C(creates), C(removes), and C(chdir) can be specified after the command. - For instance, if you only want to run a command if a certain file does not exist, use this." - - The C(executable) parameter is removed since version 2.4. If you have a need for this parameter, use the M(shell) module instead. - - For Windows targets, use the M(win_command) module instead. -author: - - Ansible Core Team - - Michael DeHaan -''' - -EXAMPLES = ''' -- name: return motd to registered var - command: cat /etc/motd - register: mymotd - -- name: Run the command if the specified file does not exist. - command: /usr/bin/make_database.sh arg1 arg2 - args: - creates: /path/to/database - -# You can also use the 'args' form to provide the options. -- name: This command will change the working directory to somedir/ and will only run when /path/to/database doesn't exist. - command: /usr/bin/make_database.sh arg1 arg2 - args: - chdir: somedir/ - creates: /path/to/database - -- name: use argv to send the command as a list. Be sure to leave command empty - command: - args: - argv: - - echo - - testing - -- name: safely use templated variable to run command. Always use the quote filter to avoid injection issues. - command: cat {{ myfile|quote }} - register: myoutput -''' - -RETURN = ''' -cmd: - description: the cmd that was run on the remote machine - returned: always - type: list - sample: - - echo - - hello -delta: - description: cmd end time - cmd start time - returned: always - type: string - sample: 0:00:00.001529 -end: - description: cmd end time - returned: always - type: string - sample: '2017-09-29 22:03:48.084657' -start: - description: cmd start time - returned: always - type: string - sample: '2017-09-29 22:03:48.083128' -''' - -import datetime -import glob -import os -import shlex - -from ansible.module_utils.basic import AnsibleModule - -# Imports needed for Zuul things -import re -import subprocess -import traceback -import threading -from ansible.module_utils.basic import heuristic_log_sanitize -from ansible.module_utils.six import ( - PY2, - PY3, - b, - binary_type, - string_types, - text_type, -) -from ansible.module_utils.six.moves import shlex_quote -from ansible.module_utils._text import to_native, to_bytes, to_text - - -LOG_STREAM_FILE = '/tmp/console-{log_uuid}.log' -PASSWD_ARG_RE = re.compile(r'^[-]{0,2}pass[-]?(word|wd)?') -# List to save stdout log lines in as we collect them -_log_lines = [] - - -class Console(object): - def __init__(self, log_uuid): - # The streamer currently will not ask us for output from - # loops. This flag uuid was set in the action plugin if this - # call was part of a loop. This avoids us leaving behind - # files that will never be read, but also means no other - # special-casing for any of this path. - if log_uuid == 'in-loop-ignore': - self.logfile_name = os.devnull - else: - self.logfile_name = LOG_STREAM_FILE.format(log_uuid=log_uuid) - - def __enter__(self): - self.logfile = open(self.logfile_name, 'ab', buffering=0) - return self - - def __exit__(self, etype, value, tb): - self.logfile.close() - - def addLine(self, ln): - # Note this format with deliminator is "inspired" by the old - # Jenkins format but with microsecond resolution instead of - # millisecond. It is kept so log parsing/formatting remains - # consistent. - ts = str(datetime.datetime.now()).encode('utf-8') - if not isinstance(ln, bytes): - try: - ln = ln.encode('utf-8') - except Exception: - ln = repr(ln).encode('utf-8') + b'\n' - outln = b'%s | %s' % (ts, ln) - self.logfile.write(outln) - - -def follow(fd, log_uuid): - newline_warning = False - with Console(log_uuid) as console: - while True: - line = fd.readline() - if not line: - break - _log_lines.append(line) - if not line.endswith(b'\n'): - line += b'\n' - newline_warning = True - console.addLine(line) - if newline_warning: - console.addLine('[Zuul] No trailing newline\n') - - -# Taken from ansible/module_utils/basic.py ... forking the method for now -# so that we can dive in and figure out how to make appropriate hook points -def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, executable=None, data=None, binary_data=False, path_prefix=None, cwd=None, - use_unsafe_shell=False, prompt_regex=None, environ_update=None, umask=None, encoding='utf-8', errors='surrogate_or_strict'): - ''' - Execute a command, returns rc, stdout, and stderr. - - :arg args: is the command to run - * If args is a list, the command will be run with shell=False. - * If args is a string and use_unsafe_shell=False it will split args to a list and run with shell=False - * If args is a string and use_unsafe_shell=True it runs with shell=True. - :kw check_rc: Whether to call fail_json in case of non zero RC. - Default False - :kw close_fds: See documentation for subprocess.Popen(). Default True - :kw executable: See documentation for subprocess.Popen(). Default None - :kw data: If given, information to write to the stdin of the command - :kw binary_data: If False, append a newline to the data. Default False - :kw path_prefix: If given, additional path to find the command in. - This adds to the PATH environment vairable so helper commands in - the same directory can also be found - :kw cwd: If given, working directory to run the command inside - :kw use_unsafe_shell: See `args` parameter. Default False - :kw prompt_regex: Regex string (not a compiled regex) which can be - used to detect prompts in the stdout which would otherwise cause - the execution to hang (especially if no input data is specified) - :kw environ_update: dictionary to *update* os.environ with - :kw umask: Umask to be used when running the command. Default None - :kw encoding: Since we return native strings, on python3 we need to - know the encoding to use to transform from bytes to text. If you - want to always get bytes back, use encoding=None. The default is - "utf-8". This does not affect transformation of strings given as - args. - :kw errors: Since we return native strings, on python3 we need to - transform stdout and stderr from bytes to text. If the bytes are - undecodable in the ``encoding`` specified, then use this error - handler to deal with them. The default is ``surrogate_or_strict`` - which means that the bytes will be decoded using the - surrogateescape error handler if available (available on all - python3 versions we support) otherwise a UnicodeError traceback - will be raised. This does not affect transformations of strings - given as args. - :returns: A 3-tuple of return code (integer), stdout (native string), - and stderr (native string). On python2, stdout and stderr are both - byte strings. On python3, stdout and stderr are text strings converted - according to the encoding and errors parameters. If you want byte - strings on python3, use encoding=None to turn decoding to text off. - ''' - - if not isinstance(args, (list, binary_type, text_type)): - msg = "Argument 'args' to run_command must be list or string" - self.fail_json(rc=257, cmd=args, msg=msg) - - shell = False - if use_unsafe_shell: - - # stringify args for unsafe/direct shell usage - if isinstance(args, list): - args = " ".join([shlex_quote(x) for x in args]) - - # not set explicitly, check if set by controller - if executable: - args = [executable, '-c', args] - elif self._shell not in (None, '/bin/sh'): - args = [self._shell, '-c', args] - else: - shell = True - else: - # ensure args are a list - if isinstance(args, (binary_type, text_type)): - # On python2.6 and below, shlex has problems with text type - # On python3, shlex needs a text type. - if PY2: - args = to_bytes(args, errors='surrogate_or_strict') - elif PY3: - args = to_text(args, errors='surrogateescape') - args = shlex.split(args) - - # expand shellisms - args = [os.path.expanduser(os.path.expandvars(x)) for x in args if x is not None] - - prompt_re = None - if prompt_regex: - if isinstance(prompt_regex, text_type): - if PY3: - prompt_regex = to_bytes(prompt_regex, errors='surrogateescape') - elif PY2: - prompt_regex = to_bytes(prompt_regex, errors='surrogate_or_strict') - try: - prompt_re = re.compile(prompt_regex, re.MULTILINE) - except re.error: - self.fail_json(msg="invalid prompt regular expression given to run_command") - - rc = 0 - msg = None - st_in = None - - # Manipulate the environ we'll send to the new process - old_env_vals = {} - # We can set this from both an attribute and per call - for key, val in self.run_command_environ_update.items(): - old_env_vals[key] = os.environ.get(key, None) - os.environ[key] = val - if environ_update: - for key, val in environ_update.items(): - old_env_vals[key] = os.environ.get(key, None) - os.environ[key] = val - if path_prefix: - old_env_vals['PATH'] = os.environ['PATH'] - os.environ['PATH'] = "%s:%s" % (path_prefix, os.environ['PATH']) - - # If using test-module and explode, the remote lib path will resemble ... - # /tmp/test_module_scratch/debug_dir/ansible/module_utils/basic.py - # If using ansible or ansible-playbook with a remote system ... - # /tmp/ansible_vmweLQ/ansible_modlib.zip/ansible/module_utils/basic.py - - # Clean out python paths set by ansiballz - if 'PYTHONPATH' in os.environ: - pypaths = os.environ['PYTHONPATH'].split(':') - pypaths = [x for x in pypaths - if not x.endswith('/ansible_modlib.zip') and - not x.endswith('/debug_dir')] - os.environ['PYTHONPATH'] = ':'.join(pypaths) - if not os.environ['PYTHONPATH']: - del os.environ['PYTHONPATH'] - - # create a printable version of the command for use - # in reporting later, which strips out things like - # passwords from the args list - to_clean_args = args - if PY2: - if isinstance(args, text_type): - to_clean_args = to_bytes(args) - else: - if isinstance(args, binary_type): - to_clean_args = to_text(args) - if isinstance(args, (text_type, binary_type)): - to_clean_args = shlex.split(to_clean_args) - - clean_args = [] - is_passwd = False - for arg in (to_native(a) for a in to_clean_args): - if is_passwd: - is_passwd = False - clean_args.append('********') - continue - if PASSWD_ARG_RE.match(arg): - sep_idx = arg.find('=') - if sep_idx > -1: - clean_args.append('%s=********' % arg[:sep_idx]) - continue - else: - is_passwd = True - arg = heuristic_log_sanitize(arg, self.no_log_values) - clean_args.append(arg) - clean_args = ' '.join(shlex_quote(arg) for arg in clean_args) - - if data: - st_in = subprocess.PIPE - - # ZUUL: changed stderr to follow stdout - kwargs = dict( - executable=executable, - shell=shell, - close_fds=close_fds, - stdin=st_in, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - - # store the pwd - prev_dir = os.getcwd() - - # make sure we're in the right working directory - if cwd and os.path.isdir(cwd): - cwd = os.path.abspath(os.path.expanduser(cwd)) - kwargs['cwd'] = cwd - try: - os.chdir(cwd) - except (OSError, IOError) as e: - self.fail_json(rc=e.errno, msg="Could not open %s, %s" % (cwd, to_native(e)), - exception=traceback.format_exc()) - - old_umask = None - if umask: - old_umask = os.umask(umask) - - t = None - fail_json_kwargs = None - - try: - if self._debug: - self.log('Executing: ' + clean_args) - - # ZUUL: Replaced the execution loop with the zuul_runner run function - - cmd = subprocess.Popen(args, **kwargs) - if self.no_log: - t = None - else: - t = threading.Thread(target=follow, args=(cmd.stdout, zuul_log_id)) - t.daemon = True - t.start() - - # ZUUL: Our log thread will catch the output so don't do that here. - - # # the communication logic here is essentially taken from that - # # of the _communicate() function in ssh.py - # - # stdout = b('') - # stderr = b('') - # - # # ZUUL: stderr follows stdout - # rpipes = [cmd.stdout] - - if data: - if not binary_data: - data += '\n' - if isinstance(data, text_type): - data = to_bytes(data) - cmd.stdin.write(data) - cmd.stdin.close() - - # while True: - # rfds, wfds, efds = select.select(rpipes, [], rpipes, 1) - # stdout += self._read_from_pipes(rpipes, rfds, cmd.stdout) - # - # # ZUUL: stderr follows stdout - # # stderr += self._read_from_pipes(rpipes, rfds, cmd.stderr) - # - # # if we're checking for prompts, do it now - # if prompt_re: - # if prompt_re.search(stdout) and not data: - # if encoding: - # stdout = to_native(stdout, encoding=encoding, errors=errors) - # else: - # stdout = stdout - # return (257, stdout, "A prompt was encountered while running a command, but no input data was specified") - # # only break out if no pipes are left to read or - # # the pipes are completely read and - # # the process is terminated - # if (not rpipes or not rfds) and cmd.poll() is not None: - # break - # # No pipes are left to read but process is not yet terminated - # # Only then it is safe to wait for the process to be finished - # # NOTE: Actually cmd.poll() is always None here if rpipes is empty - # elif not rpipes and cmd.poll() is None: - # cmd.wait() - # # The process is terminated. Since no pipes to read from are - # # left, there is no need to call select() again. - # break - - # ZUUL: If the console log follow thread *is* stuck in readline, - # we can't close stdout (attempting to do so raises an - # exception) , so this is disabled. - # cmd.stdout.close() - # cmd.stderr.close() - - rc = cmd.wait() - - # Give the thread that is writing the console log up to 10 seconds - # to catch up and exit. If it hasn't done so by then, it is very - # likely stuck in readline() because it spawed a child that is - # holding stdout or stderr open. - if t: - t.join(10) - with Console(zuul_log_id) as console: - if t.is_alive(): - console.addLine("[Zuul] standard output/error still open " - "after child exited") - # ZUUL: stdout and stderr are in the console log file - # ZUUL: return the saved log lines so we can ship them back - stdout = b('').join(_log_lines) - else: - stdout = b('') - stderr = b('') - - except (OSError, IOError) as e: - self.log("Error Executing CMD:%s Exception:%s" % (clean_args, to_native(e))) - # ZUUL: store fail_json_kwargs and fail later in finally - fail_json_kwargs = dict(rc=e.errno, msg=to_native(e), cmd=clean_args) - except Exception as e: - self.log("Error Executing CMD:%s Exception:%s" % (clean_args, to_native(traceback.format_exc()))) - # ZUUL: store fail_json_kwargs and fail later in finally - fail_json_kwargs = dict(rc=257, msg=to_native(e), exception=traceback.format_exc(), cmd=clean_args) - finally: - with Console(zuul_log_id) as console: - if t and t.is_alive(): - console.addLine("[Zuul] standard output/error still open " - "after child exited") - if fail_json_kwargs: - # we hit an exception and need to use the rc from - # fail_json_kwargs - rc = fail_json_kwargs['rc'] - - console.addLine("[Zuul] Task exit code: %s\n" % rc) - - if fail_json_kwargs: - self.fail_json(**fail_json_kwargs) - - # Restore env settings - for key, val in old_env_vals.items(): - if val is None: - del os.environ[key] - else: - os.environ[key] = val - - if old_umask: - os.umask(old_umask) - - if rc != 0 and check_rc: - msg = heuristic_log_sanitize(stderr.rstrip(), self.no_log_values) - self.fail_json(cmd=clean_args, rc=rc, stdout=stdout, stderr=stderr, msg=msg) - - # reset the pwd - os.chdir(prev_dir) - - if encoding is not None: - return (rc, to_native(stdout, encoding=encoding, errors=errors), - to_native(stderr, encoding=encoding, errors=errors)) - return (rc, stdout, stderr) - - -def check_command(module, commandline): - arguments = {'chown': 'owner', 'chmod': 'mode', 'chgrp': 'group', - 'ln': 'state=link', 'mkdir': 'state=directory', - 'rmdir': 'state=absent', 'rm': 'state=absent', 'touch': 'state=touch'} - commands = {'curl': 'get_url or uri', 'wget': 'get_url or uri', - 'svn': 'subversion', 'service': 'service', - 'mount': 'mount', 'rpm': 'yum, dnf or zypper', 'yum': 'yum', 'apt-get': 'apt', - 'tar': 'unarchive', 'unzip': 'unarchive', 'sed': 'replace, lineinfile or template', - 'dnf': 'dnf', 'zypper': 'zypper'} - become = ['sudo', 'su', 'pbrun', 'pfexec', 'runas', 'pmrun', 'machinectl'] - if isinstance(commandline, list): - command = commandline[0] - else: - command = commandline.split()[0] - command = os.path.basename(command) - - disable_suffix = "If you need to use command because {mod} is insufficient you can add" \ - " warn=False to this command task or set command_warnings=False in" \ - " ansible.cfg to get rid of this message." - substitutions = {'mod': None, 'cmd': command} - - if command in arguments: - msg = "Consider using the {mod} module with {subcmd} rather than running {cmd}. " + disable_suffix - substitutions['mod'] = 'file' - substitutions['subcmd'] = arguments[command] - module.warn(msg.format(**substitutions)) - - if command in commands: - msg = "Consider using the {mod} module rather than running {cmd}. " + disable_suffix - substitutions['mod'] = commands[command] - module.warn(msg.format(**substitutions)) - - if command in become: - module.warn("Consider using 'become', 'become_method', and 'become_user' rather than running %s" % (command,)) - - -def main(): - - # the command module is the one ansible module that does not take key=value args - # hence don't copy this one if you are looking to build others! - module = AnsibleModule( - argument_spec=dict( - _raw_params=dict(), - _uses_shell=dict(type='bool', default=False), - argv=dict(type='list'), - chdir=dict(type='path'), - executable=dict(), - creates=dict(type='path'), - removes=dict(type='path'), - # The default for this really comes from the action plugin - warn=dict(type='bool', default=True), - stdin=dict(required=False), - zuul_log_id=dict(type='str'), - ) - ) - shell = module.params['_uses_shell'] - chdir = module.params['chdir'] - executable = module.params['executable'] - args = module.params['_raw_params'] - argv = module.params['argv'] - creates = module.params['creates'] - removes = module.params['removes'] - warn = module.params['warn'] - stdin = module.params['stdin'] - zuul_log_id = module.params['zuul_log_id'] - - if not shell and executable: - module.warn("As of Ansible 2.4, the parameter 'executable' is no longer supported with the 'command' module. Not using '%s'." % executable) - executable = None - - if not zuul_log_id: - module.fail_json(rc=256, msg="zuul_log_id missing: %s" % module.params) - - if (not args or args.strip() == '') and not argv: - module.fail_json(rc=256, msg="no command given") - - if args and argv: - module.fail_json(rc=256, msg="only command or argv can be given, not both") - - if not shell and args: - args = shlex.split(args) - - args = args or argv - - if chdir: - chdir = os.path.abspath(chdir) - os.chdir(chdir) - - if creates: - # do not run the command if the line contains creates=filename - # and the filename already exists. This allows idempotence - # of command executions. - if glob.glob(creates): - module.exit_json( - cmd=args, - stdout="skipped, since %s exists" % creates, - changed=False, - rc=0 - ) - - if removes: - # do not run the command if the line contains removes=filename - # and the filename does not exist. This allows idempotence - # of command executions. - if not glob.glob(removes): - module.exit_json( - cmd=args, - stdout="skipped, since %s does not exist" % removes, - changed=False, - rc=0 - ) - - if warn: - check_command(module, args) - - startd = datetime.datetime.now() - - rc, out, err = zuul_run_command(module, args, zuul_log_id, executable=executable, use_unsafe_shell=shell, encoding=None, data=stdin) - - endd = datetime.datetime.now() - delta = endd - startd - - result = dict( - cmd=args, - stdout=out.rstrip(b"\r\n"), - stderr=err.rstrip(b"\r\n"), - rc=rc, - start=str(startd), - end=str(endd), - delta=str(delta), - changed=True, - zuul_log_id=zuul_log_id - ) - - if rc != 0: - module.fail_json(msg='non-zero return code', **result) - - module.exit_json(**result) - - -if __name__ == '__main__': - main() diff --git a/zuul/ansible/2.8/library/zuul_console.py b/zuul/ansible/2.8/library/zuul_console.py deleted file mode 120000 index 7c905e0f9..000000000 --- a/zuul/ansible/2.8/library/zuul_console.py +++ /dev/null @@ -1 +0,0 @@ -../../base/library/zuul_console.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/logconfig.py b/zuul/ansible/2.8/logconfig.py deleted file mode 120000 index 767cb2e81..000000000 --- a/zuul/ansible/2.8/logconfig.py +++ /dev/null @@ -1 +0,0 @@ -../logconfig.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/paths.py b/zuul/ansible/2.8/paths.py deleted file mode 120000 index dbdb1858e..000000000 --- a/zuul/ansible/2.8/paths.py +++ /dev/null @@ -1 +0,0 @@ -../paths.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/action/__init__.py b/zuul/ansible/2.9/action/__init__.py deleted file mode 120000 index 4048e7ac1..000000000 --- a/zuul/ansible/2.9/action/__init__.py +++ /dev/null @@ -1 +0,0 @@ -../../base/action/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/action/command.py b/zuul/ansible/2.9/action/command.py deleted file mode 120000 index 56c6b636f..000000000 --- a/zuul/ansible/2.9/action/command.py +++ /dev/null @@ -1 +0,0 @@ -../../base/action/command.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/action/command.pyi b/zuul/ansible/2.9/action/command.pyi deleted file mode 120000 index a003281ca..000000000 --- a/zuul/ansible/2.9/action/command.pyi +++ /dev/null @@ -1 +0,0 @@ -../../base/action/command.pyi
\ No newline at end of file diff --git a/zuul/ansible/2.9/action/zuul_return.py b/zuul/ansible/2.9/action/zuul_return.py deleted file mode 120000 index 83c2fc619..000000000 --- a/zuul/ansible/2.9/action/zuul_return.py +++ /dev/null @@ -1 +0,0 @@ -../../base/action/zuul_return.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/callback/__init__.py b/zuul/ansible/2.9/callback/__init__.py deleted file mode 120000 index 00b974388..000000000 --- a/zuul/ansible/2.9/callback/__init__.py +++ /dev/null @@ -1 +0,0 @@ -../../base/callback/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/callback/zuul_json.py b/zuul/ansible/2.9/callback/zuul_json.py deleted file mode 120000 index b0a07779b..000000000 --- a/zuul/ansible/2.9/callback/zuul_json.py +++ /dev/null @@ -1 +0,0 @@ -../../base/callback/zuul_json.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/callback/zuul_stream.py b/zuul/ansible/2.9/callback/zuul_stream.py deleted file mode 120000 index f75561bf4..000000000 --- a/zuul/ansible/2.9/callback/zuul_stream.py +++ /dev/null @@ -1 +0,0 @@ -../../base/callback/zuul_stream.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/callback/zuul_unreachable.py b/zuul/ansible/2.9/callback/zuul_unreachable.py deleted file mode 120000 index 205baca6f..000000000 --- a/zuul/ansible/2.9/callback/zuul_unreachable.py +++ /dev/null @@ -1 +0,0 @@ -../../base/callback/zuul_unreachable.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/filter/__init__.py b/zuul/ansible/2.9/filter/__init__.py deleted file mode 120000 index f80a4da61..000000000 --- a/zuul/ansible/2.9/filter/__init__.py +++ /dev/null @@ -1 +0,0 @@ -../../base/filter/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/filter/zuul_filters.py b/zuul/ansible/2.9/filter/zuul_filters.py deleted file mode 120000 index d406e5fe6..000000000 --- a/zuul/ansible/2.9/filter/zuul_filters.py +++ /dev/null @@ -1 +0,0 @@ -../../base/filter/zuul_filters.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/library/__init__.py b/zuul/ansible/2.9/library/__init__.py deleted file mode 120000 index 0b68ce0f4..000000000 --- a/zuul/ansible/2.9/library/__init__.py +++ /dev/null @@ -1 +0,0 @@ -../../base/library/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/library/command.py b/zuul/ansible/2.9/library/command.py deleted file mode 100755 index 5f7f31947..000000000 --- a/zuul/ansible/2.9/library/command.py +++ /dev/null @@ -1,682 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -# Copyright: (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>, and others -# Copyright: (c) 2016, Toshio Kuratomi <tkuratomi@ansible.com> -# -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - -from __future__ import absolute_import, division, print_function -__metaclass__ = type - - -ANSIBLE_METADATA = {'metadata_version': '1.1', - 'status': ['stableinterface'], - 'supported_by': 'core'} - - -# flake8: noqa -# This file shares a significant chunk of code with an upstream ansible -# function, run_command. The goal is to not have to fork quite so much -# of that function, and discussing that design with upstream means we -# should keep the changes to substantive ones only. For that reason, this -# file is purposely not enforcing pep8, as making the function pep8 clean -# would remove our ability to easily have a discussion with our friends -# upstream - -DOCUMENTATION = ''' ---- -module: command -short_description: Executes a command on a remote node -version_added: historical -description: - - The C(command) module takes the command name followed by a list of space-delimited arguments. - - The given command will be executed on all selected nodes. It will not be - processed through the shell, so variables like C($HOME) and operations - like C("<"), C(">"), C("|"), C(";") and C("&") will not work (use the M(shell) - module if you need these features). - - For Windows targets, use the M(win_command) module instead. -options: - free_form: - description: - - The command module takes a free form command to run. There is no parameter actually named 'free form'. - See the examples! - required: yes - argv: - description: - - Allows the user to provide the command as a list vs. a string. Only the string or the list form can be - provided, not both. One or the other must be provided. - version_added: "2.6" - creates: - description: - - A filename or (since 2.0) glob pattern, when it already exists, this step will B(not) be run. - removes: - description: - - A filename or (since 2.0) glob pattern, when it does not exist, this step will B(not) be run. - version_added: "0.8" - chdir: - description: - - Change into this directory before running the command. - version_added: "0.6" - warn: - description: - - If command_warnings are on in ansible.cfg, do not warn about this particular line if set to C(no). - type: bool - default: 'yes' - version_added: "1.8" - stdin: - version_added: "2.4" - description: - - Set the stdin of the command directly to the specified value. -notes: - - If you want to run a command through the shell (say you are using C(<), C(>), C(|), etc), you actually want the M(shell) module instead. - Parsing shell metacharacters can lead to unexpected commands being executed if quoting is not done correctly so it is more secure to - use the C(command) module when possible. - - " C(creates), C(removes), and C(chdir) can be specified after the command. - For instance, if you only want to run a command if a certain file does not exist, use this." - - The C(executable) parameter is removed since version 2.4. If you have a need for this parameter, use the M(shell) module instead. - - For Windows targets, use the M(win_command) module instead. -author: - - Ansible Core Team - - Michael DeHaan -''' - -EXAMPLES = ''' -- name: return motd to registered var - command: cat /etc/motd - register: mymotd - -- name: Run the command if the specified file does not exist. - command: /usr/bin/make_database.sh arg1 arg2 - args: - creates: /path/to/database - -# You can also use the 'args' form to provide the options. -- name: This command will change the working directory to somedir/ and will only run when /path/to/database doesn't exist. - command: /usr/bin/make_database.sh arg1 arg2 - args: - chdir: somedir/ - creates: /path/to/database - -- name: use argv to send the command as a list. Be sure to leave command empty - command: - args: - argv: - - echo - - testing - -- name: safely use templated variable to run command. Always use the quote filter to avoid injection issues. - command: cat {{ myfile|quote }} - register: myoutput -''' - -RETURN = ''' -cmd: - description: the cmd that was run on the remote machine - returned: always - type: list - sample: - - echo - - hello -delta: - description: cmd end time - cmd start time - returned: always - type: string - sample: 0:00:00.001529 -end: - description: cmd end time - returned: always - type: string - sample: '2017-09-29 22:03:48.084657' -start: - description: cmd start time - returned: always - type: string - sample: '2017-09-29 22:03:48.083128' -''' - -import datetime -import glob -import os -import shlex - -from ansible.module_utils.basic import AnsibleModule - -# Imports needed for Zuul things -import re -import subprocess -import traceback -import threading -from ansible.module_utils.basic import heuristic_log_sanitize -from ansible.module_utils.six import ( - PY2, - PY3, - b, - binary_type, - string_types, - text_type, -) -from ansible.module_utils.six.moves import shlex_quote -from ansible.module_utils._text import to_native, to_bytes, to_text - - -LOG_STREAM_FILE = '/tmp/console-{log_uuid}.log' -PASSWD_ARG_RE = re.compile(r'^[-]{0,2}pass[-]?(word|wd)?') -# List to save stdout log lines in as we collect them -_log_lines = [] - - -class Console(object): - def __init__(self, log_uuid): - # The streamer currently will not ask us for output from - # loops. This flag uuid was set in the action plugin if this - # call was part of a loop. This avoids us leaving behind - # files that will never be read, but also means no other - # special-casing for any of this path. - if log_uuid == 'in-loop-ignore': - self.logfile_name = os.devnull - else: - self.logfile_name = LOG_STREAM_FILE.format(log_uuid=log_uuid) - - def __enter__(self): - self.logfile = open(self.logfile_name, 'ab', buffering=0) - return self - - def __exit__(self, etype, value, tb): - self.logfile.close() - - def addLine(self, ln): - # Note this format with deliminator is "inspired" by the old - # Jenkins format but with microsecond resolution instead of - # millisecond. It is kept so log parsing/formatting remains - # consistent. - ts = str(datetime.datetime.now()).encode('utf-8') - if not isinstance(ln, bytes): - try: - ln = ln.encode('utf-8') - except Exception: - ln = repr(ln).encode('utf-8') + b'\n' - outln = b'%s | %s' % (ts, ln) - self.logfile.write(outln) - - -def follow(fd, log_uuid): - newline_warning = False - with Console(log_uuid) as console: - while True: - line = fd.readline() - if not line: - break - _log_lines.append(line) - if not line.endswith(b'\n'): - line += b'\n' - newline_warning = True - console.addLine(line) - if newline_warning: - console.addLine('[Zuul] No trailing newline\n') - - -# Taken from ansible/module_utils/basic.py ... forking the method for now -# so that we can dive in and figure out how to make appropriate hook points -def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, executable=None, data=None, binary_data=False, path_prefix=None, cwd=None, - use_unsafe_shell=False, prompt_regex=None, environ_update=None, umask=None, encoding='utf-8', errors='surrogate_or_strict'): - ''' - Execute a command, returns rc, stdout, and stderr. - - :arg args: is the command to run - * If args is a list, the command will be run with shell=False. - * If args is a string and use_unsafe_shell=False it will split args to a list and run with shell=False - * If args is a string and use_unsafe_shell=True it runs with shell=True. - :kw check_rc: Whether to call fail_json in case of non zero RC. - Default False - :kw close_fds: See documentation for subprocess.Popen(). Default True - :kw executable: See documentation for subprocess.Popen(). Default None - :kw data: If given, information to write to the stdin of the command - :kw binary_data: If False, append a newline to the data. Default False - :kw path_prefix: If given, additional path to find the command in. - This adds to the PATH environment vairable so helper commands in - the same directory can also be found - :kw cwd: If given, working directory to run the command inside - :kw use_unsafe_shell: See `args` parameter. Default False - :kw prompt_regex: Regex string (not a compiled regex) which can be - used to detect prompts in the stdout which would otherwise cause - the execution to hang (especially if no input data is specified) - :kw environ_update: dictionary to *update* os.environ with - :kw umask: Umask to be used when running the command. Default None - :kw encoding: Since we return native strings, on python3 we need to - know the encoding to use to transform from bytes to text. If you - want to always get bytes back, use encoding=None. The default is - "utf-8". This does not affect transformation of strings given as - args. - :kw errors: Since we return native strings, on python3 we need to - transform stdout and stderr from bytes to text. If the bytes are - undecodable in the ``encoding`` specified, then use this error - handler to deal with them. The default is ``surrogate_or_strict`` - which means that the bytes will be decoded using the - surrogateescape error handler if available (available on all - python3 versions we support) otherwise a UnicodeError traceback - will be raised. This does not affect transformations of strings - given as args. - :returns: A 3-tuple of return code (integer), stdout (native string), - and stderr (native string). On python2, stdout and stderr are both - byte strings. On python3, stdout and stderr are text strings converted - according to the encoding and errors parameters. If you want byte - strings on python3, use encoding=None to turn decoding to text off. - ''' - - if not isinstance(args, (list, binary_type, text_type)): - msg = "Argument 'args' to run_command must be list or string" - self.fail_json(rc=257, cmd=args, msg=msg) - - shell = False - if use_unsafe_shell: - - # stringify args for unsafe/direct shell usage - if isinstance(args, list): - args = " ".join([shlex_quote(x) for x in args]) - - # not set explicitly, check if set by controller - if executable: - args = [executable, '-c', args] - elif self._shell not in (None, '/bin/sh'): - args = [self._shell, '-c', args] - else: - shell = True - else: - # ensure args are a list - if isinstance(args, (binary_type, text_type)): - # On python2.6 and below, shlex has problems with text type - # On python3, shlex needs a text type. - if PY2: - args = to_bytes(args, errors='surrogate_or_strict') - elif PY3: - args = to_text(args, errors='surrogateescape') - args = shlex.split(args) - - # expand shellisms - args = [os.path.expanduser(os.path.expandvars(x)) for x in args if x is not None] - - prompt_re = None - if prompt_regex: - if isinstance(prompt_regex, text_type): - if PY3: - prompt_regex = to_bytes(prompt_regex, errors='surrogateescape') - elif PY2: - prompt_regex = to_bytes(prompt_regex, errors='surrogate_or_strict') - try: - prompt_re = re.compile(prompt_regex, re.MULTILINE) - except re.error: - self.fail_json(msg="invalid prompt regular expression given to run_command") - - rc = 0 - msg = None - st_in = None - - # Manipulate the environ we'll send to the new process - old_env_vals = {} - # We can set this from both an attribute and per call - for key, val in self.run_command_environ_update.items(): - old_env_vals[key] = os.environ.get(key, None) - os.environ[key] = val - if environ_update: - for key, val in environ_update.items(): - old_env_vals[key] = os.environ.get(key, None) - os.environ[key] = val - if path_prefix: - old_env_vals['PATH'] = os.environ['PATH'] - os.environ['PATH'] = "%s:%s" % (path_prefix, os.environ['PATH']) - - # If using test-module and explode, the remote lib path will resemble ... - # /tmp/test_module_scratch/debug_dir/ansible/module_utils/basic.py - # If using ansible or ansible-playbook with a remote system ... - # /tmp/ansible_vmweLQ/ansible_modlib.zip/ansible/module_utils/basic.py - - # Clean out python paths set by ansiballz - if 'PYTHONPATH' in os.environ: - pypaths = os.environ['PYTHONPATH'].split(':') - pypaths = [x for x in pypaths - if not x.endswith('/ansible_modlib.zip') and - not x.endswith('/debug_dir')] - os.environ['PYTHONPATH'] = ':'.join(pypaths) - if not os.environ['PYTHONPATH']: - del os.environ['PYTHONPATH'] - - # create a printable version of the command for use - # in reporting later, which strips out things like - # passwords from the args list - to_clean_args = args - if PY2: - if isinstance(args, text_type): - to_clean_args = to_bytes(args) - else: - if isinstance(args, binary_type): - to_clean_args = to_text(args) - if isinstance(args, (text_type, binary_type)): - to_clean_args = shlex.split(to_clean_args) - - clean_args = [] - is_passwd = False - for arg in (to_native(a) for a in to_clean_args): - if is_passwd: - is_passwd = False - clean_args.append('********') - continue - if PASSWD_ARG_RE.match(arg): - sep_idx = arg.find('=') - if sep_idx > -1: - clean_args.append('%s=********' % arg[:sep_idx]) - continue - else: - is_passwd = True - arg = heuristic_log_sanitize(arg, self.no_log_values) - clean_args.append(arg) - clean_args = ' '.join(shlex_quote(arg) for arg in clean_args) - - if data: - st_in = subprocess.PIPE - - # ZUUL: changed stderr to follow stdout - kwargs = dict( - executable=executable, - shell=shell, - close_fds=close_fds, - stdin=st_in, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - - # store the pwd - prev_dir = os.getcwd() - - # make sure we're in the right working directory - if cwd and os.path.isdir(cwd): - cwd = os.path.abspath(os.path.expanduser(cwd)) - kwargs['cwd'] = cwd - try: - os.chdir(cwd) - except (OSError, IOError) as e: - self.fail_json(rc=e.errno, msg="Could not open %s, %s" % (cwd, to_native(e)), - exception=traceback.format_exc()) - - old_umask = None - if umask: - old_umask = os.umask(umask) - - t = None - fail_json_kwargs = None - - try: - if self._debug: - self.log('Executing: ' + clean_args) - - # ZUUL: Replaced the execution loop with the zuul_runner run function - - cmd = subprocess.Popen(args, **kwargs) - if self.no_log: - t = None - else: - t = threading.Thread(target=follow, args=(cmd.stdout, zuul_log_id)) - t.daemon = True - t.start() - - # ZUUL: Our log thread will catch the output so don't do that here. - - # # the communication logic here is essentially taken from that - # # of the _communicate() function in ssh.py - # - # stdout = b('') - # stderr = b('') - # - # # ZUUL: stderr follows stdout - # rpipes = [cmd.stdout] - - if data: - if not binary_data: - data += '\n' - if isinstance(data, text_type): - data = to_bytes(data) - cmd.stdin.write(data) - cmd.stdin.close() - - # while True: - # rfds, wfds, efds = select.select(rpipes, [], rpipes, 1) - # stdout += self._read_from_pipes(rpipes, rfds, cmd.stdout) - # - # # ZUUL: stderr follows stdout - # # stderr += self._read_from_pipes(rpipes, rfds, cmd.stderr) - # - # # if we're checking for prompts, do it now - # if prompt_re: - # if prompt_re.search(stdout) and not data: - # if encoding: - # stdout = to_native(stdout, encoding=encoding, errors=errors) - # else: - # stdout = stdout - # return (257, stdout, "A prompt was encountered while running a command, but no input data was specified") - # # only break out if no pipes are left to read or - # # the pipes are completely read and - # # the process is terminated - # if (not rpipes or not rfds) and cmd.poll() is not None: - # break - # # No pipes are left to read but process is not yet terminated - # # Only then it is safe to wait for the process to be finished - # # NOTE: Actually cmd.poll() is always None here if rpipes is empty - # elif not rpipes and cmd.poll() is None: - # cmd.wait() - # # The process is terminated. Since no pipes to read from are - # # left, there is no need to call select() again. - # break - - # ZUUL: If the console log follow thread *is* stuck in readline, - # we can't close stdout (attempting to do so raises an - # exception) , so this is disabled. - # cmd.stdout.close() - # cmd.stderr.close() - - rc = cmd.wait() - - # Give the thread that is writing the console log up to 10 seconds - # to catch up and exit. If it hasn't done so by then, it is very - # likely stuck in readline() because it spawed a child that is - # holding stdout or stderr open. - if t: - t.join(10) - with Console(zuul_log_id) as console: - if t.is_alive(): - console.addLine("[Zuul] standard output/error still open " - "after child exited") - # ZUUL: stdout and stderr are in the console log file - # ZUUL: return the saved log lines so we can ship them back - stdout = b('').join(_log_lines) - else: - stdout = b('') - stderr = b('') - - except (OSError, IOError) as e: - self.log("Error Executing CMD:%s Exception:%s" % (clean_args, to_native(e))) - # ZUUL: store fail_json_kwargs and fail later in finally - fail_json_kwargs = dict(rc=e.errno, msg=to_native(e), cmd=clean_args) - except Exception as e: - self.log("Error Executing CMD:%s Exception:%s" % (clean_args, to_native(traceback.format_exc()))) - # ZUUL: store fail_json_kwargs and fail later in finally - fail_json_kwargs = dict(rc=257, msg=to_native(e), exception=traceback.format_exc(), cmd=clean_args) - finally: - with Console(zuul_log_id) as console: - if t and t.is_alive(): - console.addLine("[Zuul] standard output/error still open " - "after child exited") - if fail_json_kwargs: - # we hit an exception and need to use the rc from - # fail_json_kwargs - rc = fail_json_kwargs['rc'] - - console.addLine("[Zuul] Task exit code: %s\n" % rc) - - if fail_json_kwargs: - self.fail_json(**fail_json_kwargs) - - # Restore env settings - for key, val in old_env_vals.items(): - if val is None: - del os.environ[key] - else: - os.environ[key] = val - - if old_umask: - os.umask(old_umask) - - if rc != 0 and check_rc: - msg = heuristic_log_sanitize(stderr.rstrip(), self.no_log_values) - self.fail_json(cmd=clean_args, rc=rc, stdout=stdout, stderr=stderr, msg=msg) - - # reset the pwd - os.chdir(prev_dir) - - if encoding is not None: - return (rc, to_native(stdout, encoding=encoding, errors=errors), - to_native(stderr, encoding=encoding, errors=errors)) - return (rc, stdout, stderr) - - -def check_command(module, commandline): - arguments = {'chown': 'owner', 'chmod': 'mode', 'chgrp': 'group', - 'ln': 'state=link', 'mkdir': 'state=directory', - 'rmdir': 'state=absent', 'rm': 'state=absent', 'touch': 'state=touch'} - commands = {'curl': 'get_url or uri', 'wget': 'get_url or uri', - 'svn': 'subversion', 'service': 'service', - 'mount': 'mount', 'rpm': 'yum, dnf or zypper', 'yum': 'yum', 'apt-get': 'apt', - 'tar': 'unarchive', 'unzip': 'unarchive', 'sed': 'replace, lineinfile or template', - 'dnf': 'dnf', 'zypper': 'zypper'} - become = ['sudo', 'su', 'pbrun', 'pfexec', 'runas', 'pmrun', 'machinectl'] - if isinstance(commandline, list): - command = commandline[0] - else: - command = commandline.split()[0] - command = os.path.basename(command) - - disable_suffix = "If you need to use command because {mod} is insufficient you can add" \ - " warn=False to this command task or set command_warnings=False in" \ - " ansible.cfg to get rid of this message." - substitutions = {'mod': None, 'cmd': command} - - if command in arguments: - msg = "Consider using the {mod} module with {subcmd} rather than running {cmd}. " + disable_suffix - substitutions['mod'] = 'file' - substitutions['subcmd'] = arguments[command] - module.warn(msg.format(**substitutions)) - - if command in commands: - msg = "Consider using the {mod} module rather than running {cmd}. " + disable_suffix - substitutions['mod'] = commands[command] - module.warn(msg.format(**substitutions)) - - if command in become: - module.warn("Consider using 'become', 'become_method', and 'become_user' rather than running %s" % (command,)) - - -def main(): - - # the command module is the one ansible module that does not take key=value args - # hence don't copy this one if you are looking to build others! - module = AnsibleModule( - argument_spec=dict( - _raw_params=dict(), - _uses_shell=dict(type='bool', default=False), - argv=dict(type='list'), - chdir=dict(type='path'), - executable=dict(), - creates=dict(type='path'), - removes=dict(type='path'), - # The default for this really comes from the action plugin - warn=dict(type='bool', default=True), - stdin=dict(required=False), - zuul_log_id=dict(type='str'), - ) - ) - shell = module.params['_uses_shell'] - chdir = module.params['chdir'] - executable = module.params['executable'] - args = module.params['_raw_params'] - argv = module.params['argv'] - creates = module.params['creates'] - removes = module.params['removes'] - warn = module.params['warn'] - stdin = module.params['stdin'] - zuul_log_id = module.params['zuul_log_id'] - - if not shell and executable: - module.warn("As of Ansible 2.4, the parameter 'executable' is no longer supported with the 'command' module. Not using '%s'." % executable) - executable = None - - if not zuul_log_id: - module.fail_json(rc=256, msg="zuul_log_id missing: %s" % module.params) - - if (not args or args.strip() == '') and not argv: - module.fail_json(rc=256, msg="no command given") - - if args and argv: - module.fail_json(rc=256, msg="only command or argv can be given, not both") - - if not shell and args: - args = shlex.split(args) - - args = args or argv - - if chdir: - chdir = os.path.abspath(chdir) - os.chdir(chdir) - - if creates: - # do not run the command if the line contains creates=filename - # and the filename already exists. This allows idempotence - # of command executions. - if glob.glob(creates): - module.exit_json( - cmd=args, - stdout="skipped, since %s exists" % creates, - changed=False, - rc=0 - ) - - if removes: - # do not run the command if the line contains removes=filename - # and the filename does not exist. This allows idempotence - # of command executions. - if not glob.glob(removes): - module.exit_json( - cmd=args, - stdout="skipped, since %s does not exist" % removes, - changed=False, - rc=0 - ) - - if warn: - check_command(module, args) - - startd = datetime.datetime.now() - - rc, out, err = zuul_run_command(module, args, zuul_log_id, executable=executable, use_unsafe_shell=shell, encoding=None, data=stdin) - - endd = datetime.datetime.now() - delta = endd - startd - - result = dict( - cmd=args, - stdout=out.rstrip(b"\r\n"), - stderr=err.rstrip(b"\r\n"), - rc=rc, - start=str(startd), - end=str(endd), - delta=str(delta), - changed=True, - zuul_log_id=zuul_log_id - ) - - if rc != 0: - module.fail_json(msg='non-zero return code', **result) - - module.exit_json(**result) - - -if __name__ == '__main__': - main() diff --git a/zuul/ansible/2.9/library/zuul_console.py b/zuul/ansible/2.9/library/zuul_console.py deleted file mode 120000 index 7c905e0f9..000000000 --- a/zuul/ansible/2.9/library/zuul_console.py +++ /dev/null @@ -1 +0,0 @@ -../../base/library/zuul_console.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/logconfig.py b/zuul/ansible/2.9/logconfig.py deleted file mode 120000 index 767cb2e81..000000000 --- a/zuul/ansible/2.9/logconfig.py +++ /dev/null @@ -1 +0,0 @@ -../logconfig.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/paths.py b/zuul/ansible/2.9/paths.py deleted file mode 120000 index dbdb1858e..000000000 --- a/zuul/ansible/2.9/paths.py +++ /dev/null @@ -1 +0,0 @@ -../paths.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/command.py b/zuul/ansible/base/action/command.py index 3bb88d8e5..a969a8b2b 100644 --- a/zuul/ansible/base/action/command.py +++ b/zuul/ansible/base/action/command.py @@ -13,8 +13,10 @@ # You should have received a copy of the GNU General Public License # along with this software. If not, see <http://www.gnu.org/licenses/>. - from zuul.ansible import paths + +from ansible.module_utils.parsing.convert_bool import boolean + command = paths._import_ansible_action_plugin("command") @@ -25,10 +27,17 @@ class ActionModule(command.ActionModule): if self._task.action in ( 'command', 'shell', 'ansible.builtin.command', 'ansible.builtin.shell'): - # This is a bit lame, but we do not log loops in the - # zuul_stream.py callback. This allows us to not write - # out command.py output to files that will never be read. - if 'ansible_loop_var' in task_vars: + # Overloading the UUID is a bit lame, but it stops us + # having to modify the library command.py too much. Both + # of these below stop the creation of the files on disk + # for situations where they won't be read and cleaned-up. + skip = boolean( + self._templar.template( + "{{zuul_console_disabled|default(false)}}")) + if skip: + self._task.args['zuul_log_id'] = 'skip' + elif 'ansible_loop_var' in task_vars: + # we do not log loops in the zuul_stream.py callback. self._task.args['zuul_log_id'] = 'in-loop-ignore' else: # Get a unique key for ZUUL_LOG_ID_MAP. ZUUL_LOG_ID_MAP diff --git a/zuul/ansible/base/library/command.py b/zuul/ansible/base/library/command.py index 3c22849ae..c52412edf 100755 --- a/zuul/ansible/base/library/command.py +++ b/zuul/ansible/base/library/command.py @@ -270,6 +270,8 @@ class Console(object): # special-casing for any of this path. if log_uuid == 'in-loop-ignore': self.logfile_name = os.devnull + elif log_uuid == 'skip': + self.logfile_name = os.devnull else: self.logfile_name = LOG_STREAM_FILE.format(log_uuid=log_uuid) @@ -494,7 +496,8 @@ def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, ex try: if self._debug: - self.log('Executing: ' + self._clean_args(args)) + self.log('Executing <%s>: %s', + zuul_log_id, self._clean_args(args)) # ZUUL: Replaced the execution loop with the zuul_runner run function diff --git a/zuul/configloader.py b/zuul/configloader.py index 037fc48aa..a92f5337c 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -1740,10 +1740,11 @@ class TenantParser(object): tenant.layout = self._parseLayout( tenant, parsed_config, loading_errors, layout_uuid) + tenant.semaphore_handler = SemaphoreHandler( + self.zk_client, self.statsd, tenant.name, tenant.layout, abide, + read_only=(not bool(self.scheduler)) + ) if self.scheduler: - tenant.semaphore_handler = SemaphoreHandler( - self.zk_client, self.statsd, tenant.name, tenant.layout, abide - ) # Only call the postConfig hook if we have a scheduler as this will # change data in ZooKeeper. In case we are in a zuul-web context, # we don't want to do that. diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py index 2213293f8..d8e4df586 100644 --- a/zuul/driver/gerrit/gerritconnection.py +++ b/zuul/driver/gerrit/gerritconnection.py @@ -589,6 +589,7 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection): _poller_class = GerritPoller _ref_watcher_class = GitWatcher ref_watcher_poll_interval = 60 + submit_retry_backoff = 10 def __init__(self, driver, connection_name, connection_config): super(GerritConnection, self).__init__(driver, connection_name, @@ -753,7 +754,8 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection): if r.status_code == 400: raise HTTPBadRequestException('Received response 400: %s' % r.text) elif r.status_code != 200: - raise Exception("Received response %s" % (r.status_code,)) + raise Exception("Received response %s: %s" % ( + r.status_code, r.text)) ret = None if r.text and len(r.text) > 4: try: @@ -1315,10 +1317,10 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection): log.exception( "Bad request submitting check data to gerrit.") break - except Exception: - log.exception("Error submitting check data to gerrit, " - "attempt %s", x) - time.sleep(x * 10) + except Exception as e: + log.exception("Error submitting check data to gerrit on " + "attempt %s: %s", x, e) + time.sleep(x * self.submit_retry_backoff) def review_http(self, item, message, submit, labels, checks_api, file_comments, phase1, phase2, @@ -1375,10 +1377,12 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection): log.exception( "Bad request submitting check data to gerrit.") break - except Exception: + except Exception as e: log.exception( - "Error submitting data to gerrit, attempt %s", x) - time.sleep(x * 10) + "Error submitting data to gerrit " + "on attempt %s: %s", + x, e) + time.sleep(x * self.submit_retry_backoff) if phase2 and change.is_current_patchset and submit: for x in range(1, 4): try: @@ -1392,10 +1396,11 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection): log.exception( "Bad request submitting check data to gerrit.") break - except Exception: + except Exception as e: log.exception( - "Error submitting data to gerrit, attempt %s", x) - time.sleep(x * 10) + "Error submitting data to gerrit on attempt %s: %s", + x, e) + time.sleep(x * self.submit_retry_backoff) def queryChangeSSH(self, number, event=None): args = '--all-approvals --comments --commit-message' diff --git a/zuul/driver/github/githubreporter.py b/zuul/driver/github/githubreporter.py index 73d3a6f13..8c5e8944c 100644 --- a/zuul/driver/github/githubreporter.py +++ b/zuul/driver/github/githubreporter.py @@ -332,7 +332,17 @@ class GithubReporter(BaseReporter): for r in change.reviews: name = r['by']['name'] email = r['by']['email'] - review_users.append('Reviewed-by: {} <{}>'.format(name, email)) + username = r['by']['username'] + review_message = 'Reviewed-by:' + if name: + review_message += ' {}'.format(name) + elif username: + review_message += ' {}'.format(username) + else: + review_message += ' Anonymous' + if email: + review_message += ' <{}>'.format(email) + review_users.append(review_message) merge_message += '\n\n' merge_message += '\n'.join(review_users) diff --git a/zuul/lib/ansible-config.conf b/zuul/lib/ansible-config.conf index 35a9f59fe..2f4ccd76b 100644 --- a/zuul/lib/ansible-config.conf +++ b/zuul/lib/ansible-config.conf @@ -4,18 +4,6 @@ default_version = 5 # OpenStackSDK 0.99.0 coincides with CORS header problems in some providers requirements = openstacksdk<0.99 openshift jmespath google-cloud-storage pywinrm boto3 azure-storage-blob ibm-cos-sdk netaddr passlib -[2.8] -# Ansible 2.8.16 breaks the k8s connection plugin -# Jinja 3.1.1 is incompatible with 2.8 -requirements = ansible>=2.8,<2.9,!=2.8.16 Jinja2<3.1.0 -deprecated = true - -[2.9] -# Ansible 2.9.14 breaks the k8s connection plugin -# https://github.com/ansible/ansible/issues/72171 -requirements = ansible>=2.9,<2.10,!=2.9.14 -deprecated = true - [5] requirements = ansible>=5.0,<6.0 diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py index cc7735728..4cadf5440 100644 --- a/zuul/manager/__init__.py +++ b/zuul/manager/__init__.py @@ -287,7 +287,7 @@ class PipelineManager(metaclass=ABCMeta): change_in_pipeline = False for item in self.pipeline.getAllItems(): - if not isinstance(change, model.Change): + if not isinstance(item.change, model.Change): continue for dep_change_ref in item.change.commit_needs_changes: if item.change.equals(change): diff --git a/zuul/model.py b/zuul/model.py index c0d5e4e0c..536e7fecc 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -664,6 +664,13 @@ class PipelineState(zkobject.ZKObject): safe_pipeline = urllib.parse.quote_plus(pipeline.name) return f"/zuul/tenant/{safe_tenant}/pipeline/{safe_pipeline}" + @classmethod + def parsePath(self, path): + """Return path components for use by the REST API""" + root, safe_tenant, pipeline, safe_pipeline = path.rsplit('/', 3) + return (urllib.parse.unquote_plus(safe_tenant), + urllib.parse.unquote_plus(safe_pipeline)) + def _dirtyPath(self): return f'{self.getPath()}/dirty' @@ -3980,6 +3987,13 @@ class BuildSet(zkobject.ZKObject): def getPath(self): return f"{self.item.getPath()}/buildset/{self.uuid}" + @classmethod + def parsePath(self, path): + """Return path components for use by the REST API""" + item_path, bs, uuid = path.rsplit('/', 2) + tenant, pipeline, item_uuid = QueueItem.parsePath(item_path) + return (tenant, pipeline, item_uuid, uuid) + def serialize(self, context): data = { # "item": self.item, @@ -4397,6 +4411,13 @@ class QueueItem(zkobject.ZKObject): def itemPath(cls, pipeline_path, item_uuid): return f"{pipeline_path}/item/{item_uuid}" + @classmethod + def parsePath(self, path): + """Return path components for use by the REST API""" + pipeline_path, item, uuid = path.rsplit('/', 2) + tenant, pipeline = PipelineState.parsePath(pipeline_path) + return (tenant, pipeline, uuid) + def serialize(self, context): if isinstance(self.event, TriggerEvent): event_type = "TriggerEvent" diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py index 644b82bec..c2b6a4ddf 100755 --- a/zuul/web/__init__.py +++ b/zuul/web/__init__.py @@ -47,6 +47,7 @@ from zuul.lib.monitoring import MonitoringServer from zuul.lib.re2util import filter_allowed_disallowed from zuul.model import ( Abide, + BuildSet, Branch, ChangeQueue, DequeueEvent, @@ -796,6 +797,7 @@ class ZuulWebAPI(object): 'project/{project:.*}/branch/{branch:.*}/' 'freeze-jobs', 'pipelines': '/api/tenant/{tenant}/pipelines', + 'semaphores': '/api/tenant/{tenant}/semaphores', 'labels': '/api/tenant/{tenant}/labels', 'nodes': '/api/tenant/{tenant}/nodes', 'key': '/api/tenant/{tenant}/key/{project:.*}.pub', @@ -1535,6 +1537,44 @@ class ZuulWebAPI(object): @cherrypy.expose @cherrypy.tools.save_params() + @cherrypy.tools.json_out( + content_type='application/json; charset=utf-8', handler=json_handler, + ) + def semaphores(self, tenant_name): + tenant = self._getTenantOrRaise(tenant_name) + result = [] + names = set(tenant.layout.semaphores.keys()) + names = names.union(tenant.global_semaphores) + for semaphore_name in sorted(names): + semaphore = tenant.layout.getSemaphore( + self.zuulweb.abide, semaphore_name) + holders = tenant.semaphore_handler.semaphoreHolders(semaphore_name) + this_tenant = [] + other_tenants = 0 + for holder in holders: + (holder_tenant, holder_pipeline, + holder_item_uuid, holder_buildset_uuid + ) = BuildSet.parsePath(holder['buildset_path']) + if holder_tenant != tenant_name: + other_tenants += 1 + continue + this_tenant.append({'buildset_uuid': holder_buildset_uuid, + 'job_name': holder['job_name']}) + sem_out = {'name': semaphore.name, + 'global': semaphore.global_scope, + 'max': semaphore.max, + 'holders': { + 'count': len(this_tenant) + other_tenants, + 'this_tenant': this_tenant, + 'other_tenants': other_tenants}, + } + result.append(sem_out) + resp = cherrypy.response + resp.headers['Access-Control-Allow-Origin'] = '*' + return result + + @cherrypy.expose + @cherrypy.tools.save_params() @cherrypy.tools.websocket(handler_cls=LogStreamHandler) def console_stream(self, tenant): cherrypy.request.ws_handler.zuulweb = self.zuulweb @@ -1864,6 +1904,8 @@ class ZuulWeb(object): controller=api, action='status') route_map.connect('api', '/api/tenant/{tenant}/status/change/{change}', controller=api, action='status_change') + route_map.connect('api', '/api/tenant/{tenant_name}/semaphores', + controller=api, action='semaphores') route_map.connect('api', '/api/tenant/{tenant_name}/jobs', controller=api, action='jobs') route_map.connect('api', '/api/tenant/{tenant_name}/job/{job_name}', diff --git a/zuul/zk/semaphore.py b/zuul/zk/semaphore.py index 721a0438a..b45612c04 100644 --- a/zuul/zk/semaphore.py +++ b/zuul/zk/semaphore.py @@ -41,8 +41,12 @@ class SemaphoreHandler(ZooKeeperSimpleBase): semaphore_root = "/zuul/semaphores" global_semaphore_root = "/zuul/global-semaphores" - def __init__(self, client, statsd, tenant_name, layout, abide): + def __init__(self, client, statsd, tenant_name, layout, abide, + read_only=False): super().__init__(client) + if read_only: + statsd = None + self.read_only = read_only self.abide = abide self.layout = layout self.statsd = statsd @@ -71,6 +75,8 @@ class SemaphoreHandler(ZooKeeperSimpleBase): self.log.exception("Unable to send semaphore stats:") def acquire(self, item, job, request_resources): + if self.read_only: + raise RuntimeError("Read-only semaphore handler") if not job.semaphores: return True @@ -190,6 +196,8 @@ class SemaphoreHandler(ZooKeeperSimpleBase): break def release(self, sched, item, job, quiet=False): + if self.read_only: + raise RuntimeError("Read-only semaphore handler") if not job.semaphores: return @@ -242,6 +250,8 @@ class SemaphoreHandler(ZooKeeperSimpleBase): return 1 if semaphore is None else semaphore.max def cleanupLeaks(self): + if self.read_only: + raise RuntimeError("Read-only semaphore handler") # MODEL_API: >1 if COMPONENT_REGISTRY.model_api < 2: self.log.warning("Skipping semaphore cleanup since minimum model " |