diff options
78 files changed, 1026 insertions, 1703 deletions
diff --git a/.zuul.yaml b/.zuul.yaml index 9d9000756..06c167e9b 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -34,21 +34,6 @@ - playbooks/zuul-stream/.* - job: - name: zuul-stream-functional-2.8 - parent: zuul-stream-functional - # Force executor to use same Ansible version as "controller" node so - # that the inventory.yaml file will be correct for that version. - ansible-version: 2.8 - vars: - zuul_ansible_version: 2.8 - -- job: - name: zuul-stream-functional-2.9 - parent: zuul-stream-functional - vars: - zuul_ansible_version: 2.9 - -- job: name: zuul-stream-functional-5 parent: zuul-stream-functional vars: @@ -323,8 +308,6 @@ files: - web/.* nodeset: ubuntu-jammy - - zuul-stream-functional-2.8 - - zuul-stream-functional-2.9 - zuul-stream-functional-5 - zuul-stream-functional-6 - zuul-tox-remote @@ -354,8 +337,6 @@ files: - web/.* nodeset: ubuntu-jammy - - zuul-stream-functional-2.8 - - zuul-stream-functional-2.9 - zuul-stream-functional-5 - zuul-stream-functional-6 - zuul-tox-remote diff --git a/doc/source/admin.rst b/doc/source/admin.rst index e9c210d3b..c7e2431f5 100644 --- a/doc/source/admin.rst +++ b/doc/source/admin.rst @@ -12,5 +12,6 @@ Service Administration operation authentication monitoring + tracing client troubleshooting diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index 7df062438..ced3ad781 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -61,6 +61,81 @@ Statsd If present, this will be prefixed to all of the keys before transmitting to the statsd server. +Tracing +~~~~~~~ + +.. attr:: tracing + + Information about the optional OpenTelemetry tracing configuration. + See :ref:`tracing` for more information. + + .. attr:: enabled + :required: + + To enable tracing, set this value to ``true``. This is the only + required parameter in order to export to a collector running + locally. + + .. attr:: protocol + :default: grpc + + The OTLP wire protocol to use. + + .. value:: grpc + + Use gRPC (the default). + + .. value:: http/protobuf + + Use HTTP with protobuf encoding. + + .. attr:: endpoint + + The endpoint to use. The default is protocol specific, but + defaults to localhost in all cases. + + .. attr:: service_name + :default: zuul + + The service name may be specified here. Multiple Zuul + installations should use different values. + + .. attr:: tls_cert + + The path to the PEM encoded certificate file. Used only by + :value:`tracing.protocol.grpc`. + + .. attr:: tls_key + + The path to the PEM encoded key file. Used only by + :value:`tracing.protocol.grpc`. + + .. attr:: tls_ca + + The path to the PEM encoded CA certificate file. Used only by + :value:`tracing.protocol.grpc`. + + .. attr:: certificate_file + + The path to the PEM encoded certificate file used to verify the + endpoint. Used only by :value:`tracing.protocol.http/protobuf`. + + .. attr:: insecure + + Whether to allow an insecure connection. Used only by + :value:`tracing.protocol.grpc`. + + .. attr:: timeout + :default: 10000 + + The timeout for outgoing data in milliseconds. + + .. attr:: compression + + The compression algorithm to use. Available values depend on + the protocol and endpoint. The only universally supported value + is ``gzip``. + ZooKeeper ~~~~~~~~~ diff --git a/doc/source/examples/docker-compose.yaml b/doc/source/examples/docker-compose.yaml index ee0093b3c..af5c970de 100644 --- a/doc/source/examples/docker-compose.yaml +++ b/doc/source/examples/docker-compose.yaml @@ -26,7 +26,7 @@ services: - "../../../tools/:/var/zuul-tools/:z" # NOTE(pabelanger): Be sure to update this line each time we change the # default version of ansible for Zuul. - command: "/usr/local/lib/zuul/ansible/2.9/bin/ansible-playbook /var/playbooks/setup.yaml" + command: "/usr/local/lib/zuul/ansible/5/bin/ansible-playbook /var/playbooks/setup.yaml" networks: - zuul zk: diff --git a/doc/source/howtos/nodepool_static.rst b/doc/source/howtos/nodepool_static.rst index c10672e7b..163497bcf 100644 --- a/doc/source/howtos/nodepool_static.rst +++ b/doc/source/howtos/nodepool_static.rst @@ -69,14 +69,30 @@ Log streaming The log streaming service enables Zuul to show the live status of long-running ``shell`` or ``command`` tasks. The server side is setup by the ``zuul_console:`` task built-in to Zuul's Ansible installation. -The executor requires the ability to communicate with the job nodes on -port 19885 for this to work. +The executor requires the ability to communicate with this server on +the job nodes via port ``19885`` for this to work. -The log streaming service may leave files on the static node in the -format ``/tmp/console-<uuid>-<task_id>-<host>.log`` if jobs are -interrupted. These may be safely removed after a short period of -inactivity with a command such as +The log streaming service spools command output via files on the job +node in the format ``/tmp/console-<uuid>-<task_id>-<host>.log``. By +default, it will clean these files up automatically. + +Occasionally, a streaming file may be left if a job is interrupted. +These may be safely removed after a short period of inactivity with a +command such as .. code-block:: shell find /tmp -maxdepth 1 -name 'console-*-*-<host>.log' -mtime +2 -delete + +If the executor is unable to reach port ``19885`` (for example due to +firewall rules), or the ``zuul_console`` daemon can not be run for +some other reason, the command to clean these spool files will not be +processed and they may be left behind; on an ephemeral node this is +not usually a problem, but on a static node these files will persist. + +In this situation, , Zuul can be instructed to not to create any spool +files for ``shell`` and ``command`` tasks via setting +``zuul_console_disable: True`` (usually via a global host variable in +inventory). Live streaming of ``shell`` and ``command`` calls will of +course be unavailable in this case, but no spool files will be +created. diff --git a/doc/source/monitoring.rst b/doc/source/monitoring.rst index 1cb61ee01..2155646f7 100644 --- a/doc/source/monitoring.rst +++ b/doc/source/monitoring.rst @@ -520,6 +520,8 @@ These metrics are emitted by the Zuul :ref:`scheduler`: requests is better. Ideally, there will be no `failed` requests. +.. stat:: zuul.nodepool + .. stat:: current_requests :type: gauge diff --git a/doc/source/tracing.rst b/doc/source/tracing.rst new file mode 100644 index 000000000..e973a77dd --- /dev/null +++ b/doc/source/tracing.rst @@ -0,0 +1,23 @@ +:title: Tracing + +.. _tracing: + +Tracing +======= + +Zuul includes support for distributed `tracing`_ as described by the +OpenTelemetry project. This allows operators (and potentially users) +to visualize the progress of events and queue items through the +various Zuul components as an aid to debugging. + +OpenTelemetry defines several observability signals such as traces, +metrics, and logs. Zuul uses other systems for metrics and logs; only +traces are exported via OpenTelemetry. + +Zuul supports the OpenTelemetry Protocol (OTLP) for exporting traces. +Many observability systems support receiving traces via OTLP +(including Jaeger tracing). + +Related configuration is in the :attr:`tracing` section of ``zuul.conf``. + +_`distributed tracing`: https://opentelemetry.io/docs/concepts/observability-primer/#distributed-traces diff --git a/playbooks/zuul-stream/fixtures/test-stream.yaml b/playbooks/zuul-stream/fixtures/test-stream.yaml index 0326ae54e..488f8cb2f 100644 --- a/playbooks/zuul-stream/fixtures/test-stream.yaml +++ b/playbooks/zuul-stream/fixtures/test-stream.yaml @@ -11,6 +11,14 @@ port: 19887 when: new_console | default(false) +- name: Run command to show skipping works + vars: + zuul_console_disabled: true + hosts: node + tasks: + - name: Run quiet command + command: echo 'This command should not stream' + - name: Run some commands to show that logging works hosts: node tasks: diff --git a/playbooks/zuul-stream/functional.yaml b/playbooks/zuul-stream/functional.yaml index 63e13e3f5..fa523f7c2 100644 --- a/playbooks/zuul-stream/functional.yaml +++ b/playbooks/zuul-stream/functional.yaml @@ -72,7 +72,7 @@ mv job-output.txt job-output-success-19885.txt mv job-output.json job-output-success-19885.json - - name: Validate outputs + - name: Validate text outputs include_tasks: validate.yaml loop: - { node: 'node1', filename: 'job-output-success-19887.txt' } @@ -82,6 +82,13 @@ # node3 only listen on 19887 - { node: 'node3', filename: 'job-output-success-19887.txt' } + # This shows that zuul_console_disabled has activated and set the + # UUID to "skip" + - name: Validate json output + shell: | + egrep 'zuul_log_id": "skip"' job-output-success-19885.json + egrep 'zuul_log_id": "skip"' job-output-success-19887.json + # failure case - name: Run ansible playbook that should fail diff --git a/releasenotes/notes/ansible-2-removed-482fa9e5b944b7d4.yaml b/releasenotes/notes/ansible-2-removed-482fa9e5b944b7d4.yaml new file mode 100644 index 000000000..26c1e59ee --- /dev/null +++ b/releasenotes/notes/ansible-2-removed-482fa9e5b944b7d4.yaml @@ -0,0 +1,6 @@ +--- +upgrade: + - | + Support for Ansible versions 2.8 and 2.9 has been removed. + Migrate any existing jobs which rely on these versions to Ansible + version 5 or 6 before upgrading. diff --git a/requirements.txt b/requirements.txt index 578f73cf7..408dcbbbd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,3 +37,6 @@ cheroot!=8.1.*,!=8.2.*,!=8.3.0 # https://github.com/cherrypy/cheroot/issues/263 elasticsearch<8.0.0 PyMySQL psycopg2-binary +opentelemetry-sdk +opentelemetry-exporter-otlp-proto-grpc +opentelemetry-exporter-otlp-proto-http diff --git a/tests/base.py b/tests/base.py index cebcf2e1f..b85bce504 100644 --- a/tests/base.py +++ b/tests/base.py @@ -125,6 +125,8 @@ from zuul.lib.logutil import get_annotated_logger import tests.fakegithub import tests.fakegitlab +from tests.otlp_fixture import OTLPFixture +import opentelemetry.sdk.trace.export FIXTURE_DIR = os.path.join(os.path.dirname(__file__), 'fixtures') @@ -1031,6 +1033,11 @@ class GerritWebServer(object): self.send_response(500) self.end_headers() + def _403(self, msg): + self.send_response(403) + self.end_headers() + self.wfile.write(msg.encode('utf8')) + def _404(self): self.send_response(404) self.end_headers() @@ -1073,6 +1080,9 @@ class GerritWebServer(object): if not change: return self._404() + if not fake_gerrit._fake_submit_permission: + return self._403('submit not permitted') + candidate = self._get_change(change_id) sr = candidate.getSubmitRecords() if sr[0]['status'] != 'OK': @@ -1306,6 +1316,8 @@ class FakeGerritConnection(gerritconnection.GerritConnection): self._poller_event = poller_event self._ref_watcher_event = ref_watcher_event self._fake_submit_whole_topic = False + self._fake_submit_permission = True + self.submit_retry_backoff = 0 def onStop(self): super().onStop() @@ -4911,6 +4923,15 @@ class ZuulTestCase(BaseTestCase): if 'database' in config.sections(): _setup_fixture(config, 'database') + if 'tracing' in config.sections(): + self.otlp = OTLPFixture() + self.useFixture(self.otlp) + self.useFixture(fixtures.MonkeyPatch( + 'zuul.lib.tracing.Tracing.processor_class', + opentelemetry.sdk.trace.export.SimpleSpanProcessor)) + config.set('tracing', 'endpoint', + f'http://localhost:{self.otlp.port}') + if not self.setupSimpleLayout(config): tenant_config = None for cfg_attr in ('tenant_config', 'tenant_config_script'): @@ -5197,6 +5218,7 @@ class ZuulTestCase(BaseTestCase): and not t.name.startswith('Dummy-') and not t.name.startswith('pydevd.') and not t.name.startswith('ptvsd.') + and not t.name.startswith('OTLPFixture_') ] if len(threads) > 1: thread_map = dict(map(lambda x: (x.ident, x.name), diff --git a/tests/fixtures/config/ansible-callbacks/main28.yaml b/tests/fixtures/config/ansible-callbacks/main28.yaml deleted file mode 100644 index 371710b4f..000000000 --- a/tests/fixtures/config/ansible-callbacks/main28.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- tenant: - name: tenant-one - default-ansible-version: '2.8' - source: - gerrit: - config-projects: - - common-config diff --git a/tests/fixtures/config/ansible-callbacks/main29.yaml b/tests/fixtures/config/ansible-callbacks/main29.yaml deleted file mode 100644 index b127139a9..000000000 --- a/tests/fixtures/config/ansible-callbacks/main29.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- tenant: - name: tenant-one - default-ansible-version: '2.9' - source: - gerrit: - config-projects: - - common-config diff --git a/tests/fixtures/config/ansible-versions/git/common-config/zuul.yaml b/tests/fixtures/config/ansible-versions/git/common-config/zuul.yaml index e1bac5e01..497053c90 100644 --- a/tests/fixtures/config/ansible-versions/git/common-config/zuul.yaml +++ b/tests/fixtures/config/ansible-versions/git/common-config/zuul.yaml @@ -34,23 +34,7 @@ parent: ansible-version vars: test_ansible_version_major: 2 - test_ansible_version_minor: 8 - -- job: - name: ansible-28 - parent: ansible-version - ansible-version: 2.8 - vars: - test_ansible_version_major: 2 - test_ansible_version_minor: 8 - -- job: - name: ansible-29 - parent: ansible-version - ansible-version: 2.9 - vars: - test_ansible_version_major: 2 - test_ansible_version_minor: 9 + test_ansible_version_minor: 13 - job: name: ansible-5 @@ -73,8 +57,6 @@ check: jobs: - ansible-default - - ansible-28 - - ansible-29 - ansible-5 - ansible-6 @@ -83,7 +65,5 @@ check: jobs: - ansible-default-zuul-conf - - ansible-28 - - ansible-29 - ansible-5 - ansible-6 diff --git a/tests/fixtures/config/ansible/main28.yaml b/tests/fixtures/config/ansible/main28.yaml deleted file mode 100644 index f2add49c7..000000000 --- a/tests/fixtures/config/ansible/main28.yaml +++ /dev/null @@ -1,11 +0,0 @@ -- tenant: - name: tenant-one - default-ansible-version: '2.8' - source: - gerrit: - config-projects: - - common-config - untrusted-projects: - - org/project - - bare-role - - org/ansible diff --git a/tests/fixtures/config/ansible/main29.yaml b/tests/fixtures/config/ansible/main29.yaml deleted file mode 100644 index 758292950..000000000 --- a/tests/fixtures/config/ansible/main29.yaml +++ /dev/null @@ -1,11 +0,0 @@ -- tenant: - name: tenant-one - default-ansible-version: '2.9' - source: - gerrit: - config-projects: - - common-config - untrusted-projects: - - org/project - - bare-role - - org/ansible diff --git a/tests/fixtures/config/executor-facts/main28.yaml b/tests/fixtures/config/executor-facts/main28.yaml deleted file mode 100644 index 686899bf8..000000000 --- a/tests/fixtures/config/executor-facts/main28.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- tenant: - name: tenant-one - default-ansible-version: '2.8' - source: - gerrit: - config-projects: - - common-config - untrusted-projects: - - org/project diff --git a/tests/fixtures/config/executor-facts/main29.yaml b/tests/fixtures/config/executor-facts/main29.yaml deleted file mode 100644 index df934ff22..000000000 --- a/tests/fixtures/config/executor-facts/main29.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- tenant: - name: tenant-one - default-ansible-version: '2.9' - source: - gerrit: - config-projects: - - common-config - untrusted-projects: - - org/project diff --git a/tests/fixtures/config/inventory/git/common-config/zuul.yaml b/tests/fixtures/config/inventory/git/common-config/zuul.yaml index ca687139d..97ee54fb9 100644 --- a/tests/fixtures/config/inventory/git/common-config/zuul.yaml +++ b/tests/fixtures/config/inventory/git/common-config/zuul.yaml @@ -115,24 +115,6 @@ run: playbooks/network.yaml - job: - name: ansible-version28-inventory - nodeset: - nodes: - - name: ubuntu-xenial - label: ubuntu-xenial - ansible-version: '2.8' - run: playbooks/ansible-version.yaml - -- job: - name: ansible-version29-inventory - nodeset: - nodes: - - name: ubuntu-xenial - label: ubuntu-xenial - ansible-version: '2.9' - run: playbooks/ansible-version.yaml - -- job: name: ansible-version5-inventory nodeset: nodes: diff --git a/tests/fixtures/config/inventory/git/org_project/.zuul.yaml b/tests/fixtures/config/inventory/git/org_project/.zuul.yaml index 69ec3127e..78f65ea83 100644 --- a/tests/fixtures/config/inventory/git/org_project/.zuul.yaml +++ b/tests/fixtures/config/inventory/git/org_project/.zuul.yaml @@ -7,7 +7,5 @@ - executor-only-inventory - group-inventory - hostvars-inventory - - ansible-version28-inventory - - ansible-version29-inventory - ansible-version5-inventory - ansible-version6-inventory diff --git a/tests/fixtures/config/inventory/git/org_project3/.zuul.yaml b/tests/fixtures/config/inventory/git/org_project3/.zuul.yaml index 6cf9faf88..0f11335da 100644 --- a/tests/fixtures/config/inventory/git/org_project3/.zuul.yaml +++ b/tests/fixtures/config/inventory/git/org_project3/.zuul.yaml @@ -2,8 +2,3 @@ check: jobs: - single-inventory - - single-inventory-list - - executor-only-inventory - - group-inventory - - hostvars-inventory - - ansible-version28-inventory diff --git a/tests/fixtures/zuul-default-ansible-version.conf b/tests/fixtures/zuul-default-ansible-version.conf index 6ec8e38df..000f27723 100644 --- a/tests/fixtures/zuul-default-ansible-version.conf +++ b/tests/fixtures/zuul-default-ansible-version.conf @@ -6,7 +6,8 @@ server=127.0.0.1 [scheduler] tenant_config=main.yaml relative_priority=true -default_ansible_version=2.8 +# Used by ansible-default-zuul-conf job +default_ansible_version=6 [merger] git_dir=/tmp/zuul-test/merger-git diff --git a/tests/fixtures/zuul-tracing.conf b/tests/fixtures/zuul-tracing.conf new file mode 100644 index 000000000..e90922198 --- /dev/null +++ b/tests/fixtures/zuul-tracing.conf @@ -0,0 +1,42 @@ +[statsd] +# note, use 127.0.0.1 rather than localhost to avoid getting ipv6 +# see: https://github.com/jsocol/pystatsd/issues/61 +server=127.0.0.1 + +[scheduler] +tenant_config=main.yaml +relative_priority=true + +[merger] +git_dir=/tmp/zuul-test/merger-git +git_user_email=zuul@example.com +git_user_name=zuul + +[executor] +git_dir=/tmp/zuul-test/executor-git +load_multiplier=100 + +[connection gerrit] +driver=gerrit +server=review.example.com +user=jenkins +sshkey=fake_id_rsa_path + +[connection smtp] +driver=smtp +server=localhost +port=25 +default_from=zuul@example.com +default_to=you@example.com + +[database] +dburi=$MYSQL_FIXTURE_DBURI$ + +[web] +static_cache_expiry=1200 +root=https://zuul.example.com/ + +[tracing] +enabled=true +endpoint=http://localhost:port +service_name=zuultest
\ No newline at end of file diff --git a/tests/otlp_fixture.py b/tests/otlp_fixture.py new file mode 100644 index 000000000..633296fac --- /dev/null +++ b/tests/otlp_fixture.py @@ -0,0 +1,58 @@ +# Copyright 2022 Acme Gating, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from concurrent import futures + +import fixtures +import grpc +from opentelemetry import trace +from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import ( + TraceServiceServicer, + add_TraceServiceServicer_to_server +) +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceResponse, +) + + +class TraceServer(TraceServiceServicer): + def __init__(self, fixture): + super().__init__() + self.fixture = fixture + + def Export(self, request, context): + self.fixture.requests.append(request) + return ExportTraceServiceResponse() + + +class OTLPFixture(fixtures.Fixture): + def __init__(self): + super().__init__() + self.requests = [] + self.executor = futures.ThreadPoolExecutor( + thread_name_prefix='OTLPFixture', + max_workers=10) + self.server = grpc.server(self.executor) + add_TraceServiceServicer_to_server(TraceServer(self), self.server) + self.port = self.server.add_insecure_port('[::]:0') + # Reset global tracer provider + trace._TRACER_PROVIDER_SET_ONCE = trace.Once() + trace._TRACER_PROVIDER = None + + def _setUp(self): + self.server.start() + + def _cleanup(self): + self.server.stop() + self.server.wait_for_termination() + self.executor.shutdown() diff --git a/tests/remote/test_remote_action_modules.py b/tests/remote/test_remote_action_modules.py index 30e430b74..923b877c2 100644 --- a/tests/remote/test_remote_action_modules.py +++ b/tests/remote/test_remote_action_modules.py @@ -21,7 +21,7 @@ from tests.base import AnsibleZuulTestCase class FunctionalActionModulesMixIn: tenant_config_file = 'config/remote-action-modules/main.yaml' # This should be overriden in child classes. - ansible_version = '2.9' + ansible_version = '5' wait_timeout = 120 def _setUp(self): @@ -87,22 +87,6 @@ class FunctionalActionModulesMixIn: self._run_job('shell-good', 'SUCCESS') -class TestActionModules28(AnsibleZuulTestCase, FunctionalActionModulesMixIn): - ansible_version = '2.8' - - def setUp(self): - super().setUp() - self._setUp() - - -class TestActionModules29(AnsibleZuulTestCase, FunctionalActionModulesMixIn): - ansible_version = '2.9' - - def setUp(self): - super().setUp() - self._setUp() - - class TestActionModules5(AnsibleZuulTestCase, FunctionalActionModulesMixIn): ansible_version = '5' diff --git a/tests/remote/test_remote_zuul_json.py b/tests/remote/test_remote_zuul_json.py index e4510e7d1..4451c3630 100644 --- a/tests/remote/test_remote_zuul_json.py +++ b/tests/remote/test_remote_zuul_json.py @@ -144,22 +144,6 @@ class FunctionalZuulJSONMixIn: dateutil.parser.parse(play_end_time) -class TestZuulJSON28(AnsibleZuulTestCase, FunctionalZuulJSONMixIn): - ansible_version = '2.8' - - def setUp(self): - super().setUp() - self._setUp() - - -class TestZuulJSON29(AnsibleZuulTestCase, FunctionalZuulJSONMixIn): - ansible_version = '2.9' - - def setUp(self): - super().setUp() - self._setUp() - - class TestZuulJSON5(AnsibleZuulTestCase, FunctionalZuulJSONMixIn): ansible_version = '5' diff --git a/tests/remote/test_remote_zuul_stream.py b/tests/remote/test_remote_zuul_stream.py index b84c4b0d8..7a7fef08d 100644 --- a/tests/remote/test_remote_zuul_stream.py +++ b/tests/remote/test_remote_zuul_stream.py @@ -25,7 +25,7 @@ from tests.base import AnsibleZuulTestCase class FunctionalZuulStreamMixIn: tenant_config_file = 'config/remote-zuul-stream/main.yaml' # This should be overriden in child classes. - ansible_version = '2.9' + ansible_version = '5' def _setUp(self): self.log_console_port = 19000 + int( @@ -201,8 +201,6 @@ class FunctionalZuulStreamMixIn: self.assertLogLine( r'controller \| ok: Runtime: \d:\d\d:\d\d\.\d\d\d\d\d\d', text) self.assertLogLine('PLAY RECAP', text) - # NOTE(pabelanger): Ansible 2.8 added new stats - # skipped, rescued, ignored. self.assertLogLine( r'controller \| ok: \d+ changed: \d+ unreachable: 0 failed: 0 ' 'skipped: 0 rescued: 1 ignored: 0', text) @@ -255,24 +253,6 @@ class FunctionalZuulStreamMixIn: self.assertLogLine(regex, text) -class TestZuulStream28(AnsibleZuulTestCase, FunctionalZuulStreamMixIn): - ansible_version = '2.8' - ansible_core_version = '2.8' - - def setUp(self): - super().setUp() - self._setUp() - - -class TestZuulStream29(AnsibleZuulTestCase, FunctionalZuulStreamMixIn): - ansible_version = '2.9' - ansible_core_version = '2.9' - - def setUp(self): - super().setUp() - self._setUp() - - class TestZuulStream5(AnsibleZuulTestCase, FunctionalZuulStreamMixIn): ansible_version = '5' ansible_core_version = '2.12' diff --git a/tests/unit/test_executor.py b/tests/unit/test_executor.py index 27ca75531..31e3b74c5 100644 --- a/tests/unit/test_executor.py +++ b/tests/unit/test_executor.py @@ -870,16 +870,6 @@ class ExecutorFactsMixin: output) -class TestExecutorFacts28(AnsibleZuulTestCase, ExecutorFactsMixin): - tenant_config_file = 'config/executor-facts/main28.yaml' - ansible_major_minor = '2.8' - - -class TestExecutorFacts29(AnsibleZuulTestCase, ExecutorFactsMixin): - tenant_config_file = 'config/executor-facts/main29.yaml' - ansible_major_minor = '2.9' - - class TestExecutorFacts5(AnsibleZuulTestCase, ExecutorFactsMixin): tenant_config_file = 'config/executor-facts/main5.yaml' ansible_major_minor = '2.12' @@ -941,20 +931,6 @@ class AnsibleCallbackConfigsMixin: output) -class TestAnsibleCallbackConfigs28(AnsibleZuulTestCase, - AnsibleCallbackConfigsMixin): - config_file = 'zuul-executor-ansible-callback.conf' - tenant_config_file = 'config/ansible-callbacks/main28.yaml' - ansible_major_minor = '2.8' - - -class TestAnsibleCallbackConfigs29(AnsibleZuulTestCase, - AnsibleCallbackConfigsMixin): - config_file = 'zuul-executor-ansible-callback.conf' - tenant_config_file = 'config/ansible-callbacks/main29.yaml' - ansible_major_minor = '2.9' - - class TestAnsibleCallbackConfigs5(AnsibleZuulTestCase, AnsibleCallbackConfigsMixin): config_file = 'zuul-executor-ansible-callback.conf' diff --git a/tests/unit/test_gerrit.py b/tests/unit/test_gerrit.py index 47545b9be..e98cb80b5 100644 --- a/tests/unit/test_gerrit.py +++ b/tests/unit/test_gerrit.py @@ -359,6 +359,25 @@ class TestGerritWeb(ZuulTestCase): changes="1,1 2,1 3,1"), ], ordered=False) + def test_submit_failure(self): + # Test that we log the reason for a submit failure (403 error) + self.fake_gerrit._fake_submit_permission = False + A = self.fake_gerrit.addFakeChange('org/project1', "master", "A") + A.addApproval('Code-Review', 2) + with self.assertLogs('zuul.test.FakeGerritConnection', level='INFO' + ) as full_logs: + self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) + self.waitUntilSettled() + self.log.debug("Full logs:") + for x in full_logs.output: + self.log.debug(x) + self.assertRegexInList( + r'Error submitting data to gerrit on attempt 3: ' + 'Received response 403: submit not permitted', + full_logs.output) + + self.assertEqual(A.data['status'], 'NEW') + class TestFileComments(AnsibleZuulTestCase): config_file = 'zuul-gerrit-web.conf' diff --git a/tests/unit/test_inventory.py b/tests/unit/test_inventory.py index 83a62a0e7..acfe1a3ae 100644 --- a/tests/unit/test_inventory.py +++ b/tests/unit/test_inventory.py @@ -180,56 +180,6 @@ class TestInventoryShellType(TestInventoryBase): class TestInventoryAutoPython(TestInventoryBase): - def test_auto_python_ansible28_inventory(self): - inventory = self._get_build_inventory('ansible-version28-inventory') - - all_nodes = ('ubuntu-xenial',) - self.assertIn('all', inventory) - self.assertIn('hosts', inventory['all']) - self.assertIn('vars', inventory['all']) - for node_name in all_nodes: - self.assertIn(node_name, inventory['all']['hosts']) - node_vars = inventory['all']['hosts'][node_name] - self.assertEqual( - 'auto', node_vars['ansible_python_interpreter']) - - self.assertIn('zuul', inventory['all']['vars']) - z_vars = inventory['all']['vars']['zuul'] - self.assertIn('executor', z_vars) - self.assertIn('src_root', z_vars['executor']) - self.assertIn('job', z_vars) - self.assertEqual(z_vars['ansible_version'], '2.8') - self.assertEqual(z_vars['job'], 'ansible-version28-inventory') - self.assertEqual(z_vars['message'], 'QQ==') - - self.executor_server.release() - self.waitUntilSettled() - - def test_auto_python_ansible29_inventory(self): - inventory = self._get_build_inventory('ansible-version29-inventory') - - all_nodes = ('ubuntu-xenial',) - self.assertIn('all', inventory) - self.assertIn('hosts', inventory['all']) - self.assertIn('vars', inventory['all']) - for node_name in all_nodes: - self.assertIn(node_name, inventory['all']['hosts']) - node_vars = inventory['all']['hosts'][node_name] - self.assertEqual( - 'auto', node_vars['ansible_python_interpreter']) - - self.assertIn('zuul', inventory['all']['vars']) - z_vars = inventory['all']['vars']['zuul'] - self.assertIn('executor', z_vars) - self.assertIn('src_root', z_vars['executor']) - self.assertIn('job', z_vars) - self.assertEqual(z_vars['ansible_version'], '2.9') - self.assertEqual(z_vars['job'], 'ansible-version29-inventory') - self.assertEqual(z_vars['message'], 'QQ==') - - self.executor_server.release() - self.waitUntilSettled() - def test_auto_python_ansible5_inventory(self): inventory = self._get_build_inventory('ansible-version5-inventory') diff --git a/tests/unit/test_tracing.py b/tests/unit/test_tracing.py new file mode 100644 index 000000000..3c452dd53 --- /dev/null +++ b/tests/unit/test_tracing.py @@ -0,0 +1,199 @@ +# Copyright 2022 Acme Gating, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import time + +from tests.base import iterate_timeout, ZuulTestCase + +import zuul.lib.tracing as tracing +from opentelemetry import trace + + +def attributes_to_dict(attrlist): + ret = {} + for attr in attrlist: + ret[attr.key] = attr.value.string_value + return ret + + +class TestTracing(ZuulTestCase): + config_file = 'zuul-tracing.conf' + tenant_config_file = "config/single-tenant/main.yaml" + + def test_tracing_api(self): + tracer = trace.get_tracer("zuul") + + # We have a lot of timestamps stored as floats, so make sure + # our root span is a ZuulSpan that can handle that input. + span_info = tracing.startSavedSpan('parent-trace', + start_time=time.time(), + attributes={'startattr': 'bar'}, + include_attributes=True) + + # Simulate a reconstructed root span + span = tracing.restoreSpan(span_info) + + # Within the root span, use the more typical OpenTelemetry + # context manager api. + with trace.use_span(span): + with tracer.start_span('child1-trace') as child1_span: + link = trace.Link(child1_span.context, + attributes={'relationship': 'prev'}) + + # Make sure that we can manually start and stop a child span, + # and that it is a ZuulSpan as well. + with trace.use_span(span): + child = tracer.start_span('child2-trace', start_time=time.time(), + links=[link]) + child.end(end_time=time.time()) + + # Make sure that we can start a child span from a span + # context and not a full span: + span_context = tracing.getSpanContext(span) + with tracing.startSpanInContext(span_context, 'child3-trace') as child: + child.end(end_time=time.time()) + + # End our root span manually. + tracing.endSavedSpan(span_info, end_time=time.time(), + attributes={'endattr': 'baz'}) + + for _ in iterate_timeout(60, "request to arrive"): + if len(self.otlp.requests) == 4: + break + req1 = self.otlp.requests[0] + self.log.debug("Received:\n%s", req1) + attrs = attributes_to_dict(req1.resource_spans[0].resource.attributes) + self.assertEqual({"service.name": "zuultest"}, attrs) + self.assertEqual("zuul", + req1.resource_spans[0].scope_spans[0].scope.name) + span1 = req1.resource_spans[0].scope_spans[0].spans[0] + self.assertEqual("child1-trace", span1.name) + + req2 = self.otlp.requests[1] + self.log.debug("Received:\n%s", req2) + span2 = req2.resource_spans[0].scope_spans[0].spans[0] + self.assertEqual("child2-trace", span2.name) + self.assertEqual(span2.links[0].span_id, span1.span_id) + attrs = attributes_to_dict(span2.links[0].attributes) + self.assertEqual({"relationship": "prev"}, attrs) + + req3 = self.otlp.requests[2] + self.log.debug("Received:\n%s", req3) + span3 = req3.resource_spans[0].scope_spans[0].spans[0] + self.assertEqual("child3-trace", span3.name) + + req4 = self.otlp.requests[3] + self.log.debug("Received:\n%s", req4) + span4 = req4.resource_spans[0].scope_spans[0].spans[0] + self.assertEqual("parent-trace", span4.name) + attrs = attributes_to_dict(span4.attributes) + self.assertEqual({"startattr": "bar", + "endattr": "baz"}, attrs) + + self.assertEqual(span1.trace_id, span4.trace_id) + self.assertEqual(span2.trace_id, span4.trace_id) + self.assertEqual(span3.trace_id, span4.trace_id) + + def test_tracing_api_null(self): + tracer = trace.get_tracer("zuul") + + # Test that restoring spans and span contexts works with + # null values. + + span_info = None + # Simulate a reconstructed root span from a null value + span = tracing.restoreSpan(span_info) + + # Within the root span, use the more typical OpenTelemetry + # context manager api. + with trace.use_span(span): + with tracer.start_span('child1-trace') as child1_span: + link = trace.Link(child1_span.context, + attributes={'relationship': 'prev'}) + + # Make sure that we can manually start and stop a child span, + # and that it is a ZuulSpan as well. + with trace.use_span(span): + child = tracer.start_span('child2-trace', start_time=time.time(), + links=[link]) + child.end(end_time=time.time()) + + # Make sure that we can start a child span from a null span + # context: + span_context = None + with tracing.startSpanInContext(span_context, 'child3-trace') as child: + child.end(end_time=time.time()) + + # End our root span manually. + span.end(end_time=time.time()) + + for _ in iterate_timeout(60, "request to arrive"): + if len(self.otlp.requests) == 3: + break + req1 = self.otlp.requests[0] + self.log.debug("Received:\n%s", req1) + attrs = attributes_to_dict(req1.resource_spans[0].resource.attributes) + self.assertEqual({"service.name": "zuultest"}, attrs) + self.assertEqual("zuul", + req1.resource_spans[0].scope_spans[0].scope.name) + span1 = req1.resource_spans[0].scope_spans[0].spans[0] + self.assertEqual("child1-trace", span1.name) + + req2 = self.otlp.requests[1] + self.log.debug("Received:\n%s", req2) + span2 = req2.resource_spans[0].scope_spans[0].spans[0] + self.assertEqual("child2-trace", span2.name) + self.assertEqual(span2.links[0].span_id, span1.span_id) + attrs = attributes_to_dict(span2.links[0].attributes) + self.assertEqual({"relationship": "prev"}, attrs) + + req3 = self.otlp.requests[2] + self.log.debug("Received:\n%s", req3) + span3 = req3.resource_spans[0].scope_spans[0].spans[0] + self.assertEqual("child3-trace", span3.name) + + self.assertNotEqual(span1.trace_id, span2.trace_id) + self.assertNotEqual(span2.trace_id, span3.trace_id) + self.assertNotEqual(span1.trace_id, span3.trace_id) + + def test_tracing(self): + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + A.addApproval('Code-Review', 2) + self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) + self.waitUntilSettled() + + for _ in iterate_timeout(60, "request to arrive"): + if len(self.otlp.requests) >= 2: + break + + buildset = self.getSpan('BuildSet') + self.log.debug("Received:\n%s", buildset) + item = self.getSpan('QueueItem') + self.log.debug("Received:\n%s", item) + self.assertEqual(item.trace_id, buildset.trace_id) + self.assertNotEqual(item.span_id, buildset.span_id) + self.assertTrue(buildset.start_time_unix_nano >= + item.start_time_unix_nano) + self.assertTrue(buildset.end_time_unix_nano <= + item.end_time_unix_nano) + item_attrs = attributes_to_dict(item.attributes) + self.assertTrue(item_attrs['ref_number'] == "1") + self.assertTrue(item_attrs['ref_patchset'] == "1") + self.assertTrue('zuul_event_id' in item_attrs) + + def getSpan(self, name): + for req in self.otlp.requests: + span = req.resource_spans[0].scope_spans[0].spans[0] + if span.name == name: + return span diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py index 3d41d74c6..fe5b46e0c 100644 --- a/tests/unit/test_v3.py +++ b/tests/unit/test_v3.py @@ -3906,16 +3906,6 @@ class FunctionalAnsibleMixIn(object): output) -class TestAnsible28(AnsibleZuulTestCase, FunctionalAnsibleMixIn): - tenant_config_file = 'config/ansible/main28.yaml' - ansible_major_minor = '2.8' - - -class TestAnsible29(AnsibleZuulTestCase, FunctionalAnsibleMixIn): - tenant_config_file = 'config/ansible/main29.yaml' - ansible_major_minor = '2.9' - - class TestAnsible5(AnsibleZuulTestCase, FunctionalAnsibleMixIn): tenant_config_file = 'config/ansible/main5.yaml' ansible_major_minor = '2.12' @@ -8016,8 +8006,6 @@ class TestAnsibleVersion(AnsibleZuulTestCase): self.assertHistory([ dict(name='ansible-default', result='SUCCESS', changes='1,1'), - dict(name='ansible-28', result='SUCCESS', changes='1,1'), - dict(name='ansible-29', result='SUCCESS', changes='1,1'), dict(name='ansible-5', result='SUCCESS', changes='1,1'), dict(name='ansible-6', result='SUCCESS', changes='1,1'), ], ordered=False) @@ -8038,8 +8026,6 @@ class TestDefaultAnsibleVersion(AnsibleZuulTestCase): self.assertHistory([ dict(name='ansible-default-zuul-conf', result='SUCCESS', changes='1,1'), - dict(name='ansible-28', result='SUCCESS', changes='1,1'), - dict(name='ansible-29', result='SUCCESS', changes='1,1'), dict(name='ansible-5', result='SUCCESS', changes='1,1'), dict(name='ansible-6', result='SUCCESS', changes='1,1'), ], ordered=False) diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py index f9de11a9b..d8901fabb 100644 --- a/tests/unit/test_web.py +++ b/tests/unit/test_web.py @@ -1454,6 +1454,69 @@ class TestWebMultiTenant(BaseTestWeb): sorted(["tenant-one", "tenant-two", "tenant-four"])) +class TestWebGlobalSemaphores(BaseTestWeb): + tenant_config_file = 'config/global-semaphores-config/main.yaml' + + def test_web_semaphores(self): + self.executor_server.hold_jobs_in_build = True + A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') + self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertBuilds([ + dict(name='test-global-semaphore', changes='1,1'), + dict(name='test-common-semaphore', changes='1,1'), + dict(name='test-project1-semaphore', changes='1,1'), + dict(name='test-global-semaphore', changes='2,1'), + dict(name='test-common-semaphore', changes='2,1'), + dict(name='test-project2-semaphore', changes='2,1'), + ]) + + tenant1_buildset_uuid = self.builds[0].parameters['zuul']['buildset'] + data = self.get_url('api/tenant/tenant-one/semaphores').json() + + expected = [ + {'name': 'common-semaphore', + 'global': False, + 'max': 10, + 'holders': { + 'count': 1, + 'this_tenant': [ + {'buildset_uuid': tenant1_buildset_uuid, + 'job_name': 'test-common-semaphore'} + ], + 'other_tenants': 0 + }}, + {'name': 'global-semaphore', + 'global': True, + 'max': 100, + 'holders': { + 'count': 2, + 'this_tenant': [ + {'buildset_uuid': tenant1_buildset_uuid, + 'job_name': 'test-global-semaphore'} + ], + 'other_tenants': 1 + }}, + {'name': 'project1-semaphore', + 'global': False, + 'max': 11, + 'holders': { + 'count': 1, + 'this_tenant': [ + {'buildset_uuid': tenant1_buildset_uuid, + 'job_name': 'test-project1-semaphore'} + ], + 'other_tenants': 0 + }} + ] + self.assertEqual(expected, data) + + class TestEmptyConfig(BaseTestWeb): tenant_config_file = 'config/empty-config/main.yaml' diff --git a/tests/unit/test_zk.py b/tests/unit/test_zk.py index b1f393e47..fc6a47b93 100644 --- a/tests/unit/test_zk.py +++ b/tests/unit/test_zk.py @@ -1352,7 +1352,7 @@ class TestSystemConfigCache(ZooKeeperBaseTestCase): "use_relative_priority": True, "max_hold_expiration": 7200, "default_hold_expiration": 3600, - "default_ansible_version": "2.9", + "default_ansible_version": "5", "web_root": "/web/root", "web_status_url": "/web/status", "websocket_url": "/web/socket", diff --git a/web/src/containers/build/Artifact.jsx b/web/src/containers/build/Artifact.jsx index 4d4259440..3793222d9 100644 --- a/web/src/containers/build/Artifact.jsx +++ b/web/src/containers/build/Artifact.jsx @@ -17,6 +17,7 @@ import PropTypes from 'prop-types' import { TreeView, } from 'patternfly-react' +import ReactJson from 'react-json-view' class Artifact extends React.Component { @@ -32,7 +33,17 @@ class Artifact extends React.Component { {Object.keys(artifact.metadata).map(key => ( <tr key={key}> <td>{key}</td> - <td style={{width:'100%'}}>{artifact.metadata[key]}</td> + <td style={{width:'100%'}}> + {typeof(artifact.metadata[key]) === 'object'? + <ReactJson + src={artifact.metadata[key]} + name={null} + collapsed={true} + sortKeys={true} + enableClipboard={false} + displayDataTypes={false}/> + :artifact.metadata[key].toString()} + </td> </tr> ))} </tbody> diff --git a/zuul/ansible/2.8/action/__init__.py b/zuul/ansible/2.8/action/__init__.py deleted file mode 120000 index 4048e7ac1..000000000 --- a/zuul/ansible/2.8/action/__init__.py +++ /dev/null @@ -1 +0,0 @@ -../../base/action/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/action/command.py b/zuul/ansible/2.8/action/command.py deleted file mode 120000 index 56c6b636f..000000000 --- a/zuul/ansible/2.8/action/command.py +++ /dev/null @@ -1 +0,0 @@ -../../base/action/command.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/action/command.pyi b/zuul/ansible/2.8/action/command.pyi deleted file mode 120000 index a003281ca..000000000 --- a/zuul/ansible/2.8/action/command.pyi +++ /dev/null @@ -1 +0,0 @@ -../../base/action/command.pyi
\ No newline at end of file diff --git a/zuul/ansible/2.8/action/zuul_return.py b/zuul/ansible/2.8/action/zuul_return.py deleted file mode 120000 index 83c2fc619..000000000 --- a/zuul/ansible/2.8/action/zuul_return.py +++ /dev/null @@ -1 +0,0 @@ -../../base/action/zuul_return.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/callback/__init__.py b/zuul/ansible/2.8/callback/__init__.py deleted file mode 120000 index 00b974388..000000000 --- a/zuul/ansible/2.8/callback/__init__.py +++ /dev/null @@ -1 +0,0 @@ -../../base/callback/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/callback/zuul_json.py b/zuul/ansible/2.8/callback/zuul_json.py deleted file mode 120000 index b0a07779b..000000000 --- a/zuul/ansible/2.8/callback/zuul_json.py +++ /dev/null @@ -1 +0,0 @@ -../../base/callback/zuul_json.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/callback/zuul_stream.py b/zuul/ansible/2.8/callback/zuul_stream.py deleted file mode 120000 index f75561bf4..000000000 --- a/zuul/ansible/2.8/callback/zuul_stream.py +++ /dev/null @@ -1 +0,0 @@ -../../base/callback/zuul_stream.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/callback/zuul_unreachable.py b/zuul/ansible/2.8/callback/zuul_unreachable.py deleted file mode 120000 index 205baca6f..000000000 --- a/zuul/ansible/2.8/callback/zuul_unreachable.py +++ /dev/null @@ -1 +0,0 @@ -../../base/callback/zuul_unreachable.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/filter/__init__.py b/zuul/ansible/2.8/filter/__init__.py deleted file mode 120000 index f80a4da61..000000000 --- a/zuul/ansible/2.8/filter/__init__.py +++ /dev/null @@ -1 +0,0 @@ -../../base/filter/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/filter/zuul_filters.py b/zuul/ansible/2.8/filter/zuul_filters.py deleted file mode 120000 index d406e5fe6..000000000 --- a/zuul/ansible/2.8/filter/zuul_filters.py +++ /dev/null @@ -1 +0,0 @@ -../../base/filter/zuul_filters.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/library/__init__.py b/zuul/ansible/2.8/library/__init__.py deleted file mode 120000 index 0b68ce0f4..000000000 --- a/zuul/ansible/2.8/library/__init__.py +++ /dev/null @@ -1 +0,0 @@ -../../base/library/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/library/command.py b/zuul/ansible/2.8/library/command.py deleted file mode 100755 index 5f7f31947..000000000 --- a/zuul/ansible/2.8/library/command.py +++ /dev/null @@ -1,682 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -# Copyright: (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>, and others -# Copyright: (c) 2016, Toshio Kuratomi <tkuratomi@ansible.com> -# -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - -from __future__ import absolute_import, division, print_function -__metaclass__ = type - - -ANSIBLE_METADATA = {'metadata_version': '1.1', - 'status': ['stableinterface'], - 'supported_by': 'core'} - - -# flake8: noqa -# This file shares a significant chunk of code with an upstream ansible -# function, run_command. The goal is to not have to fork quite so much -# of that function, and discussing that design with upstream means we -# should keep the changes to substantive ones only. For that reason, this -# file is purposely not enforcing pep8, as making the function pep8 clean -# would remove our ability to easily have a discussion with our friends -# upstream - -DOCUMENTATION = ''' ---- -module: command -short_description: Executes a command on a remote node -version_added: historical -description: - - The C(command) module takes the command name followed by a list of space-delimited arguments. - - The given command will be executed on all selected nodes. It will not be - processed through the shell, so variables like C($HOME) and operations - like C("<"), C(">"), C("|"), C(";") and C("&") will not work (use the M(shell) - module if you need these features). - - For Windows targets, use the M(win_command) module instead. -options: - free_form: - description: - - The command module takes a free form command to run. There is no parameter actually named 'free form'. - See the examples! - required: yes - argv: - description: - - Allows the user to provide the command as a list vs. a string. Only the string or the list form can be - provided, not both. One or the other must be provided. - version_added: "2.6" - creates: - description: - - A filename or (since 2.0) glob pattern, when it already exists, this step will B(not) be run. - removes: - description: - - A filename or (since 2.0) glob pattern, when it does not exist, this step will B(not) be run. - version_added: "0.8" - chdir: - description: - - Change into this directory before running the command. - version_added: "0.6" - warn: - description: - - If command_warnings are on in ansible.cfg, do not warn about this particular line if set to C(no). - type: bool - default: 'yes' - version_added: "1.8" - stdin: - version_added: "2.4" - description: - - Set the stdin of the command directly to the specified value. -notes: - - If you want to run a command through the shell (say you are using C(<), C(>), C(|), etc), you actually want the M(shell) module instead. - Parsing shell metacharacters can lead to unexpected commands being executed if quoting is not done correctly so it is more secure to - use the C(command) module when possible. - - " C(creates), C(removes), and C(chdir) can be specified after the command. - For instance, if you only want to run a command if a certain file does not exist, use this." - - The C(executable) parameter is removed since version 2.4. If you have a need for this parameter, use the M(shell) module instead. - - For Windows targets, use the M(win_command) module instead. -author: - - Ansible Core Team - - Michael DeHaan -''' - -EXAMPLES = ''' -- name: return motd to registered var - command: cat /etc/motd - register: mymotd - -- name: Run the command if the specified file does not exist. - command: /usr/bin/make_database.sh arg1 arg2 - args: - creates: /path/to/database - -# You can also use the 'args' form to provide the options. -- name: This command will change the working directory to somedir/ and will only run when /path/to/database doesn't exist. - command: /usr/bin/make_database.sh arg1 arg2 - args: - chdir: somedir/ - creates: /path/to/database - -- name: use argv to send the command as a list. Be sure to leave command empty - command: - args: - argv: - - echo - - testing - -- name: safely use templated variable to run command. Always use the quote filter to avoid injection issues. - command: cat {{ myfile|quote }} - register: myoutput -''' - -RETURN = ''' -cmd: - description: the cmd that was run on the remote machine - returned: always - type: list - sample: - - echo - - hello -delta: - description: cmd end time - cmd start time - returned: always - type: string - sample: 0:00:00.001529 -end: - description: cmd end time - returned: always - type: string - sample: '2017-09-29 22:03:48.084657' -start: - description: cmd start time - returned: always - type: string - sample: '2017-09-29 22:03:48.083128' -''' - -import datetime -import glob -import os -import shlex - -from ansible.module_utils.basic import AnsibleModule - -# Imports needed for Zuul things -import re -import subprocess -import traceback -import threading -from ansible.module_utils.basic import heuristic_log_sanitize -from ansible.module_utils.six import ( - PY2, - PY3, - b, - binary_type, - string_types, - text_type, -) -from ansible.module_utils.six.moves import shlex_quote -from ansible.module_utils._text import to_native, to_bytes, to_text - - -LOG_STREAM_FILE = '/tmp/console-{log_uuid}.log' -PASSWD_ARG_RE = re.compile(r'^[-]{0,2}pass[-]?(word|wd)?') -# List to save stdout log lines in as we collect them -_log_lines = [] - - -class Console(object): - def __init__(self, log_uuid): - # The streamer currently will not ask us for output from - # loops. This flag uuid was set in the action plugin if this - # call was part of a loop. This avoids us leaving behind - # files that will never be read, but also means no other - # special-casing for any of this path. - if log_uuid == 'in-loop-ignore': - self.logfile_name = os.devnull - else: - self.logfile_name = LOG_STREAM_FILE.format(log_uuid=log_uuid) - - def __enter__(self): - self.logfile = open(self.logfile_name, 'ab', buffering=0) - return self - - def __exit__(self, etype, value, tb): - self.logfile.close() - - def addLine(self, ln): - # Note this format with deliminator is "inspired" by the old - # Jenkins format but with microsecond resolution instead of - # millisecond. It is kept so log parsing/formatting remains - # consistent. - ts = str(datetime.datetime.now()).encode('utf-8') - if not isinstance(ln, bytes): - try: - ln = ln.encode('utf-8') - except Exception: - ln = repr(ln).encode('utf-8') + b'\n' - outln = b'%s | %s' % (ts, ln) - self.logfile.write(outln) - - -def follow(fd, log_uuid): - newline_warning = False - with Console(log_uuid) as console: - while True: - line = fd.readline() - if not line: - break - _log_lines.append(line) - if not line.endswith(b'\n'): - line += b'\n' - newline_warning = True - console.addLine(line) - if newline_warning: - console.addLine('[Zuul] No trailing newline\n') - - -# Taken from ansible/module_utils/basic.py ... forking the method for now -# so that we can dive in and figure out how to make appropriate hook points -def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, executable=None, data=None, binary_data=False, path_prefix=None, cwd=None, - use_unsafe_shell=False, prompt_regex=None, environ_update=None, umask=None, encoding='utf-8', errors='surrogate_or_strict'): - ''' - Execute a command, returns rc, stdout, and stderr. - - :arg args: is the command to run - * If args is a list, the command will be run with shell=False. - * If args is a string and use_unsafe_shell=False it will split args to a list and run with shell=False - * If args is a string and use_unsafe_shell=True it runs with shell=True. - :kw check_rc: Whether to call fail_json in case of non zero RC. - Default False - :kw close_fds: See documentation for subprocess.Popen(). Default True - :kw executable: See documentation for subprocess.Popen(). Default None - :kw data: If given, information to write to the stdin of the command - :kw binary_data: If False, append a newline to the data. Default False - :kw path_prefix: If given, additional path to find the command in. - This adds to the PATH environment vairable so helper commands in - the same directory can also be found - :kw cwd: If given, working directory to run the command inside - :kw use_unsafe_shell: See `args` parameter. Default False - :kw prompt_regex: Regex string (not a compiled regex) which can be - used to detect prompts in the stdout which would otherwise cause - the execution to hang (especially if no input data is specified) - :kw environ_update: dictionary to *update* os.environ with - :kw umask: Umask to be used when running the command. Default None - :kw encoding: Since we return native strings, on python3 we need to - know the encoding to use to transform from bytes to text. If you - want to always get bytes back, use encoding=None. The default is - "utf-8". This does not affect transformation of strings given as - args. - :kw errors: Since we return native strings, on python3 we need to - transform stdout and stderr from bytes to text. If the bytes are - undecodable in the ``encoding`` specified, then use this error - handler to deal with them. The default is ``surrogate_or_strict`` - which means that the bytes will be decoded using the - surrogateescape error handler if available (available on all - python3 versions we support) otherwise a UnicodeError traceback - will be raised. This does not affect transformations of strings - given as args. - :returns: A 3-tuple of return code (integer), stdout (native string), - and stderr (native string). On python2, stdout and stderr are both - byte strings. On python3, stdout and stderr are text strings converted - according to the encoding and errors parameters. If you want byte - strings on python3, use encoding=None to turn decoding to text off. - ''' - - if not isinstance(args, (list, binary_type, text_type)): - msg = "Argument 'args' to run_command must be list or string" - self.fail_json(rc=257, cmd=args, msg=msg) - - shell = False - if use_unsafe_shell: - - # stringify args for unsafe/direct shell usage - if isinstance(args, list): - args = " ".join([shlex_quote(x) for x in args]) - - # not set explicitly, check if set by controller - if executable: - args = [executable, '-c', args] - elif self._shell not in (None, '/bin/sh'): - args = [self._shell, '-c', args] - else: - shell = True - else: - # ensure args are a list - if isinstance(args, (binary_type, text_type)): - # On python2.6 and below, shlex has problems with text type - # On python3, shlex needs a text type. - if PY2: - args = to_bytes(args, errors='surrogate_or_strict') - elif PY3: - args = to_text(args, errors='surrogateescape') - args = shlex.split(args) - - # expand shellisms - args = [os.path.expanduser(os.path.expandvars(x)) for x in args if x is not None] - - prompt_re = None - if prompt_regex: - if isinstance(prompt_regex, text_type): - if PY3: - prompt_regex = to_bytes(prompt_regex, errors='surrogateescape') - elif PY2: - prompt_regex = to_bytes(prompt_regex, errors='surrogate_or_strict') - try: - prompt_re = re.compile(prompt_regex, re.MULTILINE) - except re.error: - self.fail_json(msg="invalid prompt regular expression given to run_command") - - rc = 0 - msg = None - st_in = None - - # Manipulate the environ we'll send to the new process - old_env_vals = {} - # We can set this from both an attribute and per call - for key, val in self.run_command_environ_update.items(): - old_env_vals[key] = os.environ.get(key, None) - os.environ[key] = val - if environ_update: - for key, val in environ_update.items(): - old_env_vals[key] = os.environ.get(key, None) - os.environ[key] = val - if path_prefix: - old_env_vals['PATH'] = os.environ['PATH'] - os.environ['PATH'] = "%s:%s" % (path_prefix, os.environ['PATH']) - - # If using test-module and explode, the remote lib path will resemble ... - # /tmp/test_module_scratch/debug_dir/ansible/module_utils/basic.py - # If using ansible or ansible-playbook with a remote system ... - # /tmp/ansible_vmweLQ/ansible_modlib.zip/ansible/module_utils/basic.py - - # Clean out python paths set by ansiballz - if 'PYTHONPATH' in os.environ: - pypaths = os.environ['PYTHONPATH'].split(':') - pypaths = [x for x in pypaths - if not x.endswith('/ansible_modlib.zip') and - not x.endswith('/debug_dir')] - os.environ['PYTHONPATH'] = ':'.join(pypaths) - if not os.environ['PYTHONPATH']: - del os.environ['PYTHONPATH'] - - # create a printable version of the command for use - # in reporting later, which strips out things like - # passwords from the args list - to_clean_args = args - if PY2: - if isinstance(args, text_type): - to_clean_args = to_bytes(args) - else: - if isinstance(args, binary_type): - to_clean_args = to_text(args) - if isinstance(args, (text_type, binary_type)): - to_clean_args = shlex.split(to_clean_args) - - clean_args = [] - is_passwd = False - for arg in (to_native(a) for a in to_clean_args): - if is_passwd: - is_passwd = False - clean_args.append('********') - continue - if PASSWD_ARG_RE.match(arg): - sep_idx = arg.find('=') - if sep_idx > -1: - clean_args.append('%s=********' % arg[:sep_idx]) - continue - else: - is_passwd = True - arg = heuristic_log_sanitize(arg, self.no_log_values) - clean_args.append(arg) - clean_args = ' '.join(shlex_quote(arg) for arg in clean_args) - - if data: - st_in = subprocess.PIPE - - # ZUUL: changed stderr to follow stdout - kwargs = dict( - executable=executable, - shell=shell, - close_fds=close_fds, - stdin=st_in, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - - # store the pwd - prev_dir = os.getcwd() - - # make sure we're in the right working directory - if cwd and os.path.isdir(cwd): - cwd = os.path.abspath(os.path.expanduser(cwd)) - kwargs['cwd'] = cwd - try: - os.chdir(cwd) - except (OSError, IOError) as e: - self.fail_json(rc=e.errno, msg="Could not open %s, %s" % (cwd, to_native(e)), - exception=traceback.format_exc()) - - old_umask = None - if umask: - old_umask = os.umask(umask) - - t = None - fail_json_kwargs = None - - try: - if self._debug: - self.log('Executing: ' + clean_args) - - # ZUUL: Replaced the execution loop with the zuul_runner run function - - cmd = subprocess.Popen(args, **kwargs) - if self.no_log: - t = None - else: - t = threading.Thread(target=follow, args=(cmd.stdout, zuul_log_id)) - t.daemon = True - t.start() - - # ZUUL: Our log thread will catch the output so don't do that here. - - # # the communication logic here is essentially taken from that - # # of the _communicate() function in ssh.py - # - # stdout = b('') - # stderr = b('') - # - # # ZUUL: stderr follows stdout - # rpipes = [cmd.stdout] - - if data: - if not binary_data: - data += '\n' - if isinstance(data, text_type): - data = to_bytes(data) - cmd.stdin.write(data) - cmd.stdin.close() - - # while True: - # rfds, wfds, efds = select.select(rpipes, [], rpipes, 1) - # stdout += self._read_from_pipes(rpipes, rfds, cmd.stdout) - # - # # ZUUL: stderr follows stdout - # # stderr += self._read_from_pipes(rpipes, rfds, cmd.stderr) - # - # # if we're checking for prompts, do it now - # if prompt_re: - # if prompt_re.search(stdout) and not data: - # if encoding: - # stdout = to_native(stdout, encoding=encoding, errors=errors) - # else: - # stdout = stdout - # return (257, stdout, "A prompt was encountered while running a command, but no input data was specified") - # # only break out if no pipes are left to read or - # # the pipes are completely read and - # # the process is terminated - # if (not rpipes or not rfds) and cmd.poll() is not None: - # break - # # No pipes are left to read but process is not yet terminated - # # Only then it is safe to wait for the process to be finished - # # NOTE: Actually cmd.poll() is always None here if rpipes is empty - # elif not rpipes and cmd.poll() is None: - # cmd.wait() - # # The process is terminated. Since no pipes to read from are - # # left, there is no need to call select() again. - # break - - # ZUUL: If the console log follow thread *is* stuck in readline, - # we can't close stdout (attempting to do so raises an - # exception) , so this is disabled. - # cmd.stdout.close() - # cmd.stderr.close() - - rc = cmd.wait() - - # Give the thread that is writing the console log up to 10 seconds - # to catch up and exit. If it hasn't done so by then, it is very - # likely stuck in readline() because it spawed a child that is - # holding stdout or stderr open. - if t: - t.join(10) - with Console(zuul_log_id) as console: - if t.is_alive(): - console.addLine("[Zuul] standard output/error still open " - "after child exited") - # ZUUL: stdout and stderr are in the console log file - # ZUUL: return the saved log lines so we can ship them back - stdout = b('').join(_log_lines) - else: - stdout = b('') - stderr = b('') - - except (OSError, IOError) as e: - self.log("Error Executing CMD:%s Exception:%s" % (clean_args, to_native(e))) - # ZUUL: store fail_json_kwargs and fail later in finally - fail_json_kwargs = dict(rc=e.errno, msg=to_native(e), cmd=clean_args) - except Exception as e: - self.log("Error Executing CMD:%s Exception:%s" % (clean_args, to_native(traceback.format_exc()))) - # ZUUL: store fail_json_kwargs and fail later in finally - fail_json_kwargs = dict(rc=257, msg=to_native(e), exception=traceback.format_exc(), cmd=clean_args) - finally: - with Console(zuul_log_id) as console: - if t and t.is_alive(): - console.addLine("[Zuul] standard output/error still open " - "after child exited") - if fail_json_kwargs: - # we hit an exception and need to use the rc from - # fail_json_kwargs - rc = fail_json_kwargs['rc'] - - console.addLine("[Zuul] Task exit code: %s\n" % rc) - - if fail_json_kwargs: - self.fail_json(**fail_json_kwargs) - - # Restore env settings - for key, val in old_env_vals.items(): - if val is None: - del os.environ[key] - else: - os.environ[key] = val - - if old_umask: - os.umask(old_umask) - - if rc != 0 and check_rc: - msg = heuristic_log_sanitize(stderr.rstrip(), self.no_log_values) - self.fail_json(cmd=clean_args, rc=rc, stdout=stdout, stderr=stderr, msg=msg) - - # reset the pwd - os.chdir(prev_dir) - - if encoding is not None: - return (rc, to_native(stdout, encoding=encoding, errors=errors), - to_native(stderr, encoding=encoding, errors=errors)) - return (rc, stdout, stderr) - - -def check_command(module, commandline): - arguments = {'chown': 'owner', 'chmod': 'mode', 'chgrp': 'group', - 'ln': 'state=link', 'mkdir': 'state=directory', - 'rmdir': 'state=absent', 'rm': 'state=absent', 'touch': 'state=touch'} - commands = {'curl': 'get_url or uri', 'wget': 'get_url or uri', - 'svn': 'subversion', 'service': 'service', - 'mount': 'mount', 'rpm': 'yum, dnf or zypper', 'yum': 'yum', 'apt-get': 'apt', - 'tar': 'unarchive', 'unzip': 'unarchive', 'sed': 'replace, lineinfile or template', - 'dnf': 'dnf', 'zypper': 'zypper'} - become = ['sudo', 'su', 'pbrun', 'pfexec', 'runas', 'pmrun', 'machinectl'] - if isinstance(commandline, list): - command = commandline[0] - else: - command = commandline.split()[0] - command = os.path.basename(command) - - disable_suffix = "If you need to use command because {mod} is insufficient you can add" \ - " warn=False to this command task or set command_warnings=False in" \ - " ansible.cfg to get rid of this message." - substitutions = {'mod': None, 'cmd': command} - - if command in arguments: - msg = "Consider using the {mod} module with {subcmd} rather than running {cmd}. " + disable_suffix - substitutions['mod'] = 'file' - substitutions['subcmd'] = arguments[command] - module.warn(msg.format(**substitutions)) - - if command in commands: - msg = "Consider using the {mod} module rather than running {cmd}. " + disable_suffix - substitutions['mod'] = commands[command] - module.warn(msg.format(**substitutions)) - - if command in become: - module.warn("Consider using 'become', 'become_method', and 'become_user' rather than running %s" % (command,)) - - -def main(): - - # the command module is the one ansible module that does not take key=value args - # hence don't copy this one if you are looking to build others! - module = AnsibleModule( - argument_spec=dict( - _raw_params=dict(), - _uses_shell=dict(type='bool', default=False), - argv=dict(type='list'), - chdir=dict(type='path'), - executable=dict(), - creates=dict(type='path'), - removes=dict(type='path'), - # The default for this really comes from the action plugin - warn=dict(type='bool', default=True), - stdin=dict(required=False), - zuul_log_id=dict(type='str'), - ) - ) - shell = module.params['_uses_shell'] - chdir = module.params['chdir'] - executable = module.params['executable'] - args = module.params['_raw_params'] - argv = module.params['argv'] - creates = module.params['creates'] - removes = module.params['removes'] - warn = module.params['warn'] - stdin = module.params['stdin'] - zuul_log_id = module.params['zuul_log_id'] - - if not shell and executable: - module.warn("As of Ansible 2.4, the parameter 'executable' is no longer supported with the 'command' module. Not using '%s'." % executable) - executable = None - - if not zuul_log_id: - module.fail_json(rc=256, msg="zuul_log_id missing: %s" % module.params) - - if (not args or args.strip() == '') and not argv: - module.fail_json(rc=256, msg="no command given") - - if args and argv: - module.fail_json(rc=256, msg="only command or argv can be given, not both") - - if not shell and args: - args = shlex.split(args) - - args = args or argv - - if chdir: - chdir = os.path.abspath(chdir) - os.chdir(chdir) - - if creates: - # do not run the command if the line contains creates=filename - # and the filename already exists. This allows idempotence - # of command executions. - if glob.glob(creates): - module.exit_json( - cmd=args, - stdout="skipped, since %s exists" % creates, - changed=False, - rc=0 - ) - - if removes: - # do not run the command if the line contains removes=filename - # and the filename does not exist. This allows idempotence - # of command executions. - if not glob.glob(removes): - module.exit_json( - cmd=args, - stdout="skipped, since %s does not exist" % removes, - changed=False, - rc=0 - ) - - if warn: - check_command(module, args) - - startd = datetime.datetime.now() - - rc, out, err = zuul_run_command(module, args, zuul_log_id, executable=executable, use_unsafe_shell=shell, encoding=None, data=stdin) - - endd = datetime.datetime.now() - delta = endd - startd - - result = dict( - cmd=args, - stdout=out.rstrip(b"\r\n"), - stderr=err.rstrip(b"\r\n"), - rc=rc, - start=str(startd), - end=str(endd), - delta=str(delta), - changed=True, - zuul_log_id=zuul_log_id - ) - - if rc != 0: - module.fail_json(msg='non-zero return code', **result) - - module.exit_json(**result) - - -if __name__ == '__main__': - main() diff --git a/zuul/ansible/2.8/library/zuul_console.py b/zuul/ansible/2.8/library/zuul_console.py deleted file mode 120000 index 7c905e0f9..000000000 --- a/zuul/ansible/2.8/library/zuul_console.py +++ /dev/null @@ -1 +0,0 @@ -../../base/library/zuul_console.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/logconfig.py b/zuul/ansible/2.8/logconfig.py deleted file mode 120000 index 767cb2e81..000000000 --- a/zuul/ansible/2.8/logconfig.py +++ /dev/null @@ -1 +0,0 @@ -../logconfig.py
\ No newline at end of file diff --git a/zuul/ansible/2.8/paths.py b/zuul/ansible/2.8/paths.py deleted file mode 120000 index dbdb1858e..000000000 --- a/zuul/ansible/2.8/paths.py +++ /dev/null @@ -1 +0,0 @@ -../paths.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/action/__init__.py b/zuul/ansible/2.9/action/__init__.py deleted file mode 120000 index 4048e7ac1..000000000 --- a/zuul/ansible/2.9/action/__init__.py +++ /dev/null @@ -1 +0,0 @@ -../../base/action/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/action/command.py b/zuul/ansible/2.9/action/command.py deleted file mode 120000 index 56c6b636f..000000000 --- a/zuul/ansible/2.9/action/command.py +++ /dev/null @@ -1 +0,0 @@ -../../base/action/command.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/action/command.pyi b/zuul/ansible/2.9/action/command.pyi deleted file mode 120000 index a003281ca..000000000 --- a/zuul/ansible/2.9/action/command.pyi +++ /dev/null @@ -1 +0,0 @@ -../../base/action/command.pyi
\ No newline at end of file diff --git a/zuul/ansible/2.9/action/zuul_return.py b/zuul/ansible/2.9/action/zuul_return.py deleted file mode 120000 index 83c2fc619..000000000 --- a/zuul/ansible/2.9/action/zuul_return.py +++ /dev/null @@ -1 +0,0 @@ -../../base/action/zuul_return.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/callback/__init__.py b/zuul/ansible/2.9/callback/__init__.py deleted file mode 120000 index 00b974388..000000000 --- a/zuul/ansible/2.9/callback/__init__.py +++ /dev/null @@ -1 +0,0 @@ -../../base/callback/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/callback/zuul_json.py b/zuul/ansible/2.9/callback/zuul_json.py deleted file mode 120000 index b0a07779b..000000000 --- a/zuul/ansible/2.9/callback/zuul_json.py +++ /dev/null @@ -1 +0,0 @@ -../../base/callback/zuul_json.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/callback/zuul_stream.py b/zuul/ansible/2.9/callback/zuul_stream.py deleted file mode 120000 index f75561bf4..000000000 --- a/zuul/ansible/2.9/callback/zuul_stream.py +++ /dev/null @@ -1 +0,0 @@ -../../base/callback/zuul_stream.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/callback/zuul_unreachable.py b/zuul/ansible/2.9/callback/zuul_unreachable.py deleted file mode 120000 index 205baca6f..000000000 --- a/zuul/ansible/2.9/callback/zuul_unreachable.py +++ /dev/null @@ -1 +0,0 @@ -../../base/callback/zuul_unreachable.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/filter/__init__.py b/zuul/ansible/2.9/filter/__init__.py deleted file mode 120000 index f80a4da61..000000000 --- a/zuul/ansible/2.9/filter/__init__.py +++ /dev/null @@ -1 +0,0 @@ -../../base/filter/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/filter/zuul_filters.py b/zuul/ansible/2.9/filter/zuul_filters.py deleted file mode 120000 index d406e5fe6..000000000 --- a/zuul/ansible/2.9/filter/zuul_filters.py +++ /dev/null @@ -1 +0,0 @@ -../../base/filter/zuul_filters.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/library/__init__.py b/zuul/ansible/2.9/library/__init__.py deleted file mode 120000 index 0b68ce0f4..000000000 --- a/zuul/ansible/2.9/library/__init__.py +++ /dev/null @@ -1 +0,0 @@ -../../base/library/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/library/command.py b/zuul/ansible/2.9/library/command.py deleted file mode 100755 index 5f7f31947..000000000 --- a/zuul/ansible/2.9/library/command.py +++ /dev/null @@ -1,682 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -# Copyright: (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>, and others -# Copyright: (c) 2016, Toshio Kuratomi <tkuratomi@ansible.com> -# -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - -from __future__ import absolute_import, division, print_function -__metaclass__ = type - - -ANSIBLE_METADATA = {'metadata_version': '1.1', - 'status': ['stableinterface'], - 'supported_by': 'core'} - - -# flake8: noqa -# This file shares a significant chunk of code with an upstream ansible -# function, run_command. The goal is to not have to fork quite so much -# of that function, and discussing that design with upstream means we -# should keep the changes to substantive ones only. For that reason, this -# file is purposely not enforcing pep8, as making the function pep8 clean -# would remove our ability to easily have a discussion with our friends -# upstream - -DOCUMENTATION = ''' ---- -module: command -short_description: Executes a command on a remote node -version_added: historical -description: - - The C(command) module takes the command name followed by a list of space-delimited arguments. - - The given command will be executed on all selected nodes. It will not be - processed through the shell, so variables like C($HOME) and operations - like C("<"), C(">"), C("|"), C(";") and C("&") will not work (use the M(shell) - module if you need these features). - - For Windows targets, use the M(win_command) module instead. -options: - free_form: - description: - - The command module takes a free form command to run. There is no parameter actually named 'free form'. - See the examples! - required: yes - argv: - description: - - Allows the user to provide the command as a list vs. a string. Only the string or the list form can be - provided, not both. One or the other must be provided. - version_added: "2.6" - creates: - description: - - A filename or (since 2.0) glob pattern, when it already exists, this step will B(not) be run. - removes: - description: - - A filename or (since 2.0) glob pattern, when it does not exist, this step will B(not) be run. - version_added: "0.8" - chdir: - description: - - Change into this directory before running the command. - version_added: "0.6" - warn: - description: - - If command_warnings are on in ansible.cfg, do not warn about this particular line if set to C(no). - type: bool - default: 'yes' - version_added: "1.8" - stdin: - version_added: "2.4" - description: - - Set the stdin of the command directly to the specified value. -notes: - - If you want to run a command through the shell (say you are using C(<), C(>), C(|), etc), you actually want the M(shell) module instead. - Parsing shell metacharacters can lead to unexpected commands being executed if quoting is not done correctly so it is more secure to - use the C(command) module when possible. - - " C(creates), C(removes), and C(chdir) can be specified after the command. - For instance, if you only want to run a command if a certain file does not exist, use this." - - The C(executable) parameter is removed since version 2.4. If you have a need for this parameter, use the M(shell) module instead. - - For Windows targets, use the M(win_command) module instead. -author: - - Ansible Core Team - - Michael DeHaan -''' - -EXAMPLES = ''' -- name: return motd to registered var - command: cat /etc/motd - register: mymotd - -- name: Run the command if the specified file does not exist. - command: /usr/bin/make_database.sh arg1 arg2 - args: - creates: /path/to/database - -# You can also use the 'args' form to provide the options. -- name: This command will change the working directory to somedir/ and will only run when /path/to/database doesn't exist. - command: /usr/bin/make_database.sh arg1 arg2 - args: - chdir: somedir/ - creates: /path/to/database - -- name: use argv to send the command as a list. Be sure to leave command empty - command: - args: - argv: - - echo - - testing - -- name: safely use templated variable to run command. Always use the quote filter to avoid injection issues. - command: cat {{ myfile|quote }} - register: myoutput -''' - -RETURN = ''' -cmd: - description: the cmd that was run on the remote machine - returned: always - type: list - sample: - - echo - - hello -delta: - description: cmd end time - cmd start time - returned: always - type: string - sample: 0:00:00.001529 -end: - description: cmd end time - returned: always - type: string - sample: '2017-09-29 22:03:48.084657' -start: - description: cmd start time - returned: always - type: string - sample: '2017-09-29 22:03:48.083128' -''' - -import datetime -import glob -import os -import shlex - -from ansible.module_utils.basic import AnsibleModule - -# Imports needed for Zuul things -import re -import subprocess -import traceback -import threading -from ansible.module_utils.basic import heuristic_log_sanitize -from ansible.module_utils.six import ( - PY2, - PY3, - b, - binary_type, - string_types, - text_type, -) -from ansible.module_utils.six.moves import shlex_quote -from ansible.module_utils._text import to_native, to_bytes, to_text - - -LOG_STREAM_FILE = '/tmp/console-{log_uuid}.log' -PASSWD_ARG_RE = re.compile(r'^[-]{0,2}pass[-]?(word|wd)?') -# List to save stdout log lines in as we collect them -_log_lines = [] - - -class Console(object): - def __init__(self, log_uuid): - # The streamer currently will not ask us for output from - # loops. This flag uuid was set in the action plugin if this - # call was part of a loop. This avoids us leaving behind - # files that will never be read, but also means no other - # special-casing for any of this path. - if log_uuid == 'in-loop-ignore': - self.logfile_name = os.devnull - else: - self.logfile_name = LOG_STREAM_FILE.format(log_uuid=log_uuid) - - def __enter__(self): - self.logfile = open(self.logfile_name, 'ab', buffering=0) - return self - - def __exit__(self, etype, value, tb): - self.logfile.close() - - def addLine(self, ln): - # Note this format with deliminator is "inspired" by the old - # Jenkins format but with microsecond resolution instead of - # millisecond. It is kept so log parsing/formatting remains - # consistent. - ts = str(datetime.datetime.now()).encode('utf-8') - if not isinstance(ln, bytes): - try: - ln = ln.encode('utf-8') - except Exception: - ln = repr(ln).encode('utf-8') + b'\n' - outln = b'%s | %s' % (ts, ln) - self.logfile.write(outln) - - -def follow(fd, log_uuid): - newline_warning = False - with Console(log_uuid) as console: - while True: - line = fd.readline() - if not line: - break - _log_lines.append(line) - if not line.endswith(b'\n'): - line += b'\n' - newline_warning = True - console.addLine(line) - if newline_warning: - console.addLine('[Zuul] No trailing newline\n') - - -# Taken from ansible/module_utils/basic.py ... forking the method for now -# so that we can dive in and figure out how to make appropriate hook points -def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, executable=None, data=None, binary_data=False, path_prefix=None, cwd=None, - use_unsafe_shell=False, prompt_regex=None, environ_update=None, umask=None, encoding='utf-8', errors='surrogate_or_strict'): - ''' - Execute a command, returns rc, stdout, and stderr. - - :arg args: is the command to run - * If args is a list, the command will be run with shell=False. - * If args is a string and use_unsafe_shell=False it will split args to a list and run with shell=False - * If args is a string and use_unsafe_shell=True it runs with shell=True. - :kw check_rc: Whether to call fail_json in case of non zero RC. - Default False - :kw close_fds: See documentation for subprocess.Popen(). Default True - :kw executable: See documentation for subprocess.Popen(). Default None - :kw data: If given, information to write to the stdin of the command - :kw binary_data: If False, append a newline to the data. Default False - :kw path_prefix: If given, additional path to find the command in. - This adds to the PATH environment vairable so helper commands in - the same directory can also be found - :kw cwd: If given, working directory to run the command inside - :kw use_unsafe_shell: See `args` parameter. Default False - :kw prompt_regex: Regex string (not a compiled regex) which can be - used to detect prompts in the stdout which would otherwise cause - the execution to hang (especially if no input data is specified) - :kw environ_update: dictionary to *update* os.environ with - :kw umask: Umask to be used when running the command. Default None - :kw encoding: Since we return native strings, on python3 we need to - know the encoding to use to transform from bytes to text. If you - want to always get bytes back, use encoding=None. The default is - "utf-8". This does not affect transformation of strings given as - args. - :kw errors: Since we return native strings, on python3 we need to - transform stdout and stderr from bytes to text. If the bytes are - undecodable in the ``encoding`` specified, then use this error - handler to deal with them. The default is ``surrogate_or_strict`` - which means that the bytes will be decoded using the - surrogateescape error handler if available (available on all - python3 versions we support) otherwise a UnicodeError traceback - will be raised. This does not affect transformations of strings - given as args. - :returns: A 3-tuple of return code (integer), stdout (native string), - and stderr (native string). On python2, stdout and stderr are both - byte strings. On python3, stdout and stderr are text strings converted - according to the encoding and errors parameters. If you want byte - strings on python3, use encoding=None to turn decoding to text off. - ''' - - if not isinstance(args, (list, binary_type, text_type)): - msg = "Argument 'args' to run_command must be list or string" - self.fail_json(rc=257, cmd=args, msg=msg) - - shell = False - if use_unsafe_shell: - - # stringify args for unsafe/direct shell usage - if isinstance(args, list): - args = " ".join([shlex_quote(x) for x in args]) - - # not set explicitly, check if set by controller - if executable: - args = [executable, '-c', args] - elif self._shell not in (None, '/bin/sh'): - args = [self._shell, '-c', args] - else: - shell = True - else: - # ensure args are a list - if isinstance(args, (binary_type, text_type)): - # On python2.6 and below, shlex has problems with text type - # On python3, shlex needs a text type. - if PY2: - args = to_bytes(args, errors='surrogate_or_strict') - elif PY3: - args = to_text(args, errors='surrogateescape') - args = shlex.split(args) - - # expand shellisms - args = [os.path.expanduser(os.path.expandvars(x)) for x in args if x is not None] - - prompt_re = None - if prompt_regex: - if isinstance(prompt_regex, text_type): - if PY3: - prompt_regex = to_bytes(prompt_regex, errors='surrogateescape') - elif PY2: - prompt_regex = to_bytes(prompt_regex, errors='surrogate_or_strict') - try: - prompt_re = re.compile(prompt_regex, re.MULTILINE) - except re.error: - self.fail_json(msg="invalid prompt regular expression given to run_command") - - rc = 0 - msg = None - st_in = None - - # Manipulate the environ we'll send to the new process - old_env_vals = {} - # We can set this from both an attribute and per call - for key, val in self.run_command_environ_update.items(): - old_env_vals[key] = os.environ.get(key, None) - os.environ[key] = val - if environ_update: - for key, val in environ_update.items(): - old_env_vals[key] = os.environ.get(key, None) - os.environ[key] = val - if path_prefix: - old_env_vals['PATH'] = os.environ['PATH'] - os.environ['PATH'] = "%s:%s" % (path_prefix, os.environ['PATH']) - - # If using test-module and explode, the remote lib path will resemble ... - # /tmp/test_module_scratch/debug_dir/ansible/module_utils/basic.py - # If using ansible or ansible-playbook with a remote system ... - # /tmp/ansible_vmweLQ/ansible_modlib.zip/ansible/module_utils/basic.py - - # Clean out python paths set by ansiballz - if 'PYTHONPATH' in os.environ: - pypaths = os.environ['PYTHONPATH'].split(':') - pypaths = [x for x in pypaths - if not x.endswith('/ansible_modlib.zip') and - not x.endswith('/debug_dir')] - os.environ['PYTHONPATH'] = ':'.join(pypaths) - if not os.environ['PYTHONPATH']: - del os.environ['PYTHONPATH'] - - # create a printable version of the command for use - # in reporting later, which strips out things like - # passwords from the args list - to_clean_args = args - if PY2: - if isinstance(args, text_type): - to_clean_args = to_bytes(args) - else: - if isinstance(args, binary_type): - to_clean_args = to_text(args) - if isinstance(args, (text_type, binary_type)): - to_clean_args = shlex.split(to_clean_args) - - clean_args = [] - is_passwd = False - for arg in (to_native(a) for a in to_clean_args): - if is_passwd: - is_passwd = False - clean_args.append('********') - continue - if PASSWD_ARG_RE.match(arg): - sep_idx = arg.find('=') - if sep_idx > -1: - clean_args.append('%s=********' % arg[:sep_idx]) - continue - else: - is_passwd = True - arg = heuristic_log_sanitize(arg, self.no_log_values) - clean_args.append(arg) - clean_args = ' '.join(shlex_quote(arg) for arg in clean_args) - - if data: - st_in = subprocess.PIPE - - # ZUUL: changed stderr to follow stdout - kwargs = dict( - executable=executable, - shell=shell, - close_fds=close_fds, - stdin=st_in, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - - # store the pwd - prev_dir = os.getcwd() - - # make sure we're in the right working directory - if cwd and os.path.isdir(cwd): - cwd = os.path.abspath(os.path.expanduser(cwd)) - kwargs['cwd'] = cwd - try: - os.chdir(cwd) - except (OSError, IOError) as e: - self.fail_json(rc=e.errno, msg="Could not open %s, %s" % (cwd, to_native(e)), - exception=traceback.format_exc()) - - old_umask = None - if umask: - old_umask = os.umask(umask) - - t = None - fail_json_kwargs = None - - try: - if self._debug: - self.log('Executing: ' + clean_args) - - # ZUUL: Replaced the execution loop with the zuul_runner run function - - cmd = subprocess.Popen(args, **kwargs) - if self.no_log: - t = None - else: - t = threading.Thread(target=follow, args=(cmd.stdout, zuul_log_id)) - t.daemon = True - t.start() - - # ZUUL: Our log thread will catch the output so don't do that here. - - # # the communication logic here is essentially taken from that - # # of the _communicate() function in ssh.py - # - # stdout = b('') - # stderr = b('') - # - # # ZUUL: stderr follows stdout - # rpipes = [cmd.stdout] - - if data: - if not binary_data: - data += '\n' - if isinstance(data, text_type): - data = to_bytes(data) - cmd.stdin.write(data) - cmd.stdin.close() - - # while True: - # rfds, wfds, efds = select.select(rpipes, [], rpipes, 1) - # stdout += self._read_from_pipes(rpipes, rfds, cmd.stdout) - # - # # ZUUL: stderr follows stdout - # # stderr += self._read_from_pipes(rpipes, rfds, cmd.stderr) - # - # # if we're checking for prompts, do it now - # if prompt_re: - # if prompt_re.search(stdout) and not data: - # if encoding: - # stdout = to_native(stdout, encoding=encoding, errors=errors) - # else: - # stdout = stdout - # return (257, stdout, "A prompt was encountered while running a command, but no input data was specified") - # # only break out if no pipes are left to read or - # # the pipes are completely read and - # # the process is terminated - # if (not rpipes or not rfds) and cmd.poll() is not None: - # break - # # No pipes are left to read but process is not yet terminated - # # Only then it is safe to wait for the process to be finished - # # NOTE: Actually cmd.poll() is always None here if rpipes is empty - # elif not rpipes and cmd.poll() is None: - # cmd.wait() - # # The process is terminated. Since no pipes to read from are - # # left, there is no need to call select() again. - # break - - # ZUUL: If the console log follow thread *is* stuck in readline, - # we can't close stdout (attempting to do so raises an - # exception) , so this is disabled. - # cmd.stdout.close() - # cmd.stderr.close() - - rc = cmd.wait() - - # Give the thread that is writing the console log up to 10 seconds - # to catch up and exit. If it hasn't done so by then, it is very - # likely stuck in readline() because it spawed a child that is - # holding stdout or stderr open. - if t: - t.join(10) - with Console(zuul_log_id) as console: - if t.is_alive(): - console.addLine("[Zuul] standard output/error still open " - "after child exited") - # ZUUL: stdout and stderr are in the console log file - # ZUUL: return the saved log lines so we can ship them back - stdout = b('').join(_log_lines) - else: - stdout = b('') - stderr = b('') - - except (OSError, IOError) as e: - self.log("Error Executing CMD:%s Exception:%s" % (clean_args, to_native(e))) - # ZUUL: store fail_json_kwargs and fail later in finally - fail_json_kwargs = dict(rc=e.errno, msg=to_native(e), cmd=clean_args) - except Exception as e: - self.log("Error Executing CMD:%s Exception:%s" % (clean_args, to_native(traceback.format_exc()))) - # ZUUL: store fail_json_kwargs and fail later in finally - fail_json_kwargs = dict(rc=257, msg=to_native(e), exception=traceback.format_exc(), cmd=clean_args) - finally: - with Console(zuul_log_id) as console: - if t and t.is_alive(): - console.addLine("[Zuul] standard output/error still open " - "after child exited") - if fail_json_kwargs: - # we hit an exception and need to use the rc from - # fail_json_kwargs - rc = fail_json_kwargs['rc'] - - console.addLine("[Zuul] Task exit code: %s\n" % rc) - - if fail_json_kwargs: - self.fail_json(**fail_json_kwargs) - - # Restore env settings - for key, val in old_env_vals.items(): - if val is None: - del os.environ[key] - else: - os.environ[key] = val - - if old_umask: - os.umask(old_umask) - - if rc != 0 and check_rc: - msg = heuristic_log_sanitize(stderr.rstrip(), self.no_log_values) - self.fail_json(cmd=clean_args, rc=rc, stdout=stdout, stderr=stderr, msg=msg) - - # reset the pwd - os.chdir(prev_dir) - - if encoding is not None: - return (rc, to_native(stdout, encoding=encoding, errors=errors), - to_native(stderr, encoding=encoding, errors=errors)) - return (rc, stdout, stderr) - - -def check_command(module, commandline): - arguments = {'chown': 'owner', 'chmod': 'mode', 'chgrp': 'group', - 'ln': 'state=link', 'mkdir': 'state=directory', - 'rmdir': 'state=absent', 'rm': 'state=absent', 'touch': 'state=touch'} - commands = {'curl': 'get_url or uri', 'wget': 'get_url or uri', - 'svn': 'subversion', 'service': 'service', - 'mount': 'mount', 'rpm': 'yum, dnf or zypper', 'yum': 'yum', 'apt-get': 'apt', - 'tar': 'unarchive', 'unzip': 'unarchive', 'sed': 'replace, lineinfile or template', - 'dnf': 'dnf', 'zypper': 'zypper'} - become = ['sudo', 'su', 'pbrun', 'pfexec', 'runas', 'pmrun', 'machinectl'] - if isinstance(commandline, list): - command = commandline[0] - else: - command = commandline.split()[0] - command = os.path.basename(command) - - disable_suffix = "If you need to use command because {mod} is insufficient you can add" \ - " warn=False to this command task or set command_warnings=False in" \ - " ansible.cfg to get rid of this message." - substitutions = {'mod': None, 'cmd': command} - - if command in arguments: - msg = "Consider using the {mod} module with {subcmd} rather than running {cmd}. " + disable_suffix - substitutions['mod'] = 'file' - substitutions['subcmd'] = arguments[command] - module.warn(msg.format(**substitutions)) - - if command in commands: - msg = "Consider using the {mod} module rather than running {cmd}. " + disable_suffix - substitutions['mod'] = commands[command] - module.warn(msg.format(**substitutions)) - - if command in become: - module.warn("Consider using 'become', 'become_method', and 'become_user' rather than running %s" % (command,)) - - -def main(): - - # the command module is the one ansible module that does not take key=value args - # hence don't copy this one if you are looking to build others! - module = AnsibleModule( - argument_spec=dict( - _raw_params=dict(), - _uses_shell=dict(type='bool', default=False), - argv=dict(type='list'), - chdir=dict(type='path'), - executable=dict(), - creates=dict(type='path'), - removes=dict(type='path'), - # The default for this really comes from the action plugin - warn=dict(type='bool', default=True), - stdin=dict(required=False), - zuul_log_id=dict(type='str'), - ) - ) - shell = module.params['_uses_shell'] - chdir = module.params['chdir'] - executable = module.params['executable'] - args = module.params['_raw_params'] - argv = module.params['argv'] - creates = module.params['creates'] - removes = module.params['removes'] - warn = module.params['warn'] - stdin = module.params['stdin'] - zuul_log_id = module.params['zuul_log_id'] - - if not shell and executable: - module.warn("As of Ansible 2.4, the parameter 'executable' is no longer supported with the 'command' module. Not using '%s'." % executable) - executable = None - - if not zuul_log_id: - module.fail_json(rc=256, msg="zuul_log_id missing: %s" % module.params) - - if (not args or args.strip() == '') and not argv: - module.fail_json(rc=256, msg="no command given") - - if args and argv: - module.fail_json(rc=256, msg="only command or argv can be given, not both") - - if not shell and args: - args = shlex.split(args) - - args = args or argv - - if chdir: - chdir = os.path.abspath(chdir) - os.chdir(chdir) - - if creates: - # do not run the command if the line contains creates=filename - # and the filename already exists. This allows idempotence - # of command executions. - if glob.glob(creates): - module.exit_json( - cmd=args, - stdout="skipped, since %s exists" % creates, - changed=False, - rc=0 - ) - - if removes: - # do not run the command if the line contains removes=filename - # and the filename does not exist. This allows idempotence - # of command executions. - if not glob.glob(removes): - module.exit_json( - cmd=args, - stdout="skipped, since %s does not exist" % removes, - changed=False, - rc=0 - ) - - if warn: - check_command(module, args) - - startd = datetime.datetime.now() - - rc, out, err = zuul_run_command(module, args, zuul_log_id, executable=executable, use_unsafe_shell=shell, encoding=None, data=stdin) - - endd = datetime.datetime.now() - delta = endd - startd - - result = dict( - cmd=args, - stdout=out.rstrip(b"\r\n"), - stderr=err.rstrip(b"\r\n"), - rc=rc, - start=str(startd), - end=str(endd), - delta=str(delta), - changed=True, - zuul_log_id=zuul_log_id - ) - - if rc != 0: - module.fail_json(msg='non-zero return code', **result) - - module.exit_json(**result) - - -if __name__ == '__main__': - main() diff --git a/zuul/ansible/2.9/library/zuul_console.py b/zuul/ansible/2.9/library/zuul_console.py deleted file mode 120000 index 7c905e0f9..000000000 --- a/zuul/ansible/2.9/library/zuul_console.py +++ /dev/null @@ -1 +0,0 @@ -../../base/library/zuul_console.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/logconfig.py b/zuul/ansible/2.9/logconfig.py deleted file mode 120000 index 767cb2e81..000000000 --- a/zuul/ansible/2.9/logconfig.py +++ /dev/null @@ -1 +0,0 @@ -../logconfig.py
\ No newline at end of file diff --git a/zuul/ansible/2.9/paths.py b/zuul/ansible/2.9/paths.py deleted file mode 120000 index dbdb1858e..000000000 --- a/zuul/ansible/2.9/paths.py +++ /dev/null @@ -1 +0,0 @@ -../paths.py
\ No newline at end of file diff --git a/zuul/ansible/base/action/command.py b/zuul/ansible/base/action/command.py index 3bb88d8e5..a969a8b2b 100644 --- a/zuul/ansible/base/action/command.py +++ b/zuul/ansible/base/action/command.py @@ -13,8 +13,10 @@ # You should have received a copy of the GNU General Public License # along with this software. If not, see <http://www.gnu.org/licenses/>. - from zuul.ansible import paths + +from ansible.module_utils.parsing.convert_bool import boolean + command = paths._import_ansible_action_plugin("command") @@ -25,10 +27,17 @@ class ActionModule(command.ActionModule): if self._task.action in ( 'command', 'shell', 'ansible.builtin.command', 'ansible.builtin.shell'): - # This is a bit lame, but we do not log loops in the - # zuul_stream.py callback. This allows us to not write - # out command.py output to files that will never be read. - if 'ansible_loop_var' in task_vars: + # Overloading the UUID is a bit lame, but it stops us + # having to modify the library command.py too much. Both + # of these below stop the creation of the files on disk + # for situations where they won't be read and cleaned-up. + skip = boolean( + self._templar.template( + "{{zuul_console_disabled|default(false)}}")) + if skip: + self._task.args['zuul_log_id'] = 'skip' + elif 'ansible_loop_var' in task_vars: + # we do not log loops in the zuul_stream.py callback. self._task.args['zuul_log_id'] = 'in-loop-ignore' else: # Get a unique key for ZUUL_LOG_ID_MAP. ZUUL_LOG_ID_MAP diff --git a/zuul/ansible/base/library/command.py b/zuul/ansible/base/library/command.py index 3c22849ae..c52412edf 100755 --- a/zuul/ansible/base/library/command.py +++ b/zuul/ansible/base/library/command.py @@ -270,6 +270,8 @@ class Console(object): # special-casing for any of this path. if log_uuid == 'in-loop-ignore': self.logfile_name = os.devnull + elif log_uuid == 'skip': + self.logfile_name = os.devnull else: self.logfile_name = LOG_STREAM_FILE.format(log_uuid=log_uuid) @@ -494,7 +496,8 @@ def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, ex try: if self._debug: - self.log('Executing: ' + self._clean_args(args)) + self.log('Executing <%s>: %s', + zuul_log_id, self._clean_args(args)) # ZUUL: Replaced the execution loop with the zuul_runner run function diff --git a/zuul/configloader.py b/zuul/configloader.py index 037fc48aa..a92f5337c 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -1740,10 +1740,11 @@ class TenantParser(object): tenant.layout = self._parseLayout( tenant, parsed_config, loading_errors, layout_uuid) + tenant.semaphore_handler = SemaphoreHandler( + self.zk_client, self.statsd, tenant.name, tenant.layout, abide, + read_only=(not bool(self.scheduler)) + ) if self.scheduler: - tenant.semaphore_handler = SemaphoreHandler( - self.zk_client, self.statsd, tenant.name, tenant.layout, abide - ) # Only call the postConfig hook if we have a scheduler as this will # change data in ZooKeeper. In case we are in a zuul-web context, # we don't want to do that. diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py index 2213293f8..d8e4df586 100644 --- a/zuul/driver/gerrit/gerritconnection.py +++ b/zuul/driver/gerrit/gerritconnection.py @@ -589,6 +589,7 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection): _poller_class = GerritPoller _ref_watcher_class = GitWatcher ref_watcher_poll_interval = 60 + submit_retry_backoff = 10 def __init__(self, driver, connection_name, connection_config): super(GerritConnection, self).__init__(driver, connection_name, @@ -753,7 +754,8 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection): if r.status_code == 400: raise HTTPBadRequestException('Received response 400: %s' % r.text) elif r.status_code != 200: - raise Exception("Received response %s" % (r.status_code,)) + raise Exception("Received response %s: %s" % ( + r.status_code, r.text)) ret = None if r.text and len(r.text) > 4: try: @@ -1315,10 +1317,10 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection): log.exception( "Bad request submitting check data to gerrit.") break - except Exception: - log.exception("Error submitting check data to gerrit, " - "attempt %s", x) - time.sleep(x * 10) + except Exception as e: + log.exception("Error submitting check data to gerrit on " + "attempt %s: %s", x, e) + time.sleep(x * self.submit_retry_backoff) def review_http(self, item, message, submit, labels, checks_api, file_comments, phase1, phase2, @@ -1375,10 +1377,12 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection): log.exception( "Bad request submitting check data to gerrit.") break - except Exception: + except Exception as e: log.exception( - "Error submitting data to gerrit, attempt %s", x) - time.sleep(x * 10) + "Error submitting data to gerrit " + "on attempt %s: %s", + x, e) + time.sleep(x * self.submit_retry_backoff) if phase2 and change.is_current_patchset and submit: for x in range(1, 4): try: @@ -1392,10 +1396,11 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection): log.exception( "Bad request submitting check data to gerrit.") break - except Exception: + except Exception as e: log.exception( - "Error submitting data to gerrit, attempt %s", x) - time.sleep(x * 10) + "Error submitting data to gerrit on attempt %s: %s", + x, e) + time.sleep(x * self.submit_retry_backoff) def queryChangeSSH(self, number, event=None): args = '--all-approvals --comments --commit-message' diff --git a/zuul/driver/github/githubreporter.py b/zuul/driver/github/githubreporter.py index 73d3a6f13..8c5e8944c 100644 --- a/zuul/driver/github/githubreporter.py +++ b/zuul/driver/github/githubreporter.py @@ -332,7 +332,17 @@ class GithubReporter(BaseReporter): for r in change.reviews: name = r['by']['name'] email = r['by']['email'] - review_users.append('Reviewed-by: {} <{}>'.format(name, email)) + username = r['by']['username'] + review_message = 'Reviewed-by:' + if name: + review_message += ' {}'.format(name) + elif username: + review_message += ' {}'.format(username) + else: + review_message += ' Anonymous' + if email: + review_message += ' <{}>'.format(email) + review_users.append(review_message) merge_message += '\n\n' merge_message += '\n'.join(review_users) diff --git a/zuul/lib/ansible-config.conf b/zuul/lib/ansible-config.conf index 35a9f59fe..2f4ccd76b 100644 --- a/zuul/lib/ansible-config.conf +++ b/zuul/lib/ansible-config.conf @@ -4,18 +4,6 @@ default_version = 5 # OpenStackSDK 0.99.0 coincides with CORS header problems in some providers requirements = openstacksdk<0.99 openshift jmespath google-cloud-storage pywinrm boto3 azure-storage-blob ibm-cos-sdk netaddr passlib -[2.8] -# Ansible 2.8.16 breaks the k8s connection plugin -# Jinja 3.1.1 is incompatible with 2.8 -requirements = ansible>=2.8,<2.9,!=2.8.16 Jinja2<3.1.0 -deprecated = true - -[2.9] -# Ansible 2.9.14 breaks the k8s connection plugin -# https://github.com/ansible/ansible/issues/72171 -requirements = ansible>=2.9,<2.10,!=2.9.14 -deprecated = true - [5] requirements = ansible>=5.0,<6.0 diff --git a/zuul/lib/tracing.py b/zuul/lib/tracing.py new file mode 100644 index 000000000..42b2681f3 --- /dev/null +++ b/zuul/lib/tracing.py @@ -0,0 +1,291 @@ +# Copyright 2022 Acme Gating, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import grpc +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import \ + OTLPSpanExporter as GRPCExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import \ + OTLPSpanExporter as HTTPExporter +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider, Span +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry import trace +from opentelemetry.sdk import trace as trace_sdk + +from zuul.lib.config import get_default, any_to_bool + + +class ZuulSpan(Span): + """An implementation of Span which accepts floating point + times and converts them to the expected nanoseconds.""" + + def start(self, start_time=None, parent_context=None): + if isinstance(start_time, float): + start_time = int(start_time * (10**9)) + return super().start(start_time, parent_context) + + def end(self, end_time=None): + if isinstance(end_time, float): + end_time = int(end_time * (10**9)) + return super().end(end_time) + + +# Patch the OpenTelemetry SDK Span class to return a ZuulSpan so that +# we can supply floating point timestamps. +trace_sdk._Span = ZuulSpan + + +def _formatContext(context): + return { + 'trace_id': context.trace_id, + 'span_id': context.span_id, + } + + +def _formatAttributes(attrs): + if attrs is None: + return None + return attrs.copy() + + +def getSpanInfo(span, include_attributes=False): + """Return a dict for use in serializing a Span.""" + links = [{'context': _formatContext(l.context), + 'attributes': _formatAttributes(l.attributes)} + for l in span.links] + attrs = _formatAttributes(span.attributes) + context = span.get_span_context() + ret = { + 'name': span.name, + 'trace_id': context.trace_id, + 'span_id': context.span_id, + 'trace_flags': context.trace_flags, + 'start_time': span.start_time, + } + if links: + ret['links'] = links + if attrs: + if not include_attributes: + # Avoid setting attributes when we start saved spans + # because we have to store them in ZooKeeper and we should + # minimize what we store there (especially since it is + # usually duplicative). If you really need to set + # attributes at the start of a span (because the info is + # not available later), set include_attributes to True. + # Otherwise, we raise an error here to remind ourselves to + # avoid that programming pattern. + raise RuntimeError("Attributes were set on a saved span; " + "either set them when ending the span, " + "or set include_attributes=True") + ret['attributes'] = attrs + return ret + + +def restoreSpan(span_info, is_remote=True): + """Restore a Span from the serialized dict provided by getSpanInfo + + Return None if unable to serialize the span. + """ + tracer = trace.get_tracer("zuul") + if span_info is None: + return trace.INVALID_SPAN + required_keys = {'name', 'trace_id', 'span_id', 'trace_flags'} + if not required_keys <= set(span_info.keys()): + return trace.INVALID_SPAN + span_context = trace.SpanContext( + span_info['trace_id'], + span_info['span_id'], + is_remote=is_remote, + trace_flags=trace.TraceFlags(span_info['trace_flags']), + ) + links = [] + for link_info in span_info.get('links', []): + link_context = trace.SpanContext( + link_info['context']['trace_id'], + link_info['context']['span_id']) + link = trace.Link(link_context, link_info['attributes']) + links.append(link) + attributes = span_info.get('attributes', {}) + + span = ZuulSpan( + name=span_info['name'], + context=span_context, + parent=None, + sampler=tracer.sampler, + resource=tracer.resource, + attributes=attributes, + span_processor=tracer.span_processor, + kind=trace.SpanKind.INTERNAL, + links=links, + instrumentation_info=tracer.instrumentation_info, + record_exception=False, + set_status_on_exception=True, + limits=tracer._span_limits, + instrumentation_scope=tracer._instrumentation_scope, + ) + span.start(start_time=span_info['start_time']) + + return span + + +def startSavedSpan(*args, **kw): + """Start a span and serialize it + + This is a convenience method which starts a span (either root + or child) and immediately serializes it. + + Most spans in Zuul should use this method. + """ + tracer = trace.get_tracer("zuul") + include_attributes = kw.pop('include_attributes', False) + span = tracer.start_span(*args, **kw) + return getSpanInfo(span, include_attributes) + + +def endSavedSpan(span_info, end_time=None, attributes=None): + """End a saved span. + + This is a convenience method to restore a saved span and + immediately end it. + + Most spans in Zuul should use this method. + """ + span = restoreSpan(span_info, is_remote=False) + if span: + if attributes: + span.set_attributes(attributes) + span.end(end_time=end_time) + + +def getSpanContext(span): + """Return a dict for use in serializing a Span Context. + + The span context information used here is a lightweight + encoding of the span information so that remote child spans + can be started without access to a fully restored parent. + This is equivalent to (but not the same format) as the + OpenTelemetry trace context propogator. + """ + context = span.get_span_context() + return { + 'trace_id': context.trace_id, + 'span_id': context.span_id, + 'trace_flags': context.trace_flags, + } + + +def restoreSpanContext(span_context): + """Return a span with remote context information from getSpanContext. + + This returns a non-recording span to use as a parent span. It + avoids the necessity of fully restoring the parent span. + """ + if span_context: + span_context = trace.SpanContext( + trace_id=span_context['trace_id'], + span_id=span_context['span_id'], + is_remote=True, + trace_flags=trace.TraceFlags(span_context['trace_flags']) + ) + else: + span_context = trace.INVALID_SPAN_CONTEXT + parent = trace.NonRecordingSpan(span_context) + return parent + + +def startSpanInContext(span_context, *args, **kw): + """Start a span in a saved context. + + This restores a span from a saved context and starts a new child span. + """ + tracer = trace.get_tracer("zuul") + parent = restoreSpanContext(span_context) + with trace.use_span(parent): + return tracer.start_span(*args, **kw) + + +class Tracing: + PROTOCOL_GRPC = 'grpc' + PROTOCOL_HTTP_PROTOBUF = 'http/protobuf' + processor_class = BatchSpanProcessor + + def __init__(self, config): + service_name = get_default(config, "tracing", "service_name", "zuul") + resource = Resource(attributes={SERVICE_NAME: service_name}) + provider = TracerProvider(resource=resource) + trace.set_tracer_provider(provider) + enabled = get_default(config, "tracing", "enabled") + if not any_to_bool(enabled): + self.processor = None + return + + protocol = get_default(config, "tracing", "protocol", + self.PROTOCOL_GRPC) + endpoint = get_default(config, "tracing", "endpoint") + tls_key = get_default(config, "tracing", "tls_key") + tls_cert = get_default(config, "tracing", "tls_cert") + tls_ca = get_default(config, "tracing", "tls_ca") + certificate_file = get_default(config, "tracing", "certificate_file") + insecure = get_default(config, "tracing", "insecure") + if insecure is not None: + insecure = any_to_bool(insecure) + timeout = get_default(config, "tracing", "timeout") + if timeout is not None: + timeout = int(timeout) + compression = get_default(config, "tracing", "compression") + + if protocol == self.PROTOCOL_GRPC: + if certificate_file: + raise Exception("The certificate_file tracing option " + f"is not valid for {protocol} endpoints") + if any([tls_ca, tls_key, tls_cert]): + if tls_ca: + tls_ca = open(tls_ca, 'rb').read() + if tls_key: + tls_key = open(tls_key, 'rb').read() + if tls_cert: + tls_cert = open(tls_cert, 'rb').read() + creds = grpc.ssl_channel_credentials( + root_certificates=tls_ca, + private_key=tls_key, + certificate_chain=tls_cert) + else: + creds = None + exporter = GRPCExporter( + endpoint=endpoint, + insecure=insecure, + credentials=creds, + timeout=timeout, + compression=compression) + elif protocol == self.PROTOCOL_HTTP_PROTOBUF: + if insecure: + raise Exception("The insecure tracing option " + f"is not valid for {protocol} endpoints") + if any([tls_ca, tls_key, tls_cert]): + raise Exception("The tls_* tracing options " + f"are not valid for {protocol} endpoints") + exporter = HTTPExporter( + endpoint=endpoint, + certificate_file=certificate_file, + timeout=timeout, + compression=compression) + else: + raise Exception(f"Unknown tracing protocol {protocol}") + self.processor = self.processor_class(exporter) + provider.add_span_processor(self.processor) + + def stop(self): + if not self.processor: + return + self.processor.shutdown() diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py index 365435f3d..cd9b2381d 100644 --- a/zuul/manager/__init__.py +++ b/zuul/manager/__init__.py @@ -22,6 +22,7 @@ from zuul import model from zuul.lib.dependson import find_dependency_headers from zuul.lib.logutil import get_annotated_logger from zuul.lib.tarjan import strongly_connected_components +import zuul.lib.tracing as tracing from zuul.model import ( Change, DequeueEvent, PipelineState, PipelineChangeList, QueueItem, PipelinePostConfigEvent, @@ -30,6 +31,8 @@ from zuul.zk.change_cache import ChangeKey from zuul.zk.components import COMPONENT_REGISTRY from zuul.zk.locks import pipeline_lock +from opentelemetry import trace + class DynamicChangeQueueContextManager(object): def __init__(self, change_queue): @@ -579,7 +582,13 @@ class PipelineManager(metaclass=ABCMeta): log.info("Adding change %s to queue %s in %s" % (change, change_queue, self.pipeline)) - item = change_queue.enqueueChange(change, event) + if enqueue_time is None: + enqueue_time = time.time() + span_info = tracing.startSavedSpan( + 'QueueItem', start_time=enqueue_time) + item = change_queue.enqueueChange(change, event, + span_info=span_info, + enqueue_time=enqueue_time) self.updateBundle(item, change_queue, cycle) with item.activeContext(self.current_context): @@ -739,6 +748,15 @@ class PipelineManager(metaclass=ABCMeta): self.reportDequeue(item) item.queue.dequeueItem(item) + span_attrs = { + 'zuul_event_id': item.event.zuul_event_id, + } + for k, v in item.change.getSafeAttributes().toDict().items(): + span_attrs['ref_' + k] = v + tracing.endSavedSpan(item.current_build_set.span_info) + tracing.endSavedSpan(item.span_info, + attributes=span_attrs) + def removeItem(self, item): log = get_annotated_logger(self.log, item.event) # Remove an item from the queue, probably because it has been @@ -964,6 +982,7 @@ class PipelineManager(metaclass=ABCMeta): self.reportNormalBuildsetEnd( item.current_build_set, 'dequeue', final=False, result='DEQUEUED') + tracing.endSavedSpan(item.current_build_set.span_info) item.resetAllBuilds() for item_behind in item.items_behind: @@ -1337,7 +1356,9 @@ class PipelineManager(metaclass=ABCMeta): # isn't already set. tpc = tenant.project_configs.get(item.change.project.canonical_name) if not build_set.ref: - build_set.setConfiguration(self.current_context) + with trace.use_span(tracing.restoreSpan(item.span_info)): + span_info = tracing.startSavedSpan('BuildSet') + build_set.setConfiguration(self.current_context, span_info) # Next, if a change ahead has a broken config, then so does # this one. Record that and don't do anything else. diff --git a/zuul/model.py b/zuul/model.py index f66a5e875..254556fda 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -329,6 +329,9 @@ class Attributes(object): def __init__(self, **kw): setattr(self, '__dict__', kw) + def toDict(self): + return self.__dict__ + class Freezable(object): """A mix-in class so that an object can be made immutable""" @@ -659,6 +662,13 @@ class PipelineState(zkobject.ZKObject): safe_pipeline = urllib.parse.quote_plus(pipeline.name) return f"/zuul/tenant/{safe_tenant}/pipeline/{safe_pipeline}" + @classmethod + def parsePath(self, path): + """Return path components for use by the REST API""" + root, safe_tenant, pipeline, safe_pipeline = path.rsplit('/', 3) + return (urllib.parse.unquote_plus(safe_tenant), + urllib.parse.unquote_plus(safe_pipeline)) + def _dirtyPath(self): return f'{self.getPath()}/dirty' @@ -1087,13 +1097,16 @@ class ChangeQueue(zkobject.ZKObject): def matches(self, project_cname, branch): return (project_cname, branch) in self.project_branches - def enqueueChange(self, change, event): + def enqueueChange(self, change, event, span_info=None, enqueue_time=None): + if enqueue_time is None: + enqueue_time = time.time() item = QueueItem.new(self.zk_context, queue=self, pipeline=self.pipeline, change=change, event=event, - enqueue_time=time.time()) + span_info=span_info, + enqueue_time=enqueue_time) self.enqueueItem(item) return item @@ -3875,6 +3888,7 @@ class BuildSet(zkobject.ZKObject): tries={}, files_state=self.NEW, repo_state_state=self.NEW, + span_info=None, configured=False, configured_time=None, # When setConfigured was called start_time=None, # When the buildset reported start @@ -3951,6 +3965,13 @@ class BuildSet(zkobject.ZKObject): def getPath(self): return f"{self.item.getPath()}/buildset/{self.uuid}" + @classmethod + def parsePath(self, path): + """Return path components for use by the REST API""" + item_path, bs, uuid = path.rsplit('/', 2) + tenant, pipeline, item_uuid = QueueItem.parsePath(item_path) + return (tenant, pipeline, item_uuid, uuid) + def serialize(self, context): data = { # "item": self.item, @@ -3983,6 +4004,7 @@ class BuildSet(zkobject.ZKObject): "fail_fast": self.fail_fast, "job_graph": (self.job_graph.toDict() if self.job_graph else None), + "span_info": self.span_info, "configured_time": self.configured_time, "start_time": self.start_time, "repo_state_request_time": self.repo_state_request_time, @@ -4133,7 +4155,7 @@ class BuildSet(zkobject.ZKObject): len(self.builds), self.getStateName(self.merge_state)) - def setConfiguration(self, context): + def setConfiguration(self, context, span_info): with self.activeContext(context): # The change isn't enqueued until after it's created # so we don't know what the other changes ahead will be @@ -4153,6 +4175,7 @@ class BuildSet(zkobject.ZKObject): self.merger_items = [i.makeMergerItem() for i in items] self.configured = True self.configured_time = time.time() + self.span_info = span_info def _toChangeDict(self, item): # Inject bundle_id to dict if available, this can be used to decide @@ -4322,6 +4345,7 @@ class QueueItem(zkobject.ZKObject): current_build_set=None, item_ahead=None, items_behind=[], + span_info=None, enqueue_time=None, report_time=None, dequeue_time=None, @@ -4365,6 +4389,13 @@ class QueueItem(zkobject.ZKObject): def itemPath(cls, pipeline_path, item_uuid): return f"{pipeline_path}/item/{item_uuid}" + @classmethod + def parsePath(self, path): + """Return path components for use by the REST API""" + pipeline_path, item, uuid = path.rsplit('/', 2) + tenant, pipeline = PipelineState.parsePath(pipeline_path) + return (tenant, pipeline, uuid) + def serialize(self, context): if isinstance(self.event, TriggerEvent): event_type = "TriggerEvent" @@ -4384,6 +4415,7 @@ class QueueItem(zkobject.ZKObject): self.current_build_set.getPath()), "item_ahead": self.item_ahead and self.item_ahead.getPath(), "items_behind": [i.getPath() for i in self.items_behind], + "span_info": self.span_info, "enqueue_time": self.enqueue_time, "report_time": self.report_time, "dequeue_time": self.dequeue_time, diff --git a/zuul/scheduler.py b/zuul/scheduler.py index dfc922cf1..134b9d35a 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -42,6 +42,7 @@ from zuul.lib.monitoring import MonitoringServer from zuul.lib.queue import NamedQueue from zuul.lib.times import Times from zuul.lib.statsd import get_statsd, normalize_statsd_name +from zuul.lib.tracing import Tracing import zuul.lib.queue import zuul.lib.repl from zuul import nodepool @@ -190,6 +191,7 @@ class Scheduler(threading.Thread): self.daemon = True self.wait_for_init = wait_for_init self.hostname = socket.getfqdn() + self.tracing = Tracing(config) self.primed_event = threading.Event() # Wake up the main run loop self.wake_event = threading.Event() @@ -383,7 +385,10 @@ class Scheduler(threading.Thread): self.log.debug("Stopping monitoring server") self.monitoring_server.stop() self.monitoring_server.join() + self.log.debug("Disconnecting from ZooKeeper") self.zk_client.disconnect() + self.log.debug("Stopping tracing") + self.tracing.stop() def runCommand(self): while self._command_running: diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py index 644b82bec..c2b6a4ddf 100755 --- a/zuul/web/__init__.py +++ b/zuul/web/__init__.py @@ -47,6 +47,7 @@ from zuul.lib.monitoring import MonitoringServer from zuul.lib.re2util import filter_allowed_disallowed from zuul.model import ( Abide, + BuildSet, Branch, ChangeQueue, DequeueEvent, @@ -796,6 +797,7 @@ class ZuulWebAPI(object): 'project/{project:.*}/branch/{branch:.*}/' 'freeze-jobs', 'pipelines': '/api/tenant/{tenant}/pipelines', + 'semaphores': '/api/tenant/{tenant}/semaphores', 'labels': '/api/tenant/{tenant}/labels', 'nodes': '/api/tenant/{tenant}/nodes', 'key': '/api/tenant/{tenant}/key/{project:.*}.pub', @@ -1535,6 +1537,44 @@ class ZuulWebAPI(object): @cherrypy.expose @cherrypy.tools.save_params() + @cherrypy.tools.json_out( + content_type='application/json; charset=utf-8', handler=json_handler, + ) + def semaphores(self, tenant_name): + tenant = self._getTenantOrRaise(tenant_name) + result = [] + names = set(tenant.layout.semaphores.keys()) + names = names.union(tenant.global_semaphores) + for semaphore_name in sorted(names): + semaphore = tenant.layout.getSemaphore( + self.zuulweb.abide, semaphore_name) + holders = tenant.semaphore_handler.semaphoreHolders(semaphore_name) + this_tenant = [] + other_tenants = 0 + for holder in holders: + (holder_tenant, holder_pipeline, + holder_item_uuid, holder_buildset_uuid + ) = BuildSet.parsePath(holder['buildset_path']) + if holder_tenant != tenant_name: + other_tenants += 1 + continue + this_tenant.append({'buildset_uuid': holder_buildset_uuid, + 'job_name': holder['job_name']}) + sem_out = {'name': semaphore.name, + 'global': semaphore.global_scope, + 'max': semaphore.max, + 'holders': { + 'count': len(this_tenant) + other_tenants, + 'this_tenant': this_tenant, + 'other_tenants': other_tenants}, + } + result.append(sem_out) + resp = cherrypy.response + resp.headers['Access-Control-Allow-Origin'] = '*' + return result + + @cherrypy.expose + @cherrypy.tools.save_params() @cherrypy.tools.websocket(handler_cls=LogStreamHandler) def console_stream(self, tenant): cherrypy.request.ws_handler.zuulweb = self.zuulweb @@ -1864,6 +1904,8 @@ class ZuulWeb(object): controller=api, action='status') route_map.connect('api', '/api/tenant/{tenant}/status/change/{change}', controller=api, action='status_change') + route_map.connect('api', '/api/tenant/{tenant_name}/semaphores', + controller=api, action='semaphores') route_map.connect('api', '/api/tenant/{tenant_name}/jobs', controller=api, action='jobs') route_map.connect('api', '/api/tenant/{tenant_name}/job/{job_name}', diff --git a/zuul/zk/semaphore.py b/zuul/zk/semaphore.py index 721a0438a..b45612c04 100644 --- a/zuul/zk/semaphore.py +++ b/zuul/zk/semaphore.py @@ -41,8 +41,12 @@ class SemaphoreHandler(ZooKeeperSimpleBase): semaphore_root = "/zuul/semaphores" global_semaphore_root = "/zuul/global-semaphores" - def __init__(self, client, statsd, tenant_name, layout, abide): + def __init__(self, client, statsd, tenant_name, layout, abide, + read_only=False): super().__init__(client) + if read_only: + statsd = None + self.read_only = read_only self.abide = abide self.layout = layout self.statsd = statsd @@ -71,6 +75,8 @@ class SemaphoreHandler(ZooKeeperSimpleBase): self.log.exception("Unable to send semaphore stats:") def acquire(self, item, job, request_resources): + if self.read_only: + raise RuntimeError("Read-only semaphore handler") if not job.semaphores: return True @@ -190,6 +196,8 @@ class SemaphoreHandler(ZooKeeperSimpleBase): break def release(self, sched, item, job, quiet=False): + if self.read_only: + raise RuntimeError("Read-only semaphore handler") if not job.semaphores: return @@ -242,6 +250,8 @@ class SemaphoreHandler(ZooKeeperSimpleBase): return 1 if semaphore is None else semaphore.max def cleanupLeaks(self): + if self.read_only: + raise RuntimeError("Read-only semaphore handler") # MODEL_API: >1 if COMPONENT_REGISTRY.model_api < 2: self.log.warning("Skipping semaphore cleanup since minimum model " |