diff options
129 files changed, 2274 insertions, 544 deletions
diff --git a/.zuul.yaml b/.zuul.yaml index 72684c460..9d9000756 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -55,6 +55,12 @@ zuul_ansible_version: 5 - job: + name: zuul-stream-functional-6 + parent: zuul-stream-functional + vars: + zuul_ansible_version: 6 + +- job: name: zuul-tox description: | Zuul unit tests with ZooKeeper running @@ -320,6 +326,7 @@ - zuul-stream-functional-2.8 - zuul-stream-functional-2.9 - zuul-stream-functional-5 + - zuul-stream-functional-6 - zuul-tox-remote - zuul-quick-start: requires: nodepool-container-image @@ -350,6 +357,7 @@ - zuul-stream-functional-2.8 - zuul-stream-functional-2.9 - zuul-stream-functional-5 + - zuul-stream-functional-6 - zuul-tox-remote - zuul-quick-start: requires: nodepool-container-image diff --git a/Dockerfile b/Dockerfile index 0c1118ccb..c6cc17651 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,13 +69,7 @@ COPY --from=builder /usr/local/lib/zuul/ /usr/local/lib/zuul COPY --from=builder /tmp/openshift-install/kubectl /usr/local/bin/kubectl COPY --from=builder /tmp/openshift-install/oc /usr/local/bin/oc -# We need libc >= 2.33 due to -# https://github.com/ansible/ansible/issues/78270 -RUN echo 'APT::Default-Release "stable";' >> /etc/apt/apt.conf.d/99defaultrelease -RUN echo "deb http://deb.debian.org/debian/ bookworm main contrib non-free" > /etc/apt/sources.list.d/testing.list - RUN apt-get update \ - && apt-get -t bookworm install -y libc-bin \ && apt-get install -y skopeo \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/doc/source/config/nodeset.rst b/doc/source/config/nodeset.rst index 84c413c9b..7c1ebbbd2 100644 --- a/doc/source/config/nodeset.rst +++ b/doc/source/config/nodeset.rst @@ -40,6 +40,39 @@ branch will not immediately produce a configuration error. nodes: - web +Nodesets may also be used to express that Zuul should use the first of +multiple alternative node configurations to run a job. When a Nodeset +specifies a list of :attr:`nodeset.alternatives`, Zuul will request the +first Nodeset in the series, and if allocation fails for any reason, +Zuul will re-attempt the request with the subsequent Nodeset and so +on. The first Nodeset which is sucessfully supplied by Nodepool will +be used to run the job. An example of such a configuration follows. + +.. code-block:: yaml + + - nodeset: + name: fast-nodeset + nodes: + - label: fast-label + name: controller + + - nodeset: + name: slow-nodeset + nodes: + - label: slow-label + name: controller + + - nodeset: + name: fast-or-slow + alternatives: + - fast-nodeset + - slow-nodeset + +In the above example, a job that requested the `fast-or-slow` nodeset +would receive `fast-label` nodes if a provider was able to supply +them, otherwise it would receive `slow-label` nodes. A Nodeset may +specify nodes and groups, or alternative nodesets, but not both. + .. attr:: nodeset A Nodeset requires two attributes: @@ -54,7 +87,8 @@ branch will not immediately produce a configuration error. definition, this attribute should be omitted. .. attr:: nodes - :required: + + This attribute is required unless `alteranatives` is supplied. A list of node definitions, each of which has the following format: @@ -89,3 +123,23 @@ branch will not immediately produce a configuration error. The nodes that shall be part of the group. This is specified as a list of strings. + .. attr:: alternatives + :type: list + + A list of alternative nodesets for which requests should be + attempted in series. The first request which succeeds will be + used for the job. + + The items in the list may be either strings, in which case they + refer to other Nodesets within the layout, or they may be a + dictionary which is a nested anonymous Nodeset definition. The + two types (strings or nested definitions) may be mixed. + + An alternative Nodeset definition may in turn refer to other + alternative nodeset definitions. In this case, the tree of + definitions will be flattened in a breadth-first manner to + create the ordered list of alternatives. + + A Nodeset which specifies alternatives may not also specify + nodes or groups (this attribute is exclusive with + :attr:`nodeset.nodes` and :attr:`nodeset.groups`. diff --git a/doc/source/config/pipeline.rst b/doc/source/config/pipeline.rst index f1c294775..f4d7cce69 100644 --- a/doc/source/config/pipeline.rst +++ b/doc/source/config/pipeline.rst @@ -332,9 +332,16 @@ success, the pipeline reports back to Gerrit with ``Verified`` vote of .. attr:: merge-conflict These reporters describe what Zuul should do if it is unable to - merge in the patchset. If no merge-conflict reporters are listed - then the ``failure`` reporters will be used to notify of - unsuccessful merges. + merge the patchset into the current state of the target + branch. If no merge-conflict reporters are listed then the + ``failure`` reporters will be used. + + .. attr:: config-error + + These reporters describe what Zuul should do if it encounters a + configuration error while trying to enqueue the item. If no + config-error reporters are listed then the ``failure`` reporters + will be used. .. attr:: enqueue diff --git a/doc/source/config/project.rst b/doc/source/config/project.rst index 301e0b08d..1aa570f41 100644 --- a/doc/source/config/project.rst +++ b/doc/source/config/project.rst @@ -187,16 +187,6 @@ pipeline. :attr:`job` definition. Any attributes set on the job here will override previous versions of the job. - .. attr:: queue - - This is the same as :attr:`project.queue` but on per pipeline - level for backwards compatibility reasons. If :attr:`project.queue` - is defined this setting is ignored. - - .. note:: It is deprecated to define the queue in the pipeline - configuration. Configure it on :attr:`project.queue` - instead. - .. attr:: debug If this is set to `true`, Zuul will include debugging diff --git a/doc/source/developer/ansible.rst b/doc/source/developer/ansible.rst index 415c47df7..c3135debe 100644 --- a/doc/source/developer/ansible.rst +++ b/doc/source/developer/ansible.rst @@ -4,19 +4,11 @@ Ansible Integration Zuul contains Ansible modules and plugins to control the execution of Ansible Job content. -Build Log Support ------------------ +Zuul provides realtime build log streaming to end users so that users +can watch long-running jobs in progress. -Zuul provides realtime build log streaming to end users so that users can -watch long-running jobs in progress. As jobs may be written that execute a -shell script that could run for a long time, additional effort is expended -to stream stdout and stderr of shell tasks as they happen rather than waiting -for the command to finish. - -Zuul contains a modified version of the :ansible:module:`command` -that starts a log streaming daemon on the build node. - -.. automodule:: zuul.ansible.base.library.command +Streaming job output +-------------------- All jobs run with the :py:mod:`zuul.ansible.base.callback.zuul_stream` callback plugin enabled, which writes the build log to a file so that the @@ -35,10 +27,55 @@ exposes that log stream over a websocket connection as part of In addition to real-time streaming, Zuul also installs another callback module, :py:mod:`zuul.ansible.base.callback.zuul_json.CallbackModule` that collects all of the information about a given run into a json file which is written to the -work dir so that it can be published along with build logs. Since the streaming -log is by necessity a single text stream, choices have to be made for -readability about what data is shown and what is not shown. The json log file -is intended to allow for a richer more interactive set of data to be displayed -to the user. +work dir so that it can be published along with build logs. .. autoclass:: zuul.ansible.base.callback.zuul_json.CallbackModule + +Since the streaming log is by necessity a single text stream, choices +have to be made for readability about what data is shown and what is +not shown. The json log file is intended to allow for a richer more +interactive set of data to be displayed to the user. + +.. _zuul_console_streaming: + +Capturing live command output +----------------------------- + +As jobs may execute long-running shell scripts or other commands, +additional effort is expended to stream ``stdout`` and ``stderr`` of +shell tasks as they happen rather than waiting for the command to +finish. + +The global job configuration should run the ``zuul_console`` task as a +very early prerequisite step. + +.. automodule:: zuul.ansible.base.library.zuul_console + +This will start a daemon that listens on TCP port 19885 on the testing +node. This daemon can be queried to stream back the output of shell +tasks as described below. + +Zuul contains a modified version of Ansible's +:ansible:module:`command` module that overrides the default +implementation. + +.. automodule:: zuul.ansible.base.library.command + +This library will capture the output of the running +command and write it to a temporary file on the host the command is +running on. These files are named in the format +``/tmp/console-<uuid>-<task_id>-<host>.log`` + +The ``zuul_stream`` callback mentioned above will send a request to +the remote ``zuul_console`` daemon, providing the uuid and task id of +the task it is currently processing. The ``zuul_console`` daemon will +then read the logfile from disk and stream the data back as it +appears, which ``zuul_stream`` will then present as described above. + +The ``zuul_stream`` callback will indicate to the ``zuul_console`` +daemon when it has finished reading the task, which prompts the remote +side to remove the temporary streaming output files. In some cases, +aborting the Ansible process may not give the ``zuul_stream`` callback +the chance to send this notice, leaking the temporary files. If nodes +are ephemeral this makes little difference, but these files may be +visible on static nodes. diff --git a/doc/source/developer/model-changelog.rst b/doc/source/developer/model-changelog.rst index 0d4cb5077..d27ec8351 100644 --- a/doc/source/developer/model-changelog.rst +++ b/doc/source/developer/model-changelog.rst @@ -86,3 +86,10 @@ Version 8 :Prior Zuul version: 6.0.0 :Description: Deduplicates jobs in dependency cycles. Affects schedulers only. + +Version 9 +--------- + +:Prior Zuul version: 6.3.0 +:Description: Adds nodeset_alternatives and nodeset_index to frozen job. + Removes nodset from frozen job. Affects schedulers and executors. diff --git a/doc/source/developer/specs/index.rst b/doc/source/developer/specs/index.rst index 78c11bbb8..a75084429 100644 --- a/doc/source/developer/specs/index.rst +++ b/doc/source/developer/specs/index.rst @@ -16,11 +16,12 @@ documentation instead. .. toctree:: :maxdepth: 1 - tenant-scoped-admin-web-API - kubernetes-operator circular-dependencies - zuul-runner + community-matrix enhanced-regional-executors + kubernetes-operator + nodepool-in-zuul tenant-resource-quota - community-matrix + tenant-scoped-admin-web-API tracing + zuul-runner diff --git a/doc/source/developer/specs/nodepool-in-zuul.rst b/doc/source/developer/specs/nodepool-in-zuul.rst new file mode 100644 index 000000000..10a3ad7dc --- /dev/null +++ b/doc/source/developer/specs/nodepool-in-zuul.rst @@ -0,0 +1,743 @@ +Nodepool in Zuul +================ + +.. warning:: This is not authoritative documentation. These features + are not currently available in Zuul. They may change significantly + before final implementation, or may never be fully completed. + +The following specification describes a plan to move Nodepool's +functionality into Zuul and end development of Nodepool as a separate +application. This will allow for more node and image related features +as well as simpler maintenance and deployment. + +Introduction +------------ + +Nodepool exists as a distinct application from Zuul largely due to +historical circumstances: it was originally a process for launching +nodes, attaching them to Jenkins, detaching them from Jenkins and +deleting them. Once Zuul grew its own execution engine, Nodepool +could have been adopted into Zuul at that point, but the existing +loose API meant it was easy to maintain them separately and combining +them wasn't particularly advantageous. + +However, now we find ourselves with a very robust framework in Zuul +for dealing with ZooKeeper, multiple components, web services and REST +APIs. All of these are lagging behind in Nodepool, and it is time to +address that one way or another. We could of course upgrade +Nodepool's infrastructure to match Zuul's, or even separate out these +frameworks into third-party libraries. However, there are other +reasons to consider tighter coupling between Zuul and Nodepool, and +these tilt the scales in favor of moving Nodepool functionality into +Zuul. + +Designing Nodepool as part of Zuul would allow for more features +related to Zuul's multi-tenancy. Zuul is quite good at +fault-tolerance as well as scaling, so designing Nodepool around that +could allow for better cooperation between node launchers. Finally, +as part of Zuul, Nodepool's image lifecycle can be more easily +integrated with Zuul-based workflow. + +There are two Nodepool components: nodepool-builder and +nodepool-launcher. We will address the functionality of each in the +following sections on Image Management and Node Management. + +This spec contemplates a new Zuul component to handle image and node +management: zuul-launcher. Much of the Nodepool configuration will +become Zuul configuration as well. That is detailed in its own +section, but for now, it's enough to know that the Zuul system as a +whole will know what images and node labels are present in the +configuration. + +Image Management +---------------- + +Part of nodepool-builder's functionality is important to have as a +long-running daemon, and part of what it does would make more sense as +a Zuul job. By moving the actual image build into a Zuul job, we can +make the activity more visible to users of the system. It will be +easier for users to test changes to image builds (inasmuch as they can +propose a change and a check job can run on that change to see if the +image builds sucessfully). Build history and logs will be visible in +the usual way in the Zuul web interface. + +A frequently requested feature is the ability to verify images before +putting them into service. This is not practical with the current +implementation of Nodepool because of the loose coupling with Zuul. +However, once we are able to include Zuul jobs in the workflow of +image builds, it is easier to incorporate Zuul jobs to validate those +images as well. This spec includes a mechanism for that. + +The parts of nodepool-builder that makes sense as a long-running +daemon are the parts dealing with image lifecycles. Uploading builds +to cloud providers, keeping track of image builds and uploads, +deciding when those images should enter or leave service, and deleting +them are all better done with state management and long-running +processes (we should know -- early versions of Nodepool attempted to +do all of that with Jenkins jobs with limited success). + +The sections below describe how we will implement image management in +Zuul. + +First, a reminder that using custom images is optional with Zuul. +Many Zuul systems will be able to operate using only stock cloud +provider images. One of the strengths of nodepool-builder is that it +can build an image for Zuul without relying on any particular cloud +provider images. A Zuul system whose operator wants to use custom +images will need to bootstrap that process, and under the proposed +system where images are build in Zuul jobs, that would need to be done +using a stock cloud image. In other words, to bootstrap a system such +as OpenDev from scratch, the operators would need to use a stock cloud +image to run the job to build the custom image. Once a custom image +is available, further image builds could be run on either the stock +cloud image or the custom image. That decision is left to the +operator and involves consideration of fault tolerance and disaster +recovery scenarios. + +To build a custom image, an operator will define a fairly typical Zuul +job for each image they would like to produce. For example, a system +may have one job to build a debian-stable image, a second job for +debian-unstable, a third job for ubuntu-focal, a fourth job for +ubuntu-jammy. Zuul's job inheritance system could be very useful here +to deal with many variations of a similar process. + +Currently nodepool-builder will build an image under three +circumstances: 1) the image (or the image in a particular format) is +missing; 2) a user has directly requested a build; 3) on an automatic +interval (typically daily). To map this into Zuul, we will use Zuul's +existing pipeline functionality, but we will add a new trigger for +case #1. Case #2 can be handled by a manual Zuul enqueue command, and +case #3 by a periodic pipeline trigger. + +Since Zuul knows what images are configured and what their current +states are, it will be able to emit trigger events when it detects +that a new image (or image format) has been added to its +configuration. In these cases, the `zuul` driver in Zuul will enqueue +an `image-build` trigger event on startup or reconfiguration for every +missing image. The event will include the image name. Pipelines will +be configured to trigger on `image-build` events as well as on a timer +trigger. + +Jobs will include an extra attribute to indicate they build a +particular image. This serves two purposes; first, in the case of an +`image-build` trigger event, it will act as a matcher so that only +jobs matching the image that needs building are run. Second, it will +allow Zuul to determine which formats are needed for that image (based +on which providers are configured to use it) and include that +information as job data. + +The job will be responsible for building the image and uploading the +result to some storage system. The URLs for each image format built +should be returned to Zuul as artifacts. + +Finally, the `zuul` driver reporter will accept parameters which will +tell it to search the result data for these artifact URLs and update +the internal image state accordingly. + +An example configuration for a simple single-stage image build: + +.. code-block:: yaml + + - pipeline: + name: image + trigger: + zuul: + events: + - image-build + timer: + time: 0 0 * * * + success: + zuul: + image-built: true + image-validated: true + + - job: + name: build-debian-unstable-image + image-build-name: debian-unstable + +This job would run whenever Zuul determines it needs a new +debian-unstable image or daily at midnight. Once the job completes, +because of the ``image-built: true`` report, it will look for artifact +data like this: + +.. code-block:: yaml + + artifacts: + - name: raw image + url: https://storage.example.com/new_image.raw + metadata: + type: zuul_image + image_name: debian-unstable + format: raw + - name: qcow2 image + url: https://storage.example.com/new_image.qcow2 + metadata: + type: zuul_image + image_name: debian-unstable + format: qcow2 + +Zuul will update internal records in ZooKeeper for the image to record +the storage URLs. The zuul-launcher process will then start +background processes to download the images from the storage system +and upload them to the configured providers (much as nodepool-builder +does now with files on disk). As a special case, it may detect that +the image files are stored in a location that a provider can access +directly for import and may be able to import directly from the +storage location rather than downloading locally first. + +To handle image validation, a flag will be stored for each image +upload indicating whether it has been validated. The example above +specifies ``image-validated: true`` and therefore Zuul will put the +image into service as soon as all image uploads are complete. +However, if it were false, then Zuul would emit an `image-validate` +event after each upload is complete. A second pipeline can be +configured to perform image validation. It can run any number of +jobs, and since Zuul has complete knowledge of image states, it will +supply nodes using the new image upload (which is not yet in service +for normal jobs). An example of this might look like: + +.. code-block:: yaml + + - pipeline: + name: image-validate + trigger: + zuul: + events: + - image-validate + success: + zuul: + image-validated: true + + - job: + name: validate-debian-unstable-image + image-build-name: debian-unstable + nodeset: + nodes: + - name: node + label: debian + +The label should specify the same image that is being validated. Its +node request will be made with extra specifications so that it is +fulfilled with a node built from the image under test. This process +may repeat for each of the providers using that image (normal pipeline +queue deduplication rules may need a special case to allow this). +Once the validation jobs pass, the entry in ZooKeeper will be updated +and the image will go into regular service. + +A more specific process definition follows: + +After a buildset reports with ``image-built: true``, Zuul will scan +result data and for each artifact it finds, it will create an entry in +ZooKeeper at `/zuul/images/<image_name>/<sequence>`. Zuul will know +not to emit any more `image-build` events for that image at this +point. + +For every provider using that image, Zuul will create an entry in +ZooKeeper at +`/zuul/image-uploads/<image_name>/<image_number>/provider/<provider_name>`. +It will set the remote image ID to null and the `image-validated` flag +to whatever was specified in the reporter. + +Whenever zuul-launcher observes a new `image-upload` record without an +ID, it will: + +* Lock the whole image +* Lock each upload it can handle +* Unlocks the image while retaining the upload locks +* Downloads artifact (if needed) and uploads images to provider +* If upload requires validation, it enqueues an `image-validate` zuul driver trigger event +* Unlocks upload + +The locking sequence is so that a single launcher can perform multiple +uploads from a single artifact download if it has the opportunity. + +Once more than two builds of an image are in service, the oldest is +deleted. The image ZooKeeper record set to the `deleting` state. +Zuul-launcher will delete the uploads from the providers. The `zuul` +driver emits an `image-delete` event with item data for the image +artifact. This will trigger an image-delete job that can delete the +artifact from the cloud storage. + +All of these pipeline definitions should typically be in a single +tenant (but need not be), but the images they build are potentially +available to each tenant that includes the image definition +configuration object (see the Configuration section below). Any repo +in a tenant with an image build pipeline will be able to cause images +to be built and uploaded to providers. + +Snapshot Images +~~~~~~~~~~~~~~~ + +Nodepool does not currently support snapshot images, but the spec for +the current version of Nodepool does contemplate the possibility of a +snapshot based nodepool-builder process. Likewise, this spec does not +require us to support snapshot image builds, but in case we want to +add support in the future, we should have a plan for it. + +The image build job in Zuul could, instead of running +diskimage-builder, act on the remote node to prepare it for a +snapshot. A special job attribute could indicate that it is a +snapshot image job, and instead of having the zuul-launcher component +delete the node at the end of the job, it could snapshot the node and +record that information in ZooKeeper. Unlike an image-build job, an +image-snapshot job would need to run in each provider (similar to how +it is proposed that an image-validate job will run in each provider). +An image-delete job would not be required. + + +Node Management +--------------- + +The techniques we have developed for cooperative processing in Zuul +can be applied to the node lifecycle. This is a good time to make a +significant change to the nodepool protocol. We can achieve several +long-standing goals: + +* Scaling and fault-tolerance: rather than having a 1:N relationship + of provider:nodepool-launcher, we can have multiple zuul-launcher + processes, each of which is capable of handling any number of + providers. + +* More intentional request fulfillment: almost no intelligence goes + into selecting which provider will fulfill a given node request; by + assigning providers intentionally, we can more efficiently utilize + providers. + +* Fulfilling node requests from multiple providers: by designing + zuul-launcher for cooperative work, we can have nodesets that + request nodes which are fulfilled by different providers. Generally + we should favor the same provider for a set of nodes (since they may + need to communicate over a LAN), but if that is not feasible, + allowing multiple providers to fulfill a request will permit + nodesets with diverse node types (e.g., VM + static, or VM + + container). + +Each zuul-launcher process will execute a number of processing loops +in series; first a global request processing loop, and then a +processing loop for each provider. Each one will involve obtaining a +ZooKeeper lock so that only one zuul-launcher process will perform +each function at a time. + +Zuul-launcher will need to know about every connection in the system +so that it may have a fuul copy of the configuration, but operators +may wish to localize launchers to specific clouds. To support this, +zuul-launcher will take an optional command-line argument to indicate +on which connections it should operate. + +Currently a node request as a whole may be declined by providers. We +will make that more granular and store information about each node in +the request (in other words, individual nodes may be declined by +providers). + +All drivers for providers should implement the state machine +interface. Any state machine information currently storen in memory +in nodepool-launcher will need to move to ZooKeeper so that other +launchers can resume state machine processing. + +The individual provider loop will: + +* Lock a provider in ZooKeeper (`/zuul/provider/<name>`) +* Iterate over every node assigned to that provider in a `building` state + + * Drive the state machine + * If success, update request + * If failure, determine if it's a temporary or permanent failure + and update the request accordingly + * If quota available, unpause provider (if paused) + +The global queue process will: + +* Lock the global queue +* Iterate over every pending node request, and every node within that request + + * If all providers have failed the request, clear all temp failures + * If all providers have permanently failed the request, return error + * Identify providers capable of fulfilling the request + * Assign nodes to any provider with sufficient quota + * If no providers with sufficient quota, assign it to first (highest + priority) provider that can fulfill it later and pause that + provider + +Configuration +------------- + +The configuration currently handled by Nodepool will be refactored and +added to Zuul's configuration syntax. It will be loaded directly from +git repos like most Zuul configuration, however it will be +non-speculative (like pipelines and semaphores -- changes must merge +before they take effect). + +Information about connecting to a cloud will be added to ``zuul.conf`` +as a ``connection`` entry. The rate limit setting will be moved to +the connection configuration. Providers will then reference these +connections by name. + +Because providers and images reference global (i.e., outside tenant +scope) concepts, ZooKeeper paths for data related to those should +include the canonical name of the repo where these objects are +defined. For example, a `debian-unstable` image in the +`opendev/images` repo should be stored at +``/zuul/zuul-images/opendev.org%2fopendev%2fimages/``. This avoids +collisions if different tenants contain different image objects with +the same name. + +The actual Zuul config objects will be tenant scoped. Image +definitions which should be available to a tenant should be included +in that tenant's config. Again using the OpenDev example, the +hypothetical `opendev/images` repository should be included in every +OpenDev tenant so all of those images are available. + +Within a tenant, image names must be unique (otherwise it is a tenant +configuration error, similar to a job name collision). + +The diskimage-builder related configuration items will no longer be +necessary since they will be encoded in Zuul jobs. This will reduce +the complexity of the configuration significantly. + +The provider configuration will change as we take the opportunity to +make it more "Zuul-like". Instead of a top-level dictionary, we will +use lists. We will standardize on attributes used across drivers +where possible, as well as attributes which may be located at +different levels of the configuration. + +The goals of this reorganization are: + +* Allow projects to manage their own image lifecycle (if permitted by + site administrators). +* Manage access control to labels, images and flavors via standard + Zuul mechanisms (whether an item appears within a tenant). +* Reduce repetition and boilerplate for systems with many clouds, + labels, or images. + +The new configuration objects are: + +Image + This represents any kind of image (A Zuul image built by a job + described above, or a cloud image). By using one object to + represent both, we open the possibility of having a label in one + provider use a cloud image and in another provider use a Zuul image + (because the label will reference the image by short-name which may + resolve to a different image object in different tenants). A given + image object will specify what type it is, and any relevant + information about it (such as the username to use, etc). + +Flavor + This is a new abstraction layer to reference instance types across + different cloud providers. Much like labels today, these probably + won't have much information associated with them other than to + reserve a name for other objects to reference. For example, a site + could define a `small` and a `large` flavor. These would later be + mapped to specific instance types on clouds. + +Label + Unlike the current Nodepool ``label`` definitions, these labels will + also specify the image and flavor to use. These reference the two + objects above, which means that labels themselves contain the + high-level definition of what will be provided (e.g., a `large + ubuntu` node) while the specific mapping of what `large` and + `ubuntu` mean are left to the more specific configuration levels. + +Section + This looks a lot like the current ``provider`` configuration in + Nodepool (but also a little bit like a ``pool``). Several parts of + the Nodepool configuration (such as separating out availability + zones from providers into pools) were added as an afterthought, and + we can take the opportunity to address that here. + + A ``section`` is part of a cloud. It might be a region (if a cloud + has regions). It might be one or more availability zones within a + region. A lot of the specifics about images, flavors, subnets, + etc., will be specified here. Because a cloud may have many + sections, we will implement inheritance among sections. + +Provider + This is mostly a mapping of labels to sections and is similar to a + provider pool in the current Nodepool configuration. It exists as a + separate object so that site administrators can restrict ``section`` + definitions to central repos and allow tenant administrators to + control their own image and labels by allowing certain projects to + define providers. + + It mostly consists of a list of labels, but may also include images. + +When launching a node, relevant attributes may come from several +sources (the pool, image, flavor, or provider). Not all attributes +make sense in all locations, but where we can support them in multiple +locations, the order of application (later items override earlier +ones) will be: + +* ``image`` stanza +* ``flavor`` stanza +* ``label`` stanza +* ``section`` stanza (top level) +* ``image`` within ``section`` +* ``flavor`` within ``section`` +* ``provider`` stanza (top level) +* ``label`` within ``provider`` + +This reflects that the configuration is built upwards from general and +simple objects toward more specific objects image, flavor, label, +section, provider. Generally speaking, inherited scalar values will +override, dicts will merge, lists will concatenate. + +An example configuration follows. First, some configuration which may +appear in a central project and shared among multiple tenants: + +.. code-block:: yaml + + # Images, flavors, and labels are the building blocks of the + # configuration. + + - image: + name: centos-7 + type: zuul + # Any other image-related info such as: + # username: ... + # python-path: ... + # shell-type: ... + # A default that can be overridden by a provider: + # config-drive: true + + - image: + name: ubuntu + type: cloud + + - flavor: + name: large + + - label: + name: centos-7 + min-ready: 1 + flavor: large + image: centos-7 + + - label: + name: ubuntu + flavor: small + image: ubuntu + + # A section for each cloud+region+az + + - section: + name: rax-base + abstract: true + connection: rackspace + boot-timeout: 120 + launch-timeout: 600 + key-name: infra-root-keys-2020-05-13 + # The launcher will apply the minimum of the quota reported by the + # driver (if available) or the values here. + quota: + instances: 2000 + subnet: some-subnet + tags: + section-info: foo + # We attach both kinds of images to providers in order to provide + # image-specific info (like config-drive) or username. + images: + - name: centos-7 + config-drive: true + # This is a Zuul image + - name: ubuntu + # This is a cloud image, so the specific cloud image name is required + image-name: ibm-ubuntu-20-04-3-minimal-amd64-1 + # Other information may be provided + # username ... + # python-path: ... + # shell-type: ... + flavors: + - name: small + cloud-flavor: "Performance 8G" + - name: large + cloud-flavor: "Performance 16G" + + - section: + name: rax-dfw + parent: rax-base + region: 'DFW' + availability-zones: ["a", "b"] + + # A provider to indicate what labels are available to a tenant from + # a section. + + - provider: + name: rax-dfw-main + section: rax-dfw + labels: + - name: centos-7 + - name: ubuntu + key-name: infra-root-keys-2020-05-13 + tags: + provider-info: bar + +The following configuration might appear in a repo that is only used +in a single tenant: + +.. code-block:: yaml + + - image: + name: devstack + type: zuul + + - label: + name: devstack + + - provider: + name: rax-dfw-devstack + section: rax-dfw + # The images can be attached to the provider just as a section. + image: + - name: devstack + config-drive: true + labels: + - name: devstack + +Here is a potential static node configuration: + +.. code-block:: yaml + + - label: + name: big-static-node + + - section: + name: static-nodes + connection: null + nodes: + - name: static.example.com + labels: + - big-static-node + host-key: ... + username: zuul + + - provider: + name: static-provider + section: static-nodes + labels: + - big-static-node + +Each of the the above stanzas may only appear once in a tenant for a +given name (like pipelines or semaphores, they are singleton objects). +If they appear in more than one branch of a project, the definitions +must be identical; otherwise, or if they appear in more than one repo, +the second definition is an error. These are meant to be used in +unbranched repos. Whatever tenants they appear in will be permitted +to access those respective resources. + +The purpose of the ``provider`` stanza is to associate labels, images, +and sections. Much of the configuration related to launching an +instance (including the availability of zuul or cloud images) may be +supplied in the ``provider`` stanza and will apply to any labels +within. The ``section`` stanza also allows configuration of the same +information except for the labels themselves. The ``section`` +supplies default values and the ``provider`` can override them or add +any missing values. Images are additive -- any images that appear in +a ``provider`` will augment those that appear in a ``section``. + +The result is a modular scheme for configuration, where a single +``section`` instance can be used to set as much information as +possible that applies globally to a provider. A simple configuration +may then have a single ``provider`` instance to attach labels to that +section. A more complex installation may define a "standard" pool +that is present in every tenant, and then tenant-specific pools as +well. These pools will all attach to the same section. + +References to sections, images and labels will be internally converted +to canonical repo names to avoid ambiguity. Under the current +Nodepool system, labels are truly a global object, but under this +proposal, a label short name in one tenant may be different than one +in another. Therefore the node request will internally specify the +canonical label name instead of the short name. Users will never use +canonical names, only short names. + +For static nodes, there is some repitition to labels: first labels +must be associated with the individual nodes defined on the section, +then the labels must appear again on a provider. This allows an +operator to define a collection of static nodes centrally on a +section, then include tenant-specific sets of labels in a provider. +For the simple case where all static node labels in a section should +be available in a provider, we could consider adding a flag to the +provider to allow that (e.g., ``include-all-node-labels: true``). +Static nodes themselves are configured on a section with a ``null`` +connection (since there is no cloud provider associated with static +nodes). In this case, the additional ``nodes`` section attribute +becomes available. + +Upgrade Process +--------------- + +Most users of diskimages will need to create new jobs to build these +images. This proposal also includes significant changes to the node +allocation system which come with operational risks. + +To make the transition as minimally disruptive as possible, we will +support both systems in Zuul, and allow for selection of one system or +the other on a per-label and per-tenant basis. + +By default, if a nodeset specifies a label that is not defined by a +``label`` object in the tenant, Zuul will use the old system and place +a ZooKeeper request in ``/nodepool``. If a matching ``label`` is +available in the tenant, The request will use the new system and be +sent to ``/zuul/node-requests``. Once a tenant has completely +converted, a configuration flag may be set in the tenant configuration +and that will allow Zuul to treat nodesets that reference unknown +labels as configuration errors. A later version of Zuul will remove +the backwards compatability and make this the standard behavior. + +Because each of the systems will have unique metadata, they will not +recognize each others nodes, and it will appear to each that another +system is using part of their quota. Nodepool is already designed to +handle this case (at least, handle it as well as possible). + +Library Requirements +-------------------- + +The new zuul-launcher component will need most of Nodepool's current +dependencies, which will entail adding many third-party cloud provider +interfaces. As of writing, this uses another 420M of disk space. +Since our primary method of distribution at this point is container +images, if the additional space is a concern, we could restrict the +installation of these dependencies to only the zuul-launcher image. + +Diskimage-Builder Testing +------------------------- + +The diskimage-builder project team has come to rely on Nodepool in its +testing process. It uses Nodepool to upload images to a devstack +cloud, launch nodes from those instances, and verify that they +function. To aid in continuity of testing in the diskimage-builder +project, we will extract the OpenStack image upload and node launching +code into a simple Python script that can be used in diskimage-builder +test jobs in place of Nodepool. + +Work Items +---------- + +* In existing Nodepool convert the following drivers to statemachine: + gce, kubernetes, openshift, openshift, openstack (openstack is the + only one likely to require substantial effort, the others should be + trivial) +* Replace Nodepool with an image upload script in diskimage-builder + test jobs +* Add roles to zuul-jobs to build images using diskimage-builder +* Implement node-related config items in Zuul config and Layout +* Create zuul-launcher executable/component +* Add image-name item data +* Add image-build-name attribute to jobs + * Including job matcher based on item image-name + * Include image format information based on global config +* Add zuul driver pipeline trigger/reporter +* Add image lifecycle manager to zuul-launcher + * Emit image-build events + * Emit image-validate events + * Emit image-delete events +* Add Nodepool driver code to Zuul +* Update zuul-launcher to perform image uploads and deletion +* Implement node launch global request handler +* Implement node launch provider handlers +* Update Zuul nodepool interface to handle both Nodepool and + zuul-launcher node request queues +* Add tenant feature flag to switch between them +* Release a minor version of Zuul with support for both +* Remove Nodepool support from Zuul +* Release a major version of Zuul with only zuul-launcher support +* Retire Nodepool diff --git a/doc/source/drivers/mqtt.rst b/doc/source/drivers/mqtt.rst index 10cd4c8a2..81969ae8b 100644 --- a/doc/source/drivers/mqtt.rst +++ b/doc/source/drivers/mqtt.rst @@ -316,3 +316,9 @@ reporter. Each pipeline must provide a topic name. For example: The quality of service level to use, it can be 0, 1 or 2. Read more in this `guide <https://www.hivemq.com/blog/mqtt-essentials-part-6-mqtt-quality-of-service-levels>`_ + + .. attr:: include-returned-data + :default: false + + If set to ``true``, Zuul will include any data returned from the + job via :ref:`return_values`. diff --git a/doc/source/howtos/nodepool_static.rst b/doc/source/howtos/nodepool_static.rst index ff2d35d6a..c10672e7b 100644 --- a/doc/source/howtos/nodepool_static.rst +++ b/doc/source/howtos/nodepool_static.rst @@ -15,9 +15,9 @@ the following requirements: * Must be reachable by Zuul executors and have SSH access enabled. * Must have a user that Zuul can use for SSH. -* Must have Python 2 installed for Ansible. -* Must be reachable by Zuul executors over TCP port 19885 (console log - streaming). +* Must have an Ansible supported Python installed +* Must be reachable by Zuul executors over TCP port 19885 for console + log streaming. See :ref:`nodepool_console_streaming` When setting up your nodepool.yaml file, you will need the host keys for each node for the ``host-key`` value. This can be obtained with @@ -40,7 +40,7 @@ nodes. Place this file in ``/etc/nodepool/nodepool.yaml``: - host: localhost labels: - - name: ubuntu-xenial + - name: ubuntu-jammy providers: - name: static-vms @@ -49,14 +49,34 @@ nodes. Place this file in ``/etc/nodepool/nodepool.yaml``: - name: main nodes: - name: 192.168.1.10 - labels: ubuntu-xenial + labels: ubuntu-jammy host-key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGXqY02bdYqg1BcIf2x08zs60rS6XhlBSQ4qE47o5gb" username: zuul - name: 192.168.1.11 - labels: ubuntu-xenial + labels: ubuntu-jammy host-key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGXqY02bdYqg1BcIf2x08zs60rS6XhlBSQ5sE47o5gc" username: zuul EOF" Make sure that ``username``, ``host-key``, IP addresses and label names are customized for your environment. + +.. _nodepool_console_streaming: + +Log streaming +------------- + +The log streaming service enables Zuul to show the live status of +long-running ``shell`` or ``command`` tasks. The server side is setup +by the ``zuul_console:`` task built-in to Zuul's Ansible installation. +The executor requires the ability to communicate with the job nodes on +port 19885 for this to work. + +The log streaming service may leave files on the static node in the +format ``/tmp/console-<uuid>-<task_id>-<host>.log`` if jobs are +interrupted. These may be safely removed after a short period of +inactivity with a command such as + +.. code-block:: shell + + find /tmp -maxdepth 1 -name 'console-*-*-<host>.log' -mtime +2 -delete diff --git a/doc/source/installation.rst b/doc/source/installation.rst index a9a526f13..17665ca76 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -10,11 +10,15 @@ Nodepool ~~~~~~~~ In order to run all but the simplest jobs, Zuul uses a companion -program, Nodepool, to supply the nodes (whether dynamic cloud -instances or static hardware) used by jobs. Before starting Zuul, -ensure you have Nodepool installed and any images you require built. -Zuul only makes one requirement of these nodes: that it be able to log -in given a username and ssh private key. +program `Nodepool <https://opendev.org/zuul/nodepool>`__ to supply the +nodes (whether dynamic cloud instances or static hardware) used by +jobs. Before starting Zuul, ensure you have Nodepool installed and +any images you require built. + +Zuul must be able to log into the nodes provisioned by Nodepool with a +given username and SSH private key. Executors should also be able to +talk to nodes on TCP port 19885 for log streaming; see +:ref:`nodepool_console_streaming`. ZooKeeper ~~~~~~~~~ diff --git a/doc/source/job-content.rst b/doc/source/job-content.rst index 9b1059502..75044cf1c 100644 --- a/doc/source/job-content.rst +++ b/doc/source/job-content.rst @@ -332,6 +332,11 @@ of item. connectivity issues then previous attempts may have been cancelled, and this value will be greater than 1. + .. var:: ansible_version + + The version of the Ansible community package release used for executing + the job. + .. var:: project The item's project. This is a data structure with the following diff --git a/doc/source/tutorials/keycloak.rst b/doc/source/tutorials/keycloak.rst index 5242a4f05..896f35479 100644 --- a/doc/source/tutorials/keycloak.rst +++ b/doc/source/tutorials/keycloak.rst @@ -46,14 +46,14 @@ that we can update Zuul's configuration to add authentication. .. code-block:: shell cd zuul/doc/source/examples - sudo -E docker-compose-compose -p zuul-tutorial down + sudo -E docker-compose -p zuul-tutorial stop Restart the containers with a new Zuul configuration. .. code-block:: shell cd zuul/doc/source/examples - ZUUL_TUTORIAL_CONFIG="./keycloak/etc_zuul/" sudo -E docker-compose-compose -p zuul-tutorial up -d + ZUUL_TUTORIAL_CONFIG="./keycloak/etc_zuul/" sudo -E docker-compose -p zuul-tutorial up -d This tells docker-compose to use these Zuul `config files <https://opendev.org/zuul/zuul/src/branch/master/doc/source/examples/keycloak>`_. @@ -67,7 +67,7 @@ with this command: .. code-block:: shell cd zuul/doc/source/examples/keycloak - sudo -E docker-compose-compose -p zuul-tutorial-keycloak up -d + sudo -E docker-compose -p zuul-tutorial-keycloak up -d Once Keycloak is running, you can visit the web interface at http://localhost:8082/ diff --git a/playbooks/tutorial/admin.yaml b/playbooks/tutorial/admin.yaml index 9b36069e7..92d2b6d1f 100644 --- a/playbooks/tutorial/admin.yaml +++ b/playbooks/tutorial/admin.yaml @@ -2,13 +2,13 @@ - name: Run docker-compose down when: not local shell: - cmd: docker-compose -p zuul-tutorial down + cmd: docker-compose -p zuul-tutorial stop chdir: src/opendev.org/zuul/zuul/doc/source/examples - name: Run docker-compose down when: local shell: - cmd: docker-compose -p zuul-tutorial down + cmd: docker-compose -p zuul-tutorial stop chdir: ../../doc/source/examples # Restart with the new config @@ -55,3 +55,24 @@ until: result.status == 200 and result.json["zuul_version"] is defined changed_when: false +- name: Verify Keycloak authentication is available + uri: + url: http://localhost:9000/api/tenant/example-tenant/info + method: GET + return_content: true + status_code: 200 + body_format: json + register: result + failed_when: result.json["info"]["capabilities"]["auth"]["realms"]["zuul-demo"]["authority"] != "http://keycloak:8082/realms/zuul-demo" + changed_when: false + +- name: Verify that old builds are available + uri: + url: "http://localhost:9000/api/tenant/example-tenant/builds" + method: GET + return_content: true + status_code: 200 + body_format: json + register: result + failed_when: "result.json | length < 4" + changed_when: false diff --git a/playbooks/zuul-stream/2.7-container.yaml b/playbooks/zuul-stream/2.7-container.yaml new file mode 100644 index 000000000..76998a01d --- /dev/null +++ b/playbooks/zuul-stream/2.7-container.yaml @@ -0,0 +1,21 @@ +- name: Install docker + include_role: + name: ensure-docker + +- name: Build 2.7 container environment + shell: | + pushd {{ ansible_user_dir }}/src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/ + cat ~/.ssh/id_rsa.pub > authorized_keys + docker build -f Dockerfile.py27 -t zuul_python27 . + args: + executable: /bin/bash + +- name: Run 2.7 container + shell: | + docker run -d -p 2022:22 -p 19887:19887 zuul_python27 + docker ps + +- name: Accept host keys + shell: | + ssh-keyscan -p 2022 localhost >> ~/.ssh/known_hosts + ssh-keyscan -p 2022 127.0.0.2 >> ~/.ssh/known_hosts diff --git a/playbooks/zuul-stream/create-inventory.yaml b/playbooks/zuul-stream/create-inventory.yaml new file mode 100644 index 000000000..c2be02749 --- /dev/null +++ b/playbooks/zuul-stream/create-inventory.yaml @@ -0,0 +1,38 @@ +- name: Copy inventory + copy: + src: "{{ zuul.executor.log_root }}/zuul-info/inventory.yaml" + dest: "{{ ansible_user_dir }}/inventory.yaml" + +- name: Slurp inventory + slurp: + path: "{{ ansible_user_dir }}/inventory.yaml" + register: _inventory_yaml + +- name: Extract inventory + set_fact: + _new_inventory: "{{ _inventory_yaml['content'] | b64decode | from_yaml }}" + +- name: Setup new facts + set_fact: + _docker_inventory: + all: + children: + node: + hosts: + node3: null + hosts: + node3: + ansible_connection: ssh + ansible_host: 127.0.0.2 + ansible_port: 2022 + ansible_user: root + ansible_python_interpreter: /usr/local/bin/python2.7 + +- name: Merge all facts + set_fact: + _new_inventory: '{{ _new_inventory | combine(_docker_inventory, recursive=True) }}' + +- name: Write out inventory + copy: + content: '{{ _new_inventory | to_nice_yaml }}' + dest: '{{ ansible_user_dir }}/inventory.yaml' diff --git a/playbooks/zuul-stream/fixtures/Dockerfile.py27 b/playbooks/zuul-stream/fixtures/Dockerfile.py27 new file mode 100644 index 000000000..a30157b18 --- /dev/null +++ b/playbooks/zuul-stream/fixtures/Dockerfile.py27 @@ -0,0 +1,24 @@ +FROM python:2.7.18-buster AS buster-2.7-ssh + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update \ + && apt-get install -y dumb-init openssh-server \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir /var/run/sshd && chmod 0755 /var/run/sshd + +# This may or not be required to allow logins by preventing pam_loginuid +# trying to write out audit level things that may not work in a container +RUN sed -ri 's/session(\s+)required(\s+)pam_loginuid.so/session\1optional\2pam_loginuid.so/' /etc/pam.d/sshd + +RUN ssh-keygen -A -v + +RUN ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519 + +COPY authorized_keys /root/.ssh/authorized_keys +RUN chmod 0600 /root/.ssh/authorized_keys + +ENTRYPOINT ["/usr/bin/dumb-init", "--"] +CMD ["/usr/sbin/sshd", "-D", "-o", "ListenAddress=0.0.0.0" ] diff --git a/playbooks/zuul-stream/functional.yaml b/playbooks/zuul-stream/functional.yaml index b8a44a87c..63e13e3f5 100644 --- a/playbooks/zuul-stream/functional.yaml +++ b/playbooks/zuul-stream/functional.yaml @@ -31,11 +31,6 @@ mv job-output.txt job-output-success-19887.txt mv job-output.json job-output-success-19887.json - - name: Check protocol version - assert: - that: - - "'[node1] Reports streaming version: 1' in _success_output.stdout" - # Streamer puts out a line like # [node1] Starting to log 916b2084-4bbb-80e5-248e-000000000016-1-node1 for task TASK: Print binary data # One of the tasks in job-output shows find: results; @@ -53,10 +48,13 @@ # NOTE(ianw) 2022-07 : we deliberatly have this second step to run # against the console setup by the infrastructure executor in the # job pre playbooks as a backwards compatability sanity check. + # The py27 container job (node3) is not running an existing + # console streamer, so that will not output anything -- limit this + # out. - name: Run ansible that should succeed against extant console command: > /usr/lib/zuul/ansible/{{ zuul_ansible_version }}/bin/ansible-playbook - -e "new_console=false" + -e "new_console=false" --limit="node1,node2" src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/test-stream.yaml environment: ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json" @@ -81,6 +79,8 @@ - { node: 'node2', filename: 'job-output-success-19887.txt' } - { node: 'node1', filename: 'job-output-success-19885.txt' } - { node: 'node2', filename: 'job-output-success-19885.txt' } + # node3 only listen on 19887 + - { node: 'node3', filename: 'job-output-success-19887.txt' } # failure case @@ -105,8 +105,10 @@ shell: | egrep "^.+\| node1 \| Exception: Test module failure exception fail-task" job-output-failure.txt egrep "^.+\| node2 \| Exception: Test module failure exception fail-task" job-output-failure.txt + egrep "^.+\| node3 \| Exception: Test module failure exception fail-task" job-output-failure.txt - name: Validate output - failure item loop with exception shell: | egrep "^.+\| node1 \| Exception: Test module failure exception fail-loop" job-output-failure.txt egrep "^.+\| node2 \| Exception: Test module failure exception fail-loop" job-output-failure.txt + egrep "^.+\| node3 \| Exception: Test module failure exception fail-loop" job-output-failure.txt diff --git a/playbooks/zuul-stream/pre.yaml b/playbooks/zuul-stream/pre.yaml index 23fae3549..9753fab85 100644 --- a/playbooks/zuul-stream/pre.yaml +++ b/playbooks/zuul-stream/pre.yaml @@ -9,6 +9,12 @@ post_tasks: + - name: Setup 2.7 container environment + include_tasks: 2.7-container.yaml + + - name: Setup inventory + include_tasks: create-inventory.yaml + - name: Install pip shell: |+ python3 -m pip install --upgrade pip setuptools wheel @@ -36,11 +42,6 @@ # venvs) and the installation fails due to conflicts. SETUPTOOLS_USE_DISTUTILS: stdlib - - name: Copy inventory - copy: - src: "{{ zuul.executor.log_root }}/zuul-info/inventory.yaml" - dest: "{{ ansible_user_dir }}/inventory.yaml" - - name: Copy ansible.cfg template: src: templates/ansible.cfg.j2 diff --git a/releasenotes/notes/ansible-6-f939b4d160b41ec3.yaml b/releasenotes/notes/ansible-6-f939b4d160b41ec3.yaml new file mode 100644 index 000000000..c1bb9d534 --- /dev/null +++ b/releasenotes/notes/ansible-6-f939b4d160b41ec3.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Ansible version 6 is now available. The default Ansible version + is still 5, but version 6 may be selected by using + :attr:`job.ansible-version`. diff --git a/releasenotes/notes/change-queue-project-790553bd212b50eb.yaml b/releasenotes/notes/change-queue-project-790553bd212b50eb.yaml index a68db6f17..a88048242 100644 --- a/releasenotes/notes/change-queue-project-790553bd212b50eb.yaml +++ b/releasenotes/notes/change-queue-project-790553bd212b50eb.yaml @@ -2,5 +2,5 @@ deprecations: - | Shared ``queues`` should be configured per project now instead per - pipeline. Specifying :attr:`project.<pipeline>.queue` is deprecated + pipeline. Specifying `project.<pipeline>.queue` is deprecated and will be removed in a future release. diff --git a/releasenotes/notes/config-error-reporter-34887223d91544d1.yaml b/releasenotes/notes/config-error-reporter-34887223d91544d1.yaml new file mode 100644 index 000000000..51690f2fa --- /dev/null +++ b/releasenotes/notes/config-error-reporter-34887223d91544d1.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + A new :attr:`pipeline.config-error` pipeline reporter is available + for customizing reporter actions related to Zuul configuration + errors. diff --git a/releasenotes/notes/deprecate-ansible-2-4c22db35d3c6c765.yaml b/releasenotes/notes/deprecate-ansible-2-4c22db35d3c6c765.yaml new file mode 100644 index 000000000..09a0a128c --- /dev/null +++ b/releasenotes/notes/deprecate-ansible-2-4c22db35d3c6c765.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - | + Ansible versions 2.8 and 2.9 are now deprecated in Zuul since they + are both unmaintaned. Ansible 5 is now the default version in Zuul. diff --git a/releasenotes/notes/merge-conflict-rename-2-1e60065f196e48af.yaml b/releasenotes/notes/merge-conflict-rename-2-1e60065f196e48af.yaml new file mode 100644 index 000000000..962c1702e --- /dev/null +++ b/releasenotes/notes/merge-conflict-rename-2-1e60065f196e48af.yaml @@ -0,0 +1,6 @@ +upgrade: + - | + The previously deprecated ``merge-failure`` and + ``merge-failure-message`` pipeline configuration options have been + removed. Use ``merge-conflict`` and ``merge-conflict-message`` + respectively instead. diff --git a/releasenotes/notes/mqtt-include-returned-data-c5836db472907c42.yaml b/releasenotes/notes/mqtt-include-returned-data-c5836db472907c42.yaml new file mode 100644 index 000000000..2acb9433a --- /dev/null +++ b/releasenotes/notes/mqtt-include-returned-data-c5836db472907c42.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + The MQTT driver now supports including data returned from a job in + its reports. See + :attr:`pipeline.<reporter>.<mqtt>.include-returned-data`. diff --git a/releasenotes/notes/nodeset-alternatives-7cc39a69ac4f1481.yaml b/releasenotes/notes/nodeset-alternatives-7cc39a69ac4f1481.yaml new file mode 100644 index 000000000..e4dc4274b --- /dev/null +++ b/releasenotes/notes/nodeset-alternatives-7cc39a69ac4f1481.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Nodesets may now express an ordered list of alternatives so that + if Nodepool is unable to fulfill a request for certain labels, one + or more alternative Nodesets may be attempted instead. See + :attr:`nodeset.alternatives` for details. diff --git a/releasenotes/notes/pipeline-queue-removal-475caa7091f7e43f.yaml b/releasenotes/notes/pipeline-queue-removal-475caa7091f7e43f.yaml new file mode 100644 index 000000000..4d25d159b --- /dev/null +++ b/releasenotes/notes/pipeline-queue-removal-475caa7091f7e43f.yaml @@ -0,0 +1,6 @@ +--- +upgrade: + - | + The deprecated syntax of specifying project change queues on + pipeline configurations has been removed. Specify queues using + the project stanza now. See :attr:`queue` for more information. diff --git a/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback.yaml b/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback.yaml index 50bbbbfc5..13ddac988 100644 --- a/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback.yaml +++ b/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback.yaml @@ -1,4 +1,8 @@ - hosts: localhost - gather_facts: smart + gather_facts: false tasks: - command: echo test + + - name: Echo ansible version. + debug: + msg: Ansible version={{ ansible_version.major }}.{{ ansible_version.minor }} diff --git a/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback_plugins/test_callback.py b/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback_plugins/test_callback.py index 39ff7cd49..2597370bc 100644 --- a/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback_plugins/test_callback.py +++ b/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback_plugins/test_callback.py @@ -15,17 +15,20 @@ DOCUMENTATION = ''' class CallbackModule(CallbackBase): - CALLBACK_VERSION = 1.0 + """ + test callback + """ + CALLBACK_VERSION = 2.0 CALLBACK_NEEDS_WHITELIST = True + # aggregate means we can be loaded and not be the stdout plugin + CALLBACK_TYPE = 'aggregate' + CALLBACK_NAME = 'test_callback' def __init__(self): super(CallbackModule, self).__init__() - def set_options(self, task_keys=None, var_options=None, direct=None): - super(CallbackModule, self).set_options(task_keys=task_keys, - var_options=var_options, - direct=direct) - + def set_options(self, *args, **kw): + super(CallbackModule, self).set_options(*args, **kw) self.file_name = self.get_option('file_name') def v2_on_any(self, *args, **kwargs): diff --git a/tests/fixtures/config/ansible-callbacks/main.yaml b/tests/fixtures/config/ansible-callbacks/main.yaml index 9d01f542f..1e5247e4a 100644 --- a/tests/fixtures/config/ansible-callbacks/main.yaml +++ b/tests/fixtures/config/ansible-callbacks/main.yaml @@ -1,5 +1,6 @@ - tenant: name: tenant-one + default-ansible-version: SETME source: gerrit: config-projects: diff --git a/tests/fixtures/config/ansible-callbacks/main28.yaml b/tests/fixtures/config/ansible-callbacks/main28.yaml new file mode 100644 index 000000000..371710b4f --- /dev/null +++ b/tests/fixtures/config/ansible-callbacks/main28.yaml @@ -0,0 +1,7 @@ +- tenant: + name: tenant-one + default-ansible-version: '2.8' + source: + gerrit: + config-projects: + - common-config diff --git a/tests/fixtures/config/ansible-callbacks/main29.yaml b/tests/fixtures/config/ansible-callbacks/main29.yaml new file mode 100644 index 000000000..b127139a9 --- /dev/null +++ b/tests/fixtures/config/ansible-callbacks/main29.yaml @@ -0,0 +1,7 @@ +- tenant: + name: tenant-one + default-ansible-version: '2.9' + source: + gerrit: + config-projects: + - common-config diff --git a/tests/fixtures/config/ansible-callbacks/main5.yaml b/tests/fixtures/config/ansible-callbacks/main5.yaml new file mode 100644 index 000000000..5efc12339 --- /dev/null +++ b/tests/fixtures/config/ansible-callbacks/main5.yaml @@ -0,0 +1,7 @@ +- tenant: + name: tenant-one + default-ansible-version: '5' + source: + gerrit: + config-projects: + - common-config diff --git a/tests/fixtures/config/ansible-callbacks/main6.yaml b/tests/fixtures/config/ansible-callbacks/main6.yaml new file mode 100644 index 000000000..2467362bb --- /dev/null +++ b/tests/fixtures/config/ansible-callbacks/main6.yaml @@ -0,0 +1,7 @@ +- tenant: + name: tenant-one + default-ansible-version: '6' + source: + gerrit: + config-projects: + - common-config diff --git a/tests/fixtures/config/ansible-versions/git/common-config/zuul.yaml b/tests/fixtures/config/ansible-versions/git/common-config/zuul.yaml index 91c8d6bca..e1bac5e01 100644 --- a/tests/fixtures/config/ansible-versions/git/common-config/zuul.yaml +++ b/tests/fixtures/config/ansible-versions/git/common-config/zuul.yaml @@ -25,7 +25,7 @@ parent: ansible-version vars: test_ansible_version_major: 2 - test_ansible_version_minor: 9 + test_ansible_version_minor: 12 # This job is used by a test case specifying a different ansible version in # zuul.conf @@ -60,6 +60,14 @@ test_ansible_version_major: 2 test_ansible_version_minor: 12 +- job: + name: ansible-6 + parent: ansible-version + ansible-version: 6 + vars: + test_ansible_version_major: 2 + test_ansible_version_minor: 13 + - project: name: common-config check: @@ -68,6 +76,7 @@ - ansible-28 - ansible-29 - ansible-5 + - ansible-6 - project: name: org/project @@ -77,3 +86,4 @@ - ansible-28 - ansible-29 - ansible-5 + - ansible-6 diff --git a/tests/fixtures/config/ansible/git/org_ansible/playbooks/hello-ansible.yaml b/tests/fixtures/config/ansible/git/org_ansible/playbooks/hello-ansible.yaml index 17ddc1661..d0458c710 100644 --- a/tests/fixtures/config/ansible/git/org_ansible/playbooks/hello-ansible.yaml +++ b/tests/fixtures/config/ansible/git/org_ansible/playbooks/hello-ansible.yaml @@ -3,3 +3,7 @@ - name: hello debug: msg: hello ansible + + - name: Echo ansible version. + debug: + msg: Ansible version={{ ansible_version.major }}.{{ ansible_version.minor }} diff --git a/tests/fixtures/config/ansible/main.yaml b/tests/fixtures/config/ansible/main.yaml index 94e7aa78c..473bb5ef8 100644 --- a/tests/fixtures/config/ansible/main.yaml +++ b/tests/fixtures/config/ansible/main.yaml @@ -1,5 +1,6 @@ - tenant: name: tenant-one + default-ansible-version: SETME source: gerrit: config-projects: diff --git a/tests/fixtures/config/ansible/main28.yaml b/tests/fixtures/config/ansible/main28.yaml new file mode 100644 index 000000000..f2add49c7 --- /dev/null +++ b/tests/fixtures/config/ansible/main28.yaml @@ -0,0 +1,11 @@ +- tenant: + name: tenant-one + default-ansible-version: '2.8' + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project + - bare-role + - org/ansible diff --git a/tests/fixtures/config/ansible/main29.yaml b/tests/fixtures/config/ansible/main29.yaml new file mode 100644 index 000000000..758292950 --- /dev/null +++ b/tests/fixtures/config/ansible/main29.yaml @@ -0,0 +1,11 @@ +- tenant: + name: tenant-one + default-ansible-version: '2.9' + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project + - bare-role + - org/ansible diff --git a/tests/fixtures/config/ansible/main5.yaml b/tests/fixtures/config/ansible/main5.yaml new file mode 100644 index 000000000..b2364e80b --- /dev/null +++ b/tests/fixtures/config/ansible/main5.yaml @@ -0,0 +1,11 @@ +- tenant: + name: tenant-one + default-ansible-version: '5' + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project + - bare-role + - org/ansible diff --git a/tests/fixtures/config/ansible/main6.yaml b/tests/fixtures/config/ansible/main6.yaml new file mode 100644 index 000000000..7db6af6da --- /dev/null +++ b/tests/fixtures/config/ansible/main6.yaml @@ -0,0 +1,11 @@ +- tenant: + name: tenant-one + default-ansible-version: '6' + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project + - bare-role + - org/ansible diff --git a/tests/fixtures/config/authorization/rules-templating/git/common-config/zuul.yaml b/tests/fixtures/config/authorization/rules-templating/git/common-config/zuul.yaml index 750d578ec..b4ca647e0 100644 --- a/tests/fixtures/config/authorization/rules-templating/git/common-config/zuul.yaml +++ b/tests/fixtures/config/authorization/rules-templating/git/common-config/zuul.yaml @@ -126,6 +126,7 @@ - project: name: org/project1 + queue: integrated check: jobs: - project-merge @@ -136,7 +137,6 @@ - project1-project2-integration: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: @@ -148,6 +148,7 @@ - project: name: org/project2 + queue: integrated check: jobs: - project-merge @@ -158,7 +159,6 @@ - project1-project2-integration: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: @@ -170,6 +170,7 @@ - project: name: common-config + queue: integrated check: jobs: - project-merge @@ -180,7 +181,6 @@ - project1-project2-integration: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: diff --git a/tests/fixtures/config/authorization/single-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/authorization/single-tenant/git/common-config/zuul.yaml index 750d578ec..b4ca647e0 100644 --- a/tests/fixtures/config/authorization/single-tenant/git/common-config/zuul.yaml +++ b/tests/fixtures/config/authorization/single-tenant/git/common-config/zuul.yaml @@ -126,6 +126,7 @@ - project: name: org/project1 + queue: integrated check: jobs: - project-merge @@ -136,7 +137,6 @@ - project1-project2-integration: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: @@ -148,6 +148,7 @@ - project: name: org/project2 + queue: integrated check: jobs: - project-merge @@ -158,7 +159,6 @@ - project1-project2-integration: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: @@ -170,6 +170,7 @@ - project: name: common-config + queue: integrated check: jobs: - project-merge @@ -180,7 +181,6 @@ - project1-project2-integration: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: diff --git a/tests/fixtures/config/change-queues/git/common-config/zuul.d/config.yaml b/tests/fixtures/config/change-queues/git/common-config/zuul.d/config.yaml index 0fe373866..e548b52b6 100644 --- a/tests/fixtures/config/change-queues/git/common-config/zuul.d/config.yaml +++ b/tests/fixtures/config/change-queues/git/common-config/zuul.d/config.yaml @@ -29,8 +29,8 @@ - project: name: org/project2 + queue: integrated gate: - queue: integrated jobs: - project-test diff --git a/tests/fixtures/config/change-queues/git/org_project/.zuul.yaml b/tests/fixtures/config/change-queues/git/org_project/.zuul.yaml index b62c7ee67..765982d8f 100644 --- a/tests/fixtures/config/change-queues/git/org_project/.zuul.yaml +++ b/tests/fixtures/config/change-queues/git/org_project/.zuul.yaml @@ -4,7 +4,7 @@ per-branch: false - project: + queue: integrated gate: - queue: integrated jobs: - project-test diff --git a/tests/fixtures/config/change-queues/git/org_project3/zuul.d/project.yaml b/tests/fixtures/config/change-queues/git/org_project3/zuul.d/project.yaml index e1a297bd5..f80c5c571 100644 --- a/tests/fixtures/config/change-queues/git/org_project3/zuul.d/project.yaml +++ b/tests/fixtures/config/change-queues/git/org_project3/zuul.d/project.yaml @@ -1,5 +1,5 @@ - project: + queue: integrated-untrusted gate: - queue: integrated-untrusted jobs: - project-test diff --git a/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/zuul.yaml b/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/zuul.yaml index 57566f260..5d8f8c7a7 100644 --- a/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/zuul.yaml +++ b/tests/fixtures/config/cross-source-pagure/git/common-config-gerrit/zuul.yaml @@ -94,6 +94,7 @@ - project: name: gerrit/project1 + queue: integrated check: jobs: - project-merge @@ -104,7 +105,6 @@ - project1-project2-integration: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: @@ -116,6 +116,7 @@ - project: name: pagure/project2 + queue: integrated check: jobs: - project-merge @@ -126,7 +127,6 @@ - project1-project2-integration: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: diff --git a/tests/fixtures/config/cross-source-pagure/git/github_common-config/zuul.yaml b/tests/fixtures/config/cross-source-pagure/git/github_common-config/zuul.yaml index f7bbd4784..575020246 100644 --- a/tests/fixtures/config/cross-source-pagure/git/github_common-config/zuul.yaml +++ b/tests/fixtures/config/cross-source-pagure/git/github_common-config/zuul.yaml @@ -92,6 +92,7 @@ - project: name: github/project1 + queue: integrated check: jobs: - project-merge @@ -102,7 +103,6 @@ - project1-project2-integration: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: @@ -114,6 +114,7 @@ - project: name: pagure/project2 + queue: integrated check: jobs: - project-merge @@ -124,7 +125,6 @@ - project1-project2-integration: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: diff --git a/tests/fixtures/config/cross-source/git/common-config/zuul.yaml b/tests/fixtures/config/cross-source/git/common-config/zuul.yaml index abdc34afa..47ce3caea 100644 --- a/tests/fixtures/config/cross-source/git/common-config/zuul.yaml +++ b/tests/fixtures/config/cross-source/git/common-config/zuul.yaml @@ -125,6 +125,7 @@ - project: name: gerrit/project1 + queue: integrated check: jobs: - project-merge @@ -135,7 +136,6 @@ - project1-project2-integration: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: @@ -147,6 +147,7 @@ - project: name: github/project2 + queue: integrated check: jobs: - project-merge @@ -157,7 +158,6 @@ - project1-project2-integration: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: diff --git a/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml b/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml index dbd63c517..c822736ad 100755 --- a/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml +++ b/tests/fixtures/config/duplicate-pipeline/git/common-config/zuul.yaml @@ -36,11 +36,10 @@ - project: name: org/project + queue: integrated dup1: - queue: integrated jobs: - project-test1 dup2: - queue: integrated jobs: - project-test1 diff --git a/tests/fixtures/config/executor-facts/git/org_project/playbooks/datetime-fact.yaml b/tests/fixtures/config/executor-facts/git/org_project/playbooks/datetime-fact.yaml index 300dfa5f0..53819aa00 100644 --- a/tests/fixtures/config/executor-facts/git/org_project/playbooks/datetime-fact.yaml +++ b/tests/fixtures/config/executor-facts/git/org_project/playbooks/datetime-fact.yaml @@ -1,5 +1,5 @@ - hosts: localhost - gather_facts: smart + gather_facts: no tasks: - debug: var: date_time @@ -9,3 +9,6 @@ var: ansible_date_time - assert: that: ansible_date_time is not defined + - name: Echo ansible version + debug: + msg: Ansible version={{ ansible_version.major }}.{{ ansible_version.minor }} diff --git a/tests/fixtures/config/executor-facts/main.yaml b/tests/fixtures/config/executor-facts/main.yaml index 208e274b1..37c9dd4fc 100644 --- a/tests/fixtures/config/executor-facts/main.yaml +++ b/tests/fixtures/config/executor-facts/main.yaml @@ -1,5 +1,6 @@ - tenant: name: tenant-one + default-ansible-version: SETME source: gerrit: config-projects: diff --git a/tests/fixtures/config/executor-facts/main28.yaml b/tests/fixtures/config/executor-facts/main28.yaml new file mode 100644 index 000000000..686899bf8 --- /dev/null +++ b/tests/fixtures/config/executor-facts/main28.yaml @@ -0,0 +1,9 @@ +- tenant: + name: tenant-one + default-ansible-version: '2.8' + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project diff --git a/tests/fixtures/config/executor-facts/main29.yaml b/tests/fixtures/config/executor-facts/main29.yaml new file mode 100644 index 000000000..df934ff22 --- /dev/null +++ b/tests/fixtures/config/executor-facts/main29.yaml @@ -0,0 +1,9 @@ +- tenant: + name: tenant-one + default-ansible-version: '2.9' + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project diff --git a/tests/fixtures/config/executor-facts/main5.yaml b/tests/fixtures/config/executor-facts/main5.yaml new file mode 100644 index 000000000..55d9d10c0 --- /dev/null +++ b/tests/fixtures/config/executor-facts/main5.yaml @@ -0,0 +1,9 @@ +- tenant: + name: tenant-one + default-ansible-version: '5' + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project diff --git a/tests/fixtures/config/executor-facts/main6.yaml b/tests/fixtures/config/executor-facts/main6.yaml new file mode 100644 index 000000000..792f13402 --- /dev/null +++ b/tests/fixtures/config/executor-facts/main6.yaml @@ -0,0 +1,9 @@ +- tenant: + name: tenant-one + default-ansible-version: '6' + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project diff --git a/tests/fixtures/config/inventory/git/common-config/zuul.yaml b/tests/fixtures/config/inventory/git/common-config/zuul.yaml index 6b5fe67d0..ca687139d 100644 --- a/tests/fixtures/config/inventory/git/common-config/zuul.yaml +++ b/tests/fixtures/config/inventory/git/common-config/zuul.yaml @@ -140,3 +140,12 @@ label: ubuntu-xenial ansible-version: '5' run: playbooks/ansible-version.yaml + +- job: + name: ansible-version6-inventory + nodeset: + nodes: + - name: ubuntu-xenial + label: ubuntu-xenial + ansible-version: '6' + run: playbooks/ansible-version.yaml diff --git a/tests/fixtures/config/inventory/git/org_project/.zuul.yaml b/tests/fixtures/config/inventory/git/org_project/.zuul.yaml index 1028a0ff2..69ec3127e 100644 --- a/tests/fixtures/config/inventory/git/org_project/.zuul.yaml +++ b/tests/fixtures/config/inventory/git/org_project/.zuul.yaml @@ -10,3 +10,4 @@ - ansible-version28-inventory - ansible-version29-inventory - ansible-version5-inventory + - ansible-version6-inventory diff --git a/tests/fixtures/config/mqtt-driver/git/common-config/zuul.d/config.yaml b/tests/fixtures/config/mqtt-driver/git/common-config/zuul.d/config.yaml index c842e9424..1cee46f5c 100644 --- a/tests/fixtures/config/mqtt-driver/git/common-config/zuul.d/config.yaml +++ b/tests/fixtures/config/mqtt-driver/git/common-config/zuul.d/config.yaml @@ -7,11 +7,15 @@ start: mqtt: topic: "{tenant}/zuul_start/{pipeline}/{project}/{branch}" + # This doesn't make sense here -- there should be no return + # data yet, which is why we include it in this test. + include-returned-data: True success: gerrit: Verified: 1 mqtt: topic: "{tenant}/zuul_buildset/{pipeline}/{project}/{branch}" + include-returned-data: True failure: gerrit: Verified: -1 diff --git a/tests/fixtures/config/openstack/git/project-config/zuul.yaml b/tests/fixtures/config/openstack/git/project-config/zuul.yaml index 93bdb1132..4f06a1feb 100644 --- a/tests/fixtures/config/openstack/git/project-config/zuul.yaml +++ b/tests/fixtures/config/openstack/git/project-config/zuul.yaml @@ -80,20 +80,20 @@ - project: name: openstack/nova + queue: integrated templates: - python-jobs check: jobs: - dsvm - gate: - queue: integrated + gate: {} - project: name: openstack/keystone + queue: integrated templates: - python-jobs check: jobs: - dsvm - gate: - queue: integrated + gate: {} diff --git a/tests/fixtures/config/provides-requires-pause/git/org_project1/zuul.yaml b/tests/fixtures/config/provides-requires-pause/git/org_project1/zuul.yaml index 412fe2c18..fb8a6eea0 100644 --- a/tests/fixtures/config/provides-requires-pause/git/org_project1/zuul.yaml +++ b/tests/fixtures/config/provides-requires-pause/git/org_project1/zuul.yaml @@ -11,6 +11,7 @@ run: playbooks/image-user.yaml - project: + queue: integrated check: jobs: - image-builder @@ -18,7 +19,6 @@ dependencies: - image-builder gate: - queue: integrated jobs: - image-builder - image-user: diff --git a/tests/fixtures/config/provides-requires-pause/git/org_project2/zuul.yaml b/tests/fixtures/config/provides-requires-pause/git/org_project2/zuul.yaml index e9e6b5867..7f8de1178 100644 --- a/tests/fixtures/config/provides-requires-pause/git/org_project2/zuul.yaml +++ b/tests/fixtures/config/provides-requires-pause/git/org_project2/zuul.yaml @@ -1,8 +1,8 @@ - project: + queue: integrated check: jobs: - image-user gate: - queue: integrated jobs: - image-user diff --git a/tests/fixtures/config/remote-zuul-stream/git/org_project/playbooks/command.yaml b/tests/fixtures/config/remote-zuul-stream/git/org_project/playbooks/command.yaml index d737a1a9b..539db80b7 100644 --- a/tests/fixtures/config/remote-zuul-stream/git/org_project/playbooks/command.yaml +++ b/tests/fixtures/config/remote-zuul-stream/git/org_project/playbooks/command.yaml @@ -114,3 +114,17 @@ - name: Command Not Found command: command-not-found failed_when: false + +- hosts: compute1 + tasks: + + - name: Debug raw variable in msg + debug: + msg: '{{ ansible_version }}' + + - name: Debug raw variable in a loop + debug: + msg: '{{ ansible_version }}' + loop: + - 1 + - 2 diff --git a/tests/fixtures/config/remote-zuul-stream/git/org_project/playbooks/library/broken_module_exception.py b/tests/fixtures/config/remote-zuul-stream/git/org_project/playbooks/library/broken_module_exception.py index 6cfa0a3d0..7933f15a8 100755 --- a/tests/fixtures/config/remote-zuul-stream/git/org_project/playbooks/library/broken_module_exception.py +++ b/tests/fixtures/config/remote-zuul-stream/git/org_project/playbooks/library/broken_module_exception.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 def main(): diff --git a/tests/fixtures/config/remote-zuul-stream/git/org_project/playbooks/library/broken_module_no_result.py b/tests/fixtures/config/remote-zuul-stream/git/org_project/playbooks/library/broken_module_no_result.py index 065509d0d..0bdced5eb 100755 --- a/tests/fixtures/config/remote-zuul-stream/git/org_project/playbooks/library/broken_module_no_result.py +++ b/tests/fixtures/config/remote-zuul-stream/git/org_project/playbooks/library/broken_module_no_result.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 def main(): diff --git a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml index b29f5a654..cbc523a8b 100644 --- a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml +++ b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml @@ -148,8 +148,6 @@ - project1-project2-integration: dependencies: project-merge gate: - # This will be overridden on project level - queue: integrated-overridden jobs: - project-merge - project-test1: @@ -164,6 +162,7 @@ - project: name: org/project2 + queue: integrated check: jobs: - project-merge @@ -174,7 +173,6 @@ - project1-project2-integration: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: @@ -186,6 +184,7 @@ - project: name: common-config + queue: integrated check: jobs: - project-merge @@ -196,7 +195,6 @@ - project1-project2-integration: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: diff --git a/tests/fixtures/config/single-tenant/git/org_project3/zuul.yaml b/tests/fixtures/config/single-tenant/git/org_project3/zuul.yaml index 3effc3cb4..a588b7aa2 100644 --- a/tests/fixtures/config/single-tenant/git/org_project3/zuul.yaml +++ b/tests/fixtures/config/single-tenant/git/org_project3/zuul.yaml @@ -1,4 +1,5 @@ - project: + queue: integrated check: jobs: - project-merge @@ -11,7 +12,6 @@ - project-test1 - project-test2 gate: - queue: integrated jobs: - project-merge - project-test1: diff --git a/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml b/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml index 57d50ca43..6192249c6 100644 --- a/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml +++ b/tests/fixtures/config/sql-driver/git/common-config/zuul.yaml @@ -105,6 +105,7 @@ - project: name: org/project1 + queue: integrated check: jobs: - project-merge @@ -113,7 +114,6 @@ - project-test2: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: diff --git a/tests/fixtures/layout.yaml b/tests/fixtures/layout.yaml deleted file mode 100644 index cd8ce1906..000000000 --- a/tests/fixtures/layout.yaml +++ /dev/null @@ -1,233 +0,0 @@ -includes: - - python-file: custom_functions.py - -pipelines: - - name: check - manager: independent - source: - gerrit - trigger: - gerrit: - - event: patchset-created - success: - gerrit: - Verified: 1 - failure: - gerrit: - Verified: -1 - - - name: post - manager: independent - source: - gerrit - trigger: - gerrit: - - event: ref-updated - ref: ^(?!refs/).*$ - - - name: gate - manager: dependent - failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures - source: - gerrit - trigger: - gerrit: - - event: comment-added - approval: - - Approved: 1 - success: - gerrit: - Verified: 2 - submit: true - failure: - gerrit: - Verified: -2 - start: - gerrit: - Verified: 0 - precedence: high - - - name: unused - manager: independent - dequeue-on-new-patchset: false - source: - gerrit - trigger: - gerrit: - - event: comment-added - approval: - - Approved: 1 - - - name: dup1 - manager: independent - source: - gerrit - trigger: - gerrit: - - event: change-restored - success: - gerrit: - Verified: 1 - failure: - gerrit: - Verified: -1 - - - name: dup2 - manager: independent - source: - gerrit - trigger: - gerrit: - - event: change-restored - success: - gerrit: - Verified: 1 - failure: - gerrit: - Verified: -1 - - - name: conflict - manager: dependent - failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures - source: - gerrit - trigger: - gerrit: - - event: comment-added - approval: - - Approved: 1 - success: - gerrit: - Verified: 2 - submit: true - failure: - gerrit: - Verified: -2 - start: - gerrit: - Verified: 0 - - - name: experimental - manager: independent - source: - gerrit - trigger: - gerrit: - - event: patchset-created - success: - gerrit: {} - failure: - gerrit: {} - -jobs: - - name: ^.*-merge$ - failure-message: Unable to merge change - hold-following-changes: true - tags: merge - - name: nonvoting-project-test2 - voting: false - - name: project-testfile - files: - - '.*-requires' - - name: project1-project2-integration - queue-name: integration - - name: mutex-one - mutex: test-mutex - - name: mutex-two - mutex: test-mutex - - name: project1-merge - tags: - - project1 - - extratag - -projects: - - name: org/project - merge-mode: cherry-pick - check: - - project-merge: - - project-test1 - - project-test2 - - project-testfile - gate: - - project-merge: - - project-test1 - - project-test2 - - project-testfile - post: - - project-post - dup1: - - project-test1 - dup2: - - project-test1 - - - name: org/project1 - check: - - project1-merge: - - project1-test1 - - project1-test2 - - project1-project2-integration - gate: - - project1-merge: - - project1-test1 - - project1-test2 - - project1-project2-integration - post: - - project1-post - - - name: org/project2 - check: - - project2-merge: - - project2-test1 - - project2-test2 - - project1-project2-integration - gate: - - project2-merge: - - project2-test1 - - project2-test2 - - project1-project2-integration - post: - - project2-post - - - name: org/project3 - check: - - project3-merge: - - project3-test1 - - project3-test2 - - project1-project2-integration - gate: - - project3-merge: - - project3-test1 - - project3-test2 - - project1-project2-integration - post: - - project3-post - - - name: org/nonvoting-project - check: - - nonvoting-project-merge: - - nonvoting-project-test1 - - nonvoting-project-test2 - gate: - - nonvoting-project-merge: - - nonvoting-project-test1 - - nonvoting-project-test2 - post: - - nonvoting-project-post - - - name: org/conflict-project - conflict: - - conflict-project-merge: - - conflict-project-test1 - - conflict-project-test2 - - - name: org/noop-project - gate: - - noop - - - name: org/experimental-project - experimental: - - experimental-project-test - - - name: org/no-jobs-project - check: - - project-testfile diff --git a/tests/fixtures/layouts/crd-github.yaml b/tests/fixtures/layouts/crd-github.yaml index 6ef881f9b..bc938dec3 100644 --- a/tests/fixtures/layouts/crd-github.yaml +++ b/tests/fixtures/layouts/crd-github.yaml @@ -70,15 +70,15 @@ - project: name: org/project3 + queue: cogated gate: - queue: cogated jobs: - project3-test - project: name: org/project4 + queue: cogated gate: - queue: cogated jobs: - project4-test diff --git a/tests/fixtures/layouts/crd-gitlab.yaml b/tests/fixtures/layouts/crd-gitlab.yaml index 210390b93..1884e4026 100644 --- a/tests/fixtures/layouts/crd-gitlab.yaml +++ b/tests/fixtures/layouts/crd-gitlab.yaml @@ -55,14 +55,14 @@ - project: name: org/project3 + queue: cogated gate: - queue: cogated jobs: - project3-test - project: name: org/project4 + queue: cogated gate: - queue: cogated jobs: - - project4-test
\ No newline at end of file + - project4-test diff --git a/tests/fixtures/layouts/crd-pagure.yaml b/tests/fixtures/layouts/crd-pagure.yaml index e0828aeeb..79e041f9a 100644 --- a/tests/fixtures/layouts/crd-pagure.yaml +++ b/tests/fixtures/layouts/crd-pagure.yaml @@ -52,14 +52,14 @@ - project: name: org/project3 + queue: cogated gate: - queue: cogated jobs: - project3-test - project: name: org/project4 + queue: cogated gate: - queue: cogated jobs: - project4-test diff --git a/tests/fixtures/layouts/freeze-job-failure.yaml b/tests/fixtures/layouts/freeze-job-failure.yaml new file mode 100644 index 000000000..ae3f48324 --- /dev/null +++ b/tests/fixtures/layouts/freeze-job-failure.yaml @@ -0,0 +1,32 @@ +- pipeline: + name: check + manager: independent + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + Verified: 1 + failure: + gerrit: + Verified: -1 + +- job: + name: base + parent: null + run: playbooks/base.yaml + +- job: + name: project-test1 + run: playbooks/project-test1.yaml + +- job: + name: project-test2 + run: playbooks/project-test2.yaml + +- project: + name: org/project + check: + jobs: + - project-test2: + dependencies: project-test1 diff --git a/tests/fixtures/layouts/merge-failure.yaml b/tests/fixtures/layouts/merge-failure.yaml index 3828a06eb..2efcdced3 100644 --- a/tests/fixtures/layouts/merge-failure.yaml +++ b/tests/fixtures/layouts/merge-failure.yaml @@ -23,7 +23,7 @@ name: gate manager: dependent failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures - merge-failure-message: The merge failed! For more information... + merge-conflict-message: The merge failed! For more information... trigger: gerrit: - event: comment-added @@ -36,7 +36,7 @@ failure: gerrit: Verified: -2 - merge-failure: + merge-conflict: gerrit: Verified: -1 smtp: diff --git a/tests/fixtures/layouts/nodeset-alternatives.yaml b/tests/fixtures/layouts/nodeset-alternatives.yaml new file mode 100644 index 000000000..21f9f11ae --- /dev/null +++ b/tests/fixtures/layouts/nodeset-alternatives.yaml @@ -0,0 +1,14 @@ +- nodeset: + name: fast-nodeset + nodes: + - name: controller + label: fast-label + +- job: + name: test-job + nodeset: + alternatives: + - fast-nodeset + - nodes: + - name: controller + label: slow-label diff --git a/tests/fixtures/layouts/nodeset-fallback.yaml b/tests/fixtures/layouts/nodeset-fallback.yaml new file mode 100644 index 000000000..01869537e --- /dev/null +++ b/tests/fixtures/layouts/nodeset-fallback.yaml @@ -0,0 +1,66 @@ +- pipeline: + name: check + manager: independent + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + Verified: 1 + failure: + gerrit: + Verified: -1 + +- nodeset: + name: fast-nodeset + nodes: + - label: fast-label + name: controller + +- nodeset: + name: red-nodeset + nodes: + - label: red-label + name: controller + +- nodeset: + name: blue-nodeset + nodes: + - label: blue-label + name: controller + +# This adds an unused second level of alternatives to verify we are +# able to flatten it. +- nodeset: + name: red-or-blue-nodeset + alternatives: + - red-nodeset + - blue-nodeset + +# Test alternatives by name or anonymous nodeset +- nodeset: + name: fast-or-slow + alternatives: + - fast-nodeset + - nodes: + label: slow-label + name: controller + - red-or-blue-nodeset + +- job: + name: base + parent: null + run: playbooks/base.yaml + +- job: + name: check-job + nodeset: fast-or-slow + +- project: + name: org/project + check: + jobs: + - check-job + gate: + jobs: + - check-job diff --git a/tests/fixtures/layouts/provides-requires-two-jobs.yaml b/tests/fixtures/layouts/provides-requires-two-jobs.yaml index 9d1008752..7568c31d8 100644 --- a/tests/fixtures/layouts/provides-requires-two-jobs.yaml +++ b/tests/fixtures/layouts/provides-requires-two-jobs.yaml @@ -51,11 +51,11 @@ - project: name: org/project1 + queue: integrated check: jobs: - image-builder gate: - queue: integrated jobs: - image-builder - image-user: @@ -63,10 +63,10 @@ - project: name: org/project2 + queue: integrated check: jobs: - image-user gate: - queue: integrated jobs: - image-user diff --git a/tests/fixtures/layouts/provides-requires.yaml b/tests/fixtures/layouts/provides-requires.yaml index 17b17bab1..aeb959bfd 100644 --- a/tests/fixtures/layouts/provides-requires.yaml +++ b/tests/fixtures/layouts/provides-requires.yaml @@ -72,18 +72,19 @@ - project: name: org/project1 + queue: integrated check: jobs: - image-builder - library-builder - hold gate: - queue: integrated jobs: - image-builder - project: name: org/project2 + queue: integrated check: jobs: - image-user @@ -91,7 +92,6 @@ - library-user2 - hold gate: - queue: integrated jobs: - image-user diff --git a/tests/fixtures/layouts/regex-queue.yaml b/tests/fixtures/layouts/regex-queue.yaml index 0bacbec0e..5650e2611 100644 --- a/tests/fixtures/layouts/regex-queue.yaml +++ b/tests/fixtures/layouts/regex-queue.yaml @@ -10,8 +10,8 @@ - project: name: ^.*$ + queue: integrated gate: - queue: integrated jobs: - base diff --git a/tests/fixtures/layouts/regex-template-queue.yaml b/tests/fixtures/layouts/regex-template-queue.yaml index f809e0e43..38eb596ea 100644 --- a/tests/fixtures/layouts/regex-template-queue.yaml +++ b/tests/fixtures/layouts/regex-template-queue.yaml @@ -10,8 +10,8 @@ - project-template: name: integrated-jobs + queue: integrated gate: - queue: integrated jobs: - base diff --git a/tests/fixtures/layouts/repo-checkout-four-project.yaml b/tests/fixtures/layouts/repo-checkout-four-project.yaml index 11212e85f..433e6f584 100644 --- a/tests/fixtures/layouts/repo-checkout-four-project.yaml +++ b/tests/fixtures/layouts/repo-checkout-four-project.yaml @@ -48,40 +48,40 @@ - project: name: org/project1 + queue: integrated check: jobs: - integration gate: - queue: integrated jobs: - integration - project: name: org/project2 + queue: integrated check: jobs: - integration gate: - queue: integrated jobs: - integration - project: name: org/project3 + queue: integrated check: jobs: - integration gate: - queue: integrated jobs: - integration - project: name: org/project4 + queue: integrated check: jobs: - integration gate: - queue: integrated jobs: - integration diff --git a/tests/fixtures/layouts/repo-checkout-six-project.yaml b/tests/fixtures/layouts/repo-checkout-six-project.yaml index 48786654b..92f2f0a20 100644 --- a/tests/fixtures/layouts/repo-checkout-six-project.yaml +++ b/tests/fixtures/layouts/repo-checkout-six-project.yaml @@ -51,60 +51,60 @@ - project: name: org/project1 + queue: integrated check: jobs: - integration gate: - queue: integrated jobs: - integration - project: name: org/project2 + queue: integrated check: jobs: - integration gate: - queue: integrated jobs: - integration - project: name: org/project3 + queue: integrated check: jobs: - integration gate: - queue: integrated jobs: - integration - project: name: org/project4 + queue: integrated check: jobs: - integration gate: - queue: integrated jobs: - integration - project: name: org/project5 + queue: integrated check: jobs: - integration gate: - queue: integrated jobs: - integration - project: name: org/project6 + queue: integrated check: jobs: - integration gate: - queue: integrated jobs: - integration diff --git a/tests/fixtures/layouts/repo-checkout-two-project.yaml b/tests/fixtures/layouts/repo-checkout-two-project.yaml index 64c6ee943..6cf66a994 100644 --- a/tests/fixtures/layouts/repo-checkout-two-project.yaml +++ b/tests/fixtures/layouts/repo-checkout-two-project.yaml @@ -46,20 +46,20 @@ - project: name: org/project1 + queue: integrated check: jobs: - integration gate: - queue: integrated jobs: - integration - project: name: org/project2 + queue: integrated check: jobs: - integration gate: - queue: integrated jobs: - integration diff --git a/tests/fixtures/layouts/serial.yaml b/tests/fixtures/layouts/serial.yaml index 5a744ce82..70d16bd1a 100644 --- a/tests/fixtures/layouts/serial.yaml +++ b/tests/fixtures/layouts/serial.yaml @@ -35,14 +35,14 @@ - project: name: org/project1 + queue: shared deploy: - queue: shared jobs: - job1 - project: name: org/project2 + queue: shared deploy: - queue: shared jobs: - job1 diff --git a/tests/fixtures/layouts/template-queue.yaml b/tests/fixtures/layouts/template-queue.yaml index 407956feb..bc841e38b 100644 --- a/tests/fixtures/layouts/template-queue.yaml +++ b/tests/fixtures/layouts/template-queue.yaml @@ -10,8 +10,8 @@ - project-template: name: integrated-jobs + queue: integrated gate: - queue: integrated jobs: - base diff --git a/tests/fixtures/layouts/three-projects.yaml b/tests/fixtures/layouts/three-projects.yaml index 33e81aca5..2db072eda 100644 --- a/tests/fixtures/layouts/three-projects.yaml +++ b/tests/fixtures/layouts/three-projects.yaml @@ -56,6 +56,7 @@ - project: name: org/project1 + queue: integrated check: jobs: - project-merge @@ -66,7 +67,6 @@ - project1-project2-integration: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: @@ -78,6 +78,7 @@ - project: name: org/project2 + queue: integrated check: jobs: - project-merge @@ -88,7 +89,6 @@ - project1-project2-integration: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: @@ -100,6 +100,7 @@ - project: name: org/project3 + queue: integrated check: jobs: - project-merge @@ -110,7 +111,6 @@ - project1-project2-integration: dependencies: project-merge gate: - queue: integrated jobs: - project-merge - project-test1: diff --git a/tests/fixtures/layouts/timer-freeze-job-failure.yaml b/tests/fixtures/layouts/timer-freeze-job-failure.yaml new file mode 100644 index 000000000..2e6d709bb --- /dev/null +++ b/tests/fixtures/layouts/timer-freeze-job-failure.yaml @@ -0,0 +1,26 @@ +- pipeline: + name: periodic + manager: independent + trigger: + timer: + - time: '* * * * * */1' + +- job: + name: base + parent: null + run: playbooks/base.yaml + +- job: + name: project-test1 + run: playbooks/project-test1.yaml + +- job: + name: project-test2 + run: playbooks/project-test2.yaml + +- project: + name: org/project + periodic: + jobs: + - project-test2: + dependencies: project-test1 diff --git a/tests/fixtures/layouts/two-projects-integrated.yaml b/tests/fixtures/layouts/two-projects-integrated.yaml index 89302f62a..7e3a07a00 100644 --- a/tests/fixtures/layouts/two-projects-integrated.yaml +++ b/tests/fixtures/layouts/two-projects-integrated.yaml @@ -79,11 +79,10 @@ - project: name: org/project2 + queue: integrated check: - queue: integrated jobs: - integration gate: - queue: integrated jobs: - integration diff --git a/tests/remote/test_remote_action_modules.py b/tests/remote/test_remote_action_modules.py index bbe6db0a0..30e430b74 100644 --- a/tests/remote/test_remote_action_modules.py +++ b/tests/remote/test_remote_action_modules.py @@ -109,3 +109,11 @@ class TestActionModules5(AnsibleZuulTestCase, FunctionalActionModulesMixIn): def setUp(self): super().setUp() self._setUp() + + +class TestActionModules6(AnsibleZuulTestCase, FunctionalActionModulesMixIn): + ansible_version = '6' + + def setUp(self): + super().setUp() + self._setUp() diff --git a/tests/remote/test_remote_zuul_json.py b/tests/remote/test_remote_zuul_json.py index 120235ec9..e4510e7d1 100644 --- a/tests/remote/test_remote_zuul_json.py +++ b/tests/remote/test_remote_zuul_json.py @@ -166,3 +166,11 @@ class TestZuulJSON5(AnsibleZuulTestCase, FunctionalZuulJSONMixIn): def setUp(self): super().setUp() self._setUp() + + +class TestZuulJSON6(AnsibleZuulTestCase, FunctionalZuulJSONMixIn): + ansible_version = '6' + + def setUp(self): + super().setUp() + self._setUp() diff --git a/tests/remote/test_remote_zuul_stream.py b/tests/remote/test_remote_zuul_stream.py index 1c705127e..b84c4b0d8 100644 --- a/tests/remote/test_remote_zuul_stream.py +++ b/tests/remote/test_remote_zuul_stream.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. +import io +import logging import os import re import textwrap @@ -31,6 +33,12 @@ class FunctionalZuulStreamMixIn: self.executor_server.log_console_port = self.log_console_port self.wait_timeout = 180 self.fake_nodepool.remote_ansible = True + # This catches the Ansible output; rather than the callback + # output captured in the job log. For example if the callback + # fails, there will be an error output in this stream. + self.logger = logging.getLogger('zuul.AnsibleJob') + self.console_output = io.StringIO() + self.logger.addHandler(logging.StreamHandler(self.console_output)) ansible_remote = os.environ.get('ZUUL_REMOTE_IPV4') self.assertIsNotNone(ansible_remote) @@ -92,14 +100,20 @@ class FunctionalZuulStreamMixIn: with open(path) as f: return f.read() - def assertLogLine(self, line, log): - pattern = (r'^\d\d\d\d-\d\d-\d\d \d\d:\d\d\:\d\d\.\d\d\d\d\d\d \| %s$' - % line) + def _assertLogLine(self, line, log, full_match=True): + pattern = (r'^\d\d\d\d-\d\d-\d\d \d\d:\d\d\:\d\d\.\d\d\d\d\d\d \| %s%s' + % (line, '$' if full_match else '')) log_re = re.compile(pattern, re.MULTILINE) m = log_re.search(log) if m is None: raise Exception("'%s' not found in log" % (line,)) + def assertLogLineStartsWith(self, line, log): + self._assertLogLine(line, log, full_match=False) + + def assertLogLine(self, line, log): + self._assertLogLine(line, log, full_match=True) + def _getLogTime(self, line, log): pattern = (r'^(\d\d\d\d-\d\d-\d\d \d\d:\d\d\:\d\d\.\d\d\d\d\d\d)' r' \| %s\n' @@ -120,7 +134,21 @@ class FunctionalZuulStreamMixIn: build = self.history[-1] self.assertEqual(build.result, 'SUCCESS') + console_output = self.console_output.getvalue() + # This should be generic enough to match any callback + # plugin failures, which look something like + # + # [WARNING]: Failure using method (v2_runner_on_ok) in \ + # callback plugin + # (<ansible.plugins.callback.zuul_stream.CallbackModule object at' + # 0x7f89f72a20b0>): 'dict' object has no attribute 'startswith'" + # Callback Exception: + # ... + # + self.assertNotIn('[WARNING]: Failure using method', console_output) + text = self._get_job_output(build) + self.assertLogLine( r'RUN START: \[untrusted : review.example.com/org/project/' r'playbooks/command.yaml@master\]', text) @@ -186,6 +214,20 @@ class FunctionalZuulStreamMixIn: self.assertLess((time2 - time1) / timedelta(milliseconds=1), 9000) + # This is from the debug: msg='{{ ansible_version }}' + # testing raw variable output. To make it version + # agnostic, match just the start of + # compute1 | ok: {'string': '2.9.27'... + + # NOTE(ianw) 2022-08-24 : I don't know why the callback + # for debug: msg= doesn't put the hostname first like + # other output. Undetermined if bug or feature. + self.assertLogLineStartsWith( + r"""\{'string': '\d.""", text) + # ... handling loops is a different path, and that does + self.assertLogLineStartsWith( + r"""compute1 \| ok: \{'string': '\d.""", text) + def test_module_exception(self): job = self._run_job('module_failure_exception') with self.jobLog(job): @@ -238,3 +280,12 @@ class TestZuulStream5(AnsibleZuulTestCase, FunctionalZuulStreamMixIn): def setUp(self): super().setUp() self._setUp() + + +class TestZuulStream6(AnsibleZuulTestCase, FunctionalZuulStreamMixIn): + ansible_version = '6' + ansible_core_version = '2.13' + + def setUp(self): + super().setUp() + self._setUp() diff --git a/tests/unit/test_circular_dependencies.py b/tests/unit/test_circular_dependencies.py index 292941c13..6060855cd 100644 --- a/tests/unit/test_circular_dependencies.py +++ b/tests/unit/test_circular_dependencies.py @@ -1077,11 +1077,11 @@ class TestGerritCircularDependencies(ZuulTestCase): use_job = textwrap.dedent( """ - project: + queue: integrated check: jobs: - new-job gate: - queue: integrated jobs: - new-job """) @@ -1134,11 +1134,11 @@ class TestGerritCircularDependencies(ZuulTestCase): test_var: pass - project: + queue: integrated check: jobs: - project-vars-job gate: - queue: integrated jobs: - project-vars-job """) @@ -1330,11 +1330,11 @@ class TestGerritCircularDependencies(ZuulTestCase): test_var: pass - project: + queue: integrated check: jobs: - project-vars-job gate: - queue: integrated jobs: - project-vars-job """) diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py index b4c155240..bae4ff258 100644 --- a/tests/unit/test_connection.py +++ b/tests/unit/test_connection.py @@ -681,9 +681,13 @@ class TestMQTTConnection(ZuulTestCase): 'type': 'container_image' }} self.executor_server.returnData( - "test", A, {"zuul": {"log_url": "some-log-url", - 'artifacts': [artifact], - }} + "test", A, { + "zuul": { + "log_url": "some-log-url", + 'artifacts': [artifact], + }, + 'foo': 'bar', + } ) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() @@ -703,6 +707,9 @@ class TestMQTTConnection(ZuulTestCase): 'test') self.assertNotIn('result', mqtt_payload['buildset']['builds'][0]) self.assertNotIn('artifacts', mqtt_payload['buildset']['builds'][0]) + builds = mqtt_payload['buildset']['builds'] + test_job = [b for b in builds if b['job_name'] == 'test'][0] + self.assertNotIn('returned_data', test_job) self.assertEquals(success_event.get('topic'), 'tenant-one/zuul_buildset/check/org/project/master') @@ -720,6 +727,7 @@ class TestMQTTConnection(ZuulTestCase): self.assertEquals(test_job['dependencies'], []) self.assertEquals(test_job['artifacts'], [artifact]) self.assertEquals(test_job['log_url'], 'some-log-url/') + self.assertEquals(test_job['returned_data'], {'foo': 'bar'}) build_id = test_job["uuid"] self.assertEquals( test_job["web_url"], diff --git a/tests/unit/test_executor.py b/tests/unit/test_executor.py index d18cf4448..27ca75531 100644 --- a/tests/unit/test_executor.py +++ b/tests/unit/test_executor.py @@ -838,8 +838,10 @@ class TestLineMapping(AnsibleZuulTestCase): ) -class TestExecutorFacts(AnsibleZuulTestCase): +class ExecutorFactsMixin: + # These should be overridden in child classes. tenant_config_file = 'config/executor-facts/main.yaml' + ansible_major_minor = 'X.Y' def _get_file(self, build, path): p = os.path.join(build.jobdir.root, path) @@ -861,12 +863,39 @@ class TestExecutorFacts(AnsibleZuulTestCase): date_time = \ j[0]['plays'][0]['tasks'][0]['hosts']['localhost']['date_time'] self.assertEqual(18, len(date_time)) + build = self.getJobFromHistory('datetime-fact', result='SUCCESS') + with open(build.jobdir.job_output_file) as f: + output = f.read() + self.assertIn(f'Ansible version={self.ansible_major_minor}', + output) -class TestAnsibleCallbackConfigs(AnsibleZuulTestCase): +class TestExecutorFacts28(AnsibleZuulTestCase, ExecutorFactsMixin): + tenant_config_file = 'config/executor-facts/main28.yaml' + ansible_major_minor = '2.8' + +class TestExecutorFacts29(AnsibleZuulTestCase, ExecutorFactsMixin): + tenant_config_file = 'config/executor-facts/main29.yaml' + ansible_major_minor = '2.9' + + +class TestExecutorFacts5(AnsibleZuulTestCase, ExecutorFactsMixin): + tenant_config_file = 'config/executor-facts/main5.yaml' + ansible_major_minor = '2.12' + + +class TestExecutorFacts6(AnsibleZuulTestCase, ExecutorFactsMixin): + tenant_config_file = 'config/executor-facts/main6.yaml' + ansible_major_minor = '2.13' + + +class AnsibleCallbackConfigsMixin: config_file = 'zuul-executor-ansible-callback.conf' + + # These should be overridden in child classes. tenant_config_file = 'config/ansible-callbacks/main.yaml' + ansible_major_minor = 'X.Y' def test_ansible_callback_config(self): self.executor_server.keep_jobdir = True @@ -905,6 +934,39 @@ class TestAnsibleCallbackConfigs(AnsibleZuulTestCase): 'common-config/playbooks/callback_plugins/', c['callback_test_callback']['file_name']) self.assertTrue(os.path.isfile(callback_result_file)) + build = self.getJobFromHistory('callback-test', result='SUCCESS') + with open(build.jobdir.job_output_file) as f: + output = f.read() + self.assertIn(f'Ansible version={self.ansible_major_minor}', + output) + + +class TestAnsibleCallbackConfigs28(AnsibleZuulTestCase, + AnsibleCallbackConfigsMixin): + config_file = 'zuul-executor-ansible-callback.conf' + tenant_config_file = 'config/ansible-callbacks/main28.yaml' + ansible_major_minor = '2.8' + + +class TestAnsibleCallbackConfigs29(AnsibleZuulTestCase, + AnsibleCallbackConfigsMixin): + config_file = 'zuul-executor-ansible-callback.conf' + tenant_config_file = 'config/ansible-callbacks/main29.yaml' + ansible_major_minor = '2.9' + + +class TestAnsibleCallbackConfigs5(AnsibleZuulTestCase, + AnsibleCallbackConfigsMixin): + config_file = 'zuul-executor-ansible-callback.conf' + tenant_config_file = 'config/ansible-callbacks/main5.yaml' + ansible_major_minor = '2.12' + + +class TestAnsibleCallbackConfigs6(AnsibleZuulTestCase, + AnsibleCallbackConfigsMixin): + config_file = 'zuul-executor-ansible-callback.conf' + tenant_config_file = 'config/ansible-callbacks/main6.yaml' + ansible_major_minor = '2.13' class TestExecutorEnvironment(AnsibleZuulTestCase): diff --git a/tests/unit/test_inventory.py b/tests/unit/test_inventory.py index 8f5cca9ac..83a62a0e7 100644 --- a/tests/unit/test_inventory.py +++ b/tests/unit/test_inventory.py @@ -104,6 +104,7 @@ class TestInventoryGithub(TestInventoryBase): z_vars = inventory['all']['vars']['zuul'] self.assertIn('executor', z_vars) self.assertIn('src_root', z_vars['executor']) + self.assertIn('ansible_version', z_vars) self.assertIn('job', z_vars) self.assertIn('event_id', z_vars) self.assertEqual(z_vars['job'], 'single-inventory') @@ -137,6 +138,7 @@ class TestInventoryPythonPath(TestInventoryBase): z_vars = inventory['all']['vars']['zuul'] self.assertIn('executor', z_vars) self.assertIn('src_root', z_vars['executor']) + self.assertIn('ansible_version', z_vars) self.assertIn('job', z_vars) self.assertEqual(z_vars['job'], 'single-inventory') self.assertEqual(z_vars['message'], 'QQ==') @@ -167,6 +169,7 @@ class TestInventoryShellType(TestInventoryBase): z_vars = inventory['all']['vars']['zuul'] self.assertIn('executor', z_vars) self.assertIn('src_root', z_vars['executor']) + self.assertIn('ansible_version', z_vars) self.assertIn('job', z_vars) self.assertEqual(z_vars['job'], 'single-inventory') self.assertEqual(z_vars['message'], 'QQ==') @@ -195,6 +198,7 @@ class TestInventoryAutoPython(TestInventoryBase): self.assertIn('executor', z_vars) self.assertIn('src_root', z_vars['executor']) self.assertIn('job', z_vars) + self.assertEqual(z_vars['ansible_version'], '2.8') self.assertEqual(z_vars['job'], 'ansible-version28-inventory') self.assertEqual(z_vars['message'], 'QQ==') @@ -219,6 +223,7 @@ class TestInventoryAutoPython(TestInventoryBase): self.assertIn('executor', z_vars) self.assertIn('src_root', z_vars['executor']) self.assertIn('job', z_vars) + self.assertEqual(z_vars['ansible_version'], '2.9') self.assertEqual(z_vars['job'], 'ansible-version29-inventory') self.assertEqual(z_vars['message'], 'QQ==') @@ -243,12 +248,37 @@ class TestInventoryAutoPython(TestInventoryBase): self.assertIn('executor', z_vars) self.assertIn('src_root', z_vars['executor']) self.assertIn('job', z_vars) + self.assertEqual(z_vars['ansible_version'], '5') self.assertEqual(z_vars['job'], 'ansible-version5-inventory') self.assertEqual(z_vars['message'], 'QQ==') self.executor_server.release() self.waitUntilSettled() + def test_auto_python_ansible6_inventory(self): + inventory = self._get_build_inventory('ansible-version6-inventory') + + all_nodes = ('ubuntu-xenial',) + self.assertIn('all', inventory) + self.assertIn('hosts', inventory['all']) + self.assertIn('vars', inventory['all']) + for node_name in all_nodes: + self.assertIn(node_name, inventory['all']['hosts']) + node_vars = inventory['all']['hosts'][node_name] + self.assertEqual( + 'auto', node_vars['ansible_python_interpreter']) + + self.assertIn('zuul', inventory['all']['vars']) + z_vars = inventory['all']['vars']['zuul'] + self.assertIn('executor', z_vars) + self.assertIn('src_root', z_vars['executor']) + self.assertIn('job', z_vars) + self.assertEqual(z_vars['job'], 'ansible-version6-inventory') + self.assertEqual(z_vars['message'], 'QQ==') + + self.executor_server.release() + self.waitUntilSettled() + class TestInventory(TestInventoryBase): diff --git a/tests/unit/test_model_upgrade.py b/tests/unit/test_model_upgrade.py index 2004b317b..020045859 100644 --- a/tests/unit/test_model_upgrade.py +++ b/tests/unit/test_model_upgrade.py @@ -215,6 +215,44 @@ class TestModelUpgrade(ZuulTestCase): # code paths are exercised in existing tests since small secrets # don't use the blob store. + @model_version(8) + def test_model_8_9(self): + # This excercises the upgrade to nodeset_alternates + first = self.scheds.first + second = self.createScheduler() + second.start() + self.assertEqual(len(self.scheds), 2) + for _ in iterate_timeout(10, "until priming is complete"): + state_one = first.sched.local_layout_state.get("tenant-one") + if state_one: + break + + for _ in iterate_timeout( + 10, "all schedulers to have the same layout state"): + if (second.sched.local_layout_state.get( + "tenant-one") == state_one): + break + + self.fake_nodepool.pause() + with second.sched.layout_update_lock, second.sched.run_handler_lock: + A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled(matcher=[first]) + + self.model_test_component_info.model_api = 9 + with first.sched.layout_update_lock, first.sched.run_handler_lock: + self.fake_nodepool.unpause() + self.waitUntilSettled(matcher=[second]) + + self.waitUntilSettled() + self.assertHistory([ + dict(name='project-merge', result='SUCCESS', changes='1,1'), + dict(name='project-test1', result='SUCCESS', changes='1,1'), + dict(name='project-test2', result='SUCCESS', changes='1,1'), + dict(name='project1-project2-integration', + result='SUCCESS', changes='1,1'), + ], ordered=False) + class TestSemaphoreModelUpgrade(ZuulTestCase): tenant_config_file = 'config/semaphore/main.yaml' diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py index 66c508fea..5d8099686 100644 --- a/tests/unit/test_scheduler.py +++ b/tests/unit/test_scheduler.py @@ -36,6 +36,7 @@ from zuul.driver.gerrit import gerritreporter import zuul.scheduler import zuul.model import zuul.merger.merger +from zuul.lib import yamlutil as yaml from tests.base import ( SSLZuulTestCase, @@ -5343,6 +5344,11 @@ For CI problems and help debugging, contact ci@example.org""" self.assertIn('Error merging gerrit/org/project', B.messages[0]) self.assertNotIn('logs.example.com', B.messages[0]) self.assertNotIn('SKIPPED', B.messages[0]) + buildsets = list( + self.scheds.first.connections.connections[ + 'database'].getBuildsets()) + self.assertEqual(buildsets[0].result, 'MERGE_CONFLICT') + self.assertIn('This change or one of', buildsets[0].message) def test_submit_failure(self): A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') @@ -5357,6 +5363,44 @@ For CI problems and help debugging, contact ci@example.org""" 'database'].getBuildsets()) self.assertEqual(buildsets[0].result, 'MERGE_FAILURE') + @simple_layout('layouts/timer-freeze-job-failure.yaml') + def test_periodic_freeze_job_failure(self): + self.waitUntilSettled() + + for x in iterate_timeout(30, 'buildset complete'): + buildsets = list( + self.scheds.first.connections.connections[ + 'database'].getBuildsets()) + if buildsets: + break + # Stop queuing timer triggered jobs so that the assertions + # below don't race against more jobs being queued. + self.commitConfigUpdate('org/common-config', 'layouts/no-timer.yaml') + self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) + self.waitUntilSettled() + # If APScheduler is in mid-event when we remove the job, we + # can end up with one more event firing, so give it an extra + # second to settle. + time.sleep(3) + self.waitUntilSettled() + + self.assertEqual(buildsets[0].result, 'CONFIG_ERROR') + self.assertIn('Job project-test2 depends on project-test1 ' + 'which was not run', buildsets[0].message) + + @simple_layout('layouts/freeze-job-failure.yaml') + def test_freeze_job_failure(self): + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + buildsets = list( + self.scheds.first.connections.connections[ + 'database'].getBuildsets()) + self.assertEqual(buildsets[0].result, 'CONFIG_ERROR') + self.assertIn('Job project-test2 depends on project-test1 ' + 'which was not run', buildsets[0].message) + @simple_layout('layouts/nonvoting-pipeline.yaml') def test_nonvoting_pipeline(self): "Test that a nonvoting pipeline (experimental) can still report" @@ -6046,6 +6090,50 @@ For CI problems and help debugging, contact ci@example.org""" self.assertFalse(node['_lock']) self.assertEqual(node['state'], 'ready') + @simple_layout('layouts/nodeset-fallback.yaml') + def test_nodeset_fallback(self): + # Test that nodeset fallback works + self.executor_server.hold_jobs_in_build = True + + # Verify that we get the correct number and order of + # alternates from our nested config. + tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') + job = tenant.layout.getJob('check-job') + alts = job.flattenNodesetAlternatives(tenant.layout) + self.assertEqual(4, len(alts)) + self.assertEqual('fast-nodeset', alts[0].name) + self.assertEqual('', alts[1].name) + self.assertEqual('red-nodeset', alts[2].name) + self.assertEqual('blue-nodeset', alts[3].name) + + self.fake_nodepool.pause() + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + req = self.fake_nodepool.getNodeRequests()[0] + self.fake_nodepool.addFailRequest(req) + + self.fake_nodepool.unpause() + self.waitUntilSettled() + + build = self.getBuildByName('check-job') + inv_path = os.path.join(build.jobdir.root, 'ansible', 'inventory.yaml') + inventory = yaml.safe_load(open(inv_path, 'r')) + label = inventory['all']['hosts']['controller']['nodepool']['label'] + self.assertEqual('slow-label', label) + + self.executor_server.hold_jobs_in_build = False + self.executor_server.release() + self.waitUntilSettled() + + self.assertEqual(A.data['status'], 'NEW') + self.assertEqual(A.reported, 1) + self.assertNotIn('NODE_FAILURE', A.messages[0]) + self.assertHistory([ + dict(name='check-job', result='SUCCESS', changes='1,1'), + ], ordered=False) + @simple_layout('layouts/multiple-templates.yaml') def test_multiple_project_templates(self): # Test that applying multiple project templates to a project diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py index a89bb3007..3d41d74c6 100644 --- a/tests/unit/test_v3.py +++ b/tests/unit/test_v3.py @@ -2682,6 +2682,69 @@ class TestInRepoConfig(ZuulTestCase): self.assertIn('Debug information:', A.messages[0], "A should have debug info") + def test_nodeset_alternates_cycle(self): + in_repo_conf = textwrap.dedent( + """ + - nodeset: + name: red + alternatives: [blue] + - nodeset: + name: blue + alternatives: [red] + - job: + name: project-test1 + run: playbooks/project-test1.yaml + nodeset: blue + """) + + file_dict = {'.zuul.yaml': in_repo_conf} + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', + files=file_dict) + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertEqual(A.reported, 1) + self.assertIn("cycle detected", A.messages[0]) + + def test_nodeset_alternates_missing_from_nodeset(self): + in_repo_conf = textwrap.dedent( + """ + - nodeset: + name: red + alternatives: [blue] + - job: + name: project-test1 + run: playbooks/project-test1.yaml + """) + + file_dict = {'.zuul.yaml': in_repo_conf} + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', + files=file_dict) + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertEqual(A.reported, 1) + self.assertIn('nodeset "blue" was not found', A.messages[0]) + + def test_nodeset_alternates_missing_from_job(self): + in_repo_conf = textwrap.dedent( + """ + - job: + name: project-test1 + run: playbooks/project-test1.yaml + nodeset: + alternatives: [red] + """) + + file_dict = {'.zuul.yaml': in_repo_conf} + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', + files=file_dict) + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertEqual(A.reported, 1) + self.assertIn('nodeset "red" was not found', A.messages[0]) + @skipIfMultiScheduler() # See comment in TestInRepoConfigDir.scheduler_count for further # details. @@ -3733,9 +3796,9 @@ class TestInRepoJoin(ZuulTestCase): class FunctionalAnsibleMixIn(object): # A temporary class to hold new tests while others are disabled + # These should be overridden in child classes. tenant_config_file = 'config/ansible/main.yaml' - # This should be overriden in child classes. - ansible_version = '2.9' + ansible_major_minor = 'X.Y' def test_playbook(self): # This test runs a bit long and needs extra time. @@ -3826,6 +3889,7 @@ class FunctionalAnsibleMixIn(object): self.assertEqual(build_bubblewrap.result, 'SUCCESS') def test_repo_ansible(self): + self.executor_server.keep_jobdir = True A = self.fake_gerrit.addFakeChange('org/ansible', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() @@ -3835,18 +3899,31 @@ class FunctionalAnsibleMixIn(object): self.assertHistory([ dict(name='hello-ansible', result='SUCCESS', changes='1,1'), ]) + build = self.getJobFromHistory('hello-ansible', result='SUCCESS') + with open(build.jobdir.job_output_file) as f: + output = f.read() + self.assertIn(f'Ansible version={self.ansible_major_minor}', + output) class TestAnsible28(AnsibleZuulTestCase, FunctionalAnsibleMixIn): - ansible_version = '2.8' + tenant_config_file = 'config/ansible/main28.yaml' + ansible_major_minor = '2.8' class TestAnsible29(AnsibleZuulTestCase, FunctionalAnsibleMixIn): - ansible_version = '2.9' + tenant_config_file = 'config/ansible/main29.yaml' + ansible_major_minor = '2.9' class TestAnsible5(AnsibleZuulTestCase, FunctionalAnsibleMixIn): - ansible_version = '5' + tenant_config_file = 'config/ansible/main5.yaml' + ansible_major_minor = '2.12' + + +class TestAnsible6(AnsibleZuulTestCase, FunctionalAnsibleMixIn): + tenant_config_file = 'config/ansible/main6.yaml' + ansible_major_minor = '2.13' class TestPrePlaybooks(AnsibleZuulTestCase): @@ -7942,6 +8019,7 @@ class TestAnsibleVersion(AnsibleZuulTestCase): dict(name='ansible-28', result='SUCCESS', changes='1,1'), dict(name='ansible-29', result='SUCCESS', changes='1,1'), dict(name='ansible-5', result='SUCCESS', changes='1,1'), + dict(name='ansible-6', result='SUCCESS', changes='1,1'), ], ordered=False) @@ -7963,6 +8041,7 @@ class TestDefaultAnsibleVersion(AnsibleZuulTestCase): dict(name='ansible-28', result='SUCCESS', changes='1,1'), dict(name='ansible-29', result='SUCCESS', changes='1,1'), dict(name='ansible-5', result='SUCCESS', changes='1,1'), + dict(name='ansible-6', result='SUCCESS', changes='1,1'), ], ordered=False) diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py index ba1931436..f9de11a9b 100644 --- a/tests/unit/test_web.py +++ b/tests/unit/test_web.py @@ -341,6 +341,82 @@ class TestWeb(BaseTestWeb): self.assertEqual(1, len(data), data) self.assertEqual("org/project1", data[0]['project'], data) + @simple_layout('layouts/nodeset-alternatives.yaml') + def test_web_find_job_nodeset_alternatives(self): + # test a complex nodeset + data = self.get_url('api/tenant/tenant-one/job/test-job').json() + + self.assertEqual([ + {'abstract': False, + 'ansible_version': None, + 'attempts': 3, + 'branches': [], + 'cleanup_run': [], + 'deduplicate': 'auto', + 'dependencies': [], + 'description': None, + 'extra_variables': {}, + 'files': [], + 'final': False, + 'group_variables': {}, + 'host_variables': {}, + 'intermediate': False, + 'irrelevant_files': [], + 'match_on_config_updates': True, + 'name': 'test-job', + 'nodeset_alternatives': [{'alternatives': [], + 'groups': [], + 'name': 'fast-nodeset', + 'nodes': [{'aliases': [], + 'comment': None, + 'hold_job': None, + 'id': None, + 'label': 'fast-label', + 'name': 'controller', + 'requestor': None, + 'state': 'unknown', + 'tenant_name': None, + 'user_data': None}]}, + {'alternatives': [], + 'groups': [], + 'name': '', + 'nodes': [{'aliases': [], + 'comment': None, + 'hold_job': None, + 'id': None, + 'label': 'slow-label', + 'name': 'controller', + 'requestor': None, + 'state': 'unknown', + 'tenant_name': None, + 'user_data': None}]}], + 'override_checkout': None, + 'parent': 'base', + 'post_review': None, + 'post_run': [], + 'pre_run': [], + 'protected': None, + 'provides': [], + 'required_projects': [], + 'requires': [], + 'roles': [{'implicit': True, + 'project_canonical_name': + 'review.example.com/org/common-config', + 'target_name': 'common-config', + 'type': 'zuul'}], + 'run': [], + 'semaphores': [], + 'source_context': {'branch': 'master', + 'path': 'zuul.yaml', + 'project': 'org/common-config'}, + 'tags': [], + 'timeout': None, + 'variables': {}, + 'variant_description': '', + 'voting': True, + 'workspace_scheme': 'golang', + }], data) + def test_web_find_job(self): # can we fetch the variants for a single job data = self.get_url('api/tenant/tenant-one/job/project-test1').json() @@ -384,6 +460,7 @@ class TestWeb(BaseTestWeb): 'match_on_config_updates': True, 'final': False, 'nodeset': { + 'alternatives': [], 'groups': [], 'name': '', 'nodes': [{'comment': None, @@ -435,6 +512,7 @@ class TestWeb(BaseTestWeb): 'match_on_config_updates': True, 'final': False, 'nodeset': { + 'alternatives': [], 'groups': [], 'name': '', 'nodes': [{'comment': None, @@ -776,14 +854,11 @@ class TestWeb(BaseTestWeb): 'merge_mode': 'merge-resolve', 'pipelines': [{ 'name': 'check', - 'queue_name': None, 'jobs': jobs, }, { 'name': 'gate', - 'queue_name': 'integrated-overridden', 'jobs': jobs, }, {'name': 'post', - 'queue_name': None, 'jobs': [[ {'abstract': False, 'ansible_version': None, @@ -1066,7 +1141,7 @@ class TestWeb(BaseTestWeb): job_params = { 'job': 'project-test1', - 'ansible_version': '2.9', + 'ansible_version': '5', 'timeout': None, 'post_timeout': None, 'items': [], @@ -1074,6 +1149,7 @@ class TestWeb(BaseTestWeb): 'branch': 'master', 'cleanup_playbooks': [], 'nodeset': { + 'alternatives': [], 'groups': [], 'name': '', 'nodes': [ @@ -1164,14 +1240,15 @@ class TestWeb(BaseTestWeb): "noop") job_params = { - 'ansible_version': '2.9', + 'ansible_version': '5', 'branch': 'master', 'extra_vars': {}, 'group_vars': {}, 'host_vars': {}, 'items': [], 'job': 'noop', - 'nodeset': {'groups': [], 'name': '', 'nodes': []}, + 'nodeset': {'alternatives': [], + 'groups': [], 'name': '', 'nodes': []}, 'override_branch': None, 'override_checkout': None, 'post_timeout': None, diff --git a/web/src/containers/job/JobVariant.jsx b/web/src/containers/job/JobVariant.jsx index bec2276ef..9621cf333 100644 --- a/web/src/containers/job/JobVariant.jsx +++ b/web/src/containers/job/JobVariant.jsx @@ -107,7 +107,8 @@ class JobVariant extends React.Component { const jobInfos = [ 'source_context', 'builds', 'status', 'parent', 'attempts', 'timeout', 'semaphores', - 'nodeset', 'variables', 'override_checkout', + 'nodeset', 'nodeset_alternatives', 'variables', + 'override_checkout', ] jobInfos.forEach(key => { let label = key @@ -173,7 +174,15 @@ class JobVariant extends React.Component { ) nice_label = (<span><ClusterIcon /> Required nodes</span>) } - + if (label === 'nodeset_alternatives') { + value = value.map((alt, idx) => { + return (<> + {(idx > 0 ? <span>or</span>:<></>)} + <Nodeset nodeset={alt} /> + </>) + }) + nice_label = (<span><ClusterIcon /> Required nodes</span>) + } if (label === 'parent') { value = ( <Link to={tenant.linkPrefix + '/job/' + value}> diff --git a/zuul/ansible/6/action/__init__.py b/zuul/ansible/6/action/__init__.py new file mode 120000 index 000000000..4048e7ac1 --- /dev/null +++ b/zuul/ansible/6/action/__init__.py @@ -0,0 +1 @@ +../../base/action/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/6/action/command.py b/zuul/ansible/6/action/command.py new file mode 120000 index 000000000..56c6b636f --- /dev/null +++ b/zuul/ansible/6/action/command.py @@ -0,0 +1 @@ +../../base/action/command.py
\ No newline at end of file diff --git a/zuul/ansible/6/action/command.pyi b/zuul/ansible/6/action/command.pyi new file mode 120000 index 000000000..a003281ca --- /dev/null +++ b/zuul/ansible/6/action/command.pyi @@ -0,0 +1 @@ +../../base/action/command.pyi
\ No newline at end of file diff --git a/zuul/ansible/6/action/zuul_return.py b/zuul/ansible/6/action/zuul_return.py new file mode 120000 index 000000000..83c2fc619 --- /dev/null +++ b/zuul/ansible/6/action/zuul_return.py @@ -0,0 +1 @@ +../../base/action/zuul_return.py
\ No newline at end of file diff --git a/zuul/ansible/6/callback/__init__.py b/zuul/ansible/6/callback/__init__.py new file mode 120000 index 000000000..00b974388 --- /dev/null +++ b/zuul/ansible/6/callback/__init__.py @@ -0,0 +1 @@ +../../base/callback/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/6/callback/zuul_json.py b/zuul/ansible/6/callback/zuul_json.py new file mode 120000 index 000000000..b0a07779b --- /dev/null +++ b/zuul/ansible/6/callback/zuul_json.py @@ -0,0 +1 @@ +../../base/callback/zuul_json.py
\ No newline at end of file diff --git a/zuul/ansible/6/callback/zuul_stream.py b/zuul/ansible/6/callback/zuul_stream.py new file mode 120000 index 000000000..f75561bf4 --- /dev/null +++ b/zuul/ansible/6/callback/zuul_stream.py @@ -0,0 +1 @@ +../../base/callback/zuul_stream.py
\ No newline at end of file diff --git a/zuul/ansible/6/callback/zuul_unreachable.py b/zuul/ansible/6/callback/zuul_unreachable.py new file mode 120000 index 000000000..205baca6f --- /dev/null +++ b/zuul/ansible/6/callback/zuul_unreachable.py @@ -0,0 +1 @@ +../../base/callback/zuul_unreachable.py
\ No newline at end of file diff --git a/zuul/ansible/6/filter/__init__.py b/zuul/ansible/6/filter/__init__.py new file mode 120000 index 000000000..f80a4da61 --- /dev/null +++ b/zuul/ansible/6/filter/__init__.py @@ -0,0 +1 @@ +../../base/filter/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/6/filter/zuul_filters.py b/zuul/ansible/6/filter/zuul_filters.py new file mode 120000 index 000000000..d406e5fe6 --- /dev/null +++ b/zuul/ansible/6/filter/zuul_filters.py @@ -0,0 +1 @@ +../../base/filter/zuul_filters.py
\ No newline at end of file diff --git a/zuul/ansible/6/library/__init__.py b/zuul/ansible/6/library/__init__.py new file mode 120000 index 000000000..0b68ce0f4 --- /dev/null +++ b/zuul/ansible/6/library/__init__.py @@ -0,0 +1 @@ +../../base/library/__init__.py
\ No newline at end of file diff --git a/zuul/ansible/6/library/command.py b/zuul/ansible/6/library/command.py new file mode 120000 index 000000000..9c7633169 --- /dev/null +++ b/zuul/ansible/6/library/command.py @@ -0,0 +1 @@ +../../base/library/command.py
\ No newline at end of file diff --git a/zuul/ansible/6/library/zuul_console.py b/zuul/ansible/6/library/zuul_console.py new file mode 120000 index 000000000..7c905e0f9 --- /dev/null +++ b/zuul/ansible/6/library/zuul_console.py @@ -0,0 +1 @@ +../../base/library/zuul_console.py
\ No newline at end of file diff --git a/zuul/ansible/6/logconfig.py b/zuul/ansible/6/logconfig.py new file mode 120000 index 000000000..767cb2e81 --- /dev/null +++ b/zuul/ansible/6/logconfig.py @@ -0,0 +1 @@ +../logconfig.py
\ No newline at end of file diff --git a/zuul/ansible/6/paths.py b/zuul/ansible/6/paths.py new file mode 120000 index 000000000..dbdb1858e --- /dev/null +++ b/zuul/ansible/6/paths.py @@ -0,0 +1 @@ +../paths.py
\ No newline at end of file diff --git a/zuul/ansible/base/callback/zuul_stream.py b/zuul/ansible/base/callback/zuul_stream.py index f31983ed6..b5c14691b 100644 --- a/zuul/ansible/base/callback/zuul_stream.py +++ b/zuul/ansible/base/callback/zuul_stream.py @@ -43,13 +43,18 @@ import threading import time from ansible.plugins.callback import default +from ansible.module_utils._text import to_text from zuul.ansible import paths from zuul.ansible import logconfig -LOG_STREAM_PORT = int(os.environ.get("ZUUL_CONSOLE_PORT", 19885)) LOG_STREAM_VERSION = 0 +# This is intended to be only used for testing where we change the +# port so we can run another instance that doesn't conflict with one +# setup by the test environment +LOG_STREAM_PORT = int(os.environ.get("ZUUL_CONSOLE_PORT", 19885)) + def zuul_filter_result(result): """Remove keys from shell/command output. @@ -121,6 +126,21 @@ class CallbackModule(default.CallbackModule): self._logger = logging.getLogger('zuul.executor.ansible') def _log(self, msg, ts=None, job=True, executor=False, debug=False): + # With the default "linear" strategy (and likely others), + # Ansible will send the on_task_start callback, and then fork + # a worker process to execute that task. Since we spawn a + # thread in the on_task_start callback, we can end up emitting + # a log message in this method while Ansible is forking. If a + # forked process inherits a Python file object (i.e., stdout) + # that is locked by a thread that doesn't exist in the fork + # (i.e., this one), it can deadlock when trying to flush the + # file object. To minimize the chances of that happening, we + # should avoid using _display outside the main thread. + # Therefore: + + # Do not set executor=True from any calls from a thread + # spawned in this callback. + msg = msg.rstrip() if job: now = ts or datetime.datetime.now() @@ -143,10 +163,6 @@ class CallbackModule(default.CallbackModule): s.settimeout(None) return s except socket.timeout: - self._log( - "Timeout exception waiting for the logger. " - "Please check connectivity to [%s:%s]" - % (ip, port), executor=True) self._log_streamline( "localhost", "Timeout exception waiting for the logger. " @@ -155,16 +171,12 @@ class CallbackModule(default.CallbackModule): return None except Exception: if logger_retries % 10 == 0: - self._log("[%s] Waiting on logger" % host, - executor=True, debug=True) + self._log("[%s] Waiting on logger" % host) logger_retries += 1 time.sleep(0.1) continue def _read_log(self, host, ip, port, log_id, task_name, hosts): - self._log("[%s] Starting to log %s for task %s" - % (host, log_id, task_name), job=False, executor=True) - s = self._read_log_connect(host, ip, port) if s is None: # Can't connect; _read_log_connect() already logged an @@ -188,9 +200,6 @@ class CallbackModule(default.CallbackModule): return else: self._zuul_console_version = int(buff) - self._log('[%s] Reports streaming version: %d' % - (host, self._zuul_console_version), - job=False, executor=True) if self._zuul_console_version >= 1: msg = 's:%s\n' % log_id @@ -315,13 +324,13 @@ class CallbackModule(default.CallbackModule): hosts = self._get_task_hosts(task) for host, inventory_hostname in hosts: port = LOG_STREAM_PORT - if host in ('localhost', '127.0.0.1'): + if (host in ('localhost', '127.0.0.1')): # Don't try to stream from localhost continue ip = play_vars[host].get( 'ansible_host', play_vars[host].get( 'ansible_inventory_host')) - if ip in ('localhost', '127.0.0.1'): + if (ip in ('localhost', '127.0.0.1')): # Don't try to stream from localhost continue if play_vars[host].get('ansible_connection') in ('winrm',): @@ -349,6 +358,9 @@ class CallbackModule(default.CallbackModule): log_id = "%s-%s-%s" % ( self._task._uuid, count, log_host) + self._log("[%s] Starting to log %s for task %s" + % (host, log_id, task_name), + job=False, executor=True) streamer = threading.Thread( target=self._read_log, args=( host, ip, port, log_id, task_name, hosts)) @@ -369,7 +381,7 @@ class CallbackModule(default.CallbackModule): streamer.join(30) if streamer.is_alive(): msg = "[Zuul] Log Stream did not terminate" - self._log(msg, job=True, executor=True) + self._log(msg) self._streamers_stop = False def _process_result_for_localhost(self, result, is_task=True): @@ -492,8 +504,7 @@ class CallbackModule(default.CallbackModule): if result._task.loop and 'results' in result_dict: # items have their own events pass - - elif result_dict.get('msg', '').startswith('MODULE FAILURE'): + elif to_text(result_dict.get('msg', '')).startswith('MODULE FAILURE'): self._log_module_failure(result, result_dict) elif result._task.action == 'debug': # this is a debug statement, handle it special @@ -512,7 +523,7 @@ class CallbackModule(default.CallbackModule): # user provided. Note that msg may be a multi line block quote # so we handle that here as well. if keyname == 'msg': - msg_lines = result_dict['msg'].rstrip().split('\n') + msg_lines = to_text(result_dict['msg']).rstrip().split('\n') for msg_line in msg_lines: self._log(msg=msg_line) else: @@ -535,10 +546,18 @@ class CallbackModule(default.CallbackModule): elif result_dict.get('msg') == 'All items completed': self._log_message(result, result_dict['msg']) else: - self._log_message( - result, - "Runtime: {delta}".format( - **result_dict)) + if 'delta' in result_dict: + self._log_message( + result, + "Runtime: {delta}".format( + **result_dict)) + else: + # NOTE(ianw) 2022-08-24 : *Fairly* sure that you only + # fall into here when the call actually fails (and has + # not start/end time), but it is ignored by + # failed_when matching. + self._log_message(result, msg='ERROR (ignored)', + result_dict=result_dict) def v2_runner_item_on_ok(self, result): result_dict = dict(result._result) @@ -554,7 +573,7 @@ class CallbackModule(default.CallbackModule): # changes. loop_var = result_dict.get('ansible_loop_var', 'item') - if result_dict.get('msg', '').startswith('MODULE FAILURE'): + if to_text(result_dict.get('msg', '')).startswith('MODULE FAILURE'): self._log_module_failure(result, result_dict) elif result._task.action not in ('command', 'shell', 'win_command', 'win_shell'): @@ -597,7 +616,7 @@ class CallbackModule(default.CallbackModule): # changes. loop_var = result_dict.get('ansible_loop_var', 'item') - if result_dict.get('msg', '').startswith('MODULE FAILURE'): + if to_text(result_dict.get('msg', '')).startswith('MODULE FAILURE'): self._log_module_failure(result, result_dict) elif result._task.action not in ('command', 'shell', 'win_command', 'win_shell'): @@ -730,7 +749,13 @@ class CallbackModule(default.CallbackModule): msg = result_dict['msg'] result_dict = None if msg: - msg_lines = msg.rstrip().split('\n') + # ensure msg is a string; e.g. + # + # debug: + # msg: '{{ var }}' + # + # may not be! + msg_lines = to_text(msg).rstrip().split('\n') if len(msg_lines) > 1: self._log("{host} | {status}:".format( host=hostname, status=status)) diff --git a/zuul/configloader.py b/zuul/configloader.py index eb468518f..037fc48aa 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -488,14 +488,26 @@ class NodeSetParser(object): vs.Required('nodes'): to_list(str), } - nodeset = {vs.Required('nodes'): to_list(node), - 'groups': to_list(group), - '_source_context': model.SourceContext, - '_start_mark': model.ZuulMark, - } + real_nodeset = {vs.Required('nodes'): to_list(node), + 'groups': to_list(group), + } + + alt_nodeset = {vs.Required('alternatives'): + [vs.Any(real_nodeset, str)]} + top_nodeset = {'_source_context': model.SourceContext, + '_start_mark': model.ZuulMark, + } if not anonymous: - nodeset[vs.Required('name')] = str + top_nodeset[vs.Required('name')] = str + + top_real_nodeset = real_nodeset.copy() + top_real_nodeset.update(top_nodeset) + top_alt_nodeset = alt_nodeset.copy() + top_alt_nodeset.update(top_nodeset) + + nodeset = vs.Any(top_real_nodeset, top_alt_nodeset) + return vs.Schema(nodeset) def fromYaml(self, conf, anonymous=False): @@ -503,6 +515,24 @@ class NodeSetParser(object): self.anon_schema(conf) else: self.schema(conf) + + if 'alternatives' in conf: + return self.loadAlternatives(conf) + else: + return self.loadNodeset(conf) + + def loadAlternatives(self, conf): + ns = model.NodeSet(conf.get('name')) + ns.source_context = conf.get('_source_context') + ns.start_mark = conf.get('_start_mark') + for alt in conf['alternatives']: + if isinstance(alt, str): + ns.addAlternative(alt) + else: + ns.addAlternative(self.loadNodeset(alt)) + return ns + + def loadNodeset(self, conf): ns = model.NodeSet(conf.get('name')) ns.source_context = conf.get('_source_context') ns.start_mark = conf.get('_start_mark') @@ -981,8 +1011,6 @@ class ProjectTemplateParser(object): job_list = [vs.Any(str, job)] pipeline_contents = { - # TODO(tobiash): Remove pipeline specific queue after deprecation - 'queue': str, 'debug': bool, 'fail-fast': bool, 'jobs': job_list @@ -1014,8 +1042,6 @@ class ProjectTemplateParser(object): continue project_pipeline = model.ProjectPipelineConfig() project_template.pipelines[pipeline_name] = project_pipeline - # TODO(tobiash): Remove pipeline specific queue after deprecation - project_pipeline.queue_name = conf_pipeline.get('queue') project_pipeline.debug = conf_pipeline.get('debug') project_pipeline.fail_fast = conf_pipeline.get( 'fail-fast') @@ -1070,8 +1096,6 @@ class ProjectParser(object): job_list = [vs.Any(str, job)] pipeline_contents = { - # TODO(tobiash): Remove pipeline specific queue after deprecation - 'queue': str, 'debug': bool, 'fail-fast': bool, 'jobs': job_list @@ -1176,6 +1200,7 @@ class PipelineParser(object): 'success': 'success_actions', 'failure': 'failure_actions', 'merge-conflict': 'merge_conflict_actions', + 'config-error': 'config_error_actions', 'no-jobs': 'no_jobs_actions', 'disabled': 'disabled_actions', 'dequeue': 'dequeue_actions', @@ -1226,7 +1251,6 @@ class PipelineParser(object): 'failure-message': str, 'start-message': str, 'merge-conflict-message': str, - 'merge-failure-message': str, 'enqueue-message': str, 'no-jobs-message': str, 'footer-message': str, @@ -1249,8 +1273,8 @@ class PipelineParser(object): pipeline['reject'] = self.getDriverSchema('reject') pipeline['trigger'] = vs.Required(self.getDriverSchema('trigger')) for action in ['enqueue', 'start', 'success', 'failure', - 'merge-conflict', 'merge-failure', 'no-jobs', - 'disabled', 'dequeue']: + 'merge-conflict', 'no-jobs', 'disabled', + 'dequeue', 'config-error']: pipeline[action] = self.getDriverSchema('reporter') return vs.Schema(pipeline) @@ -1266,15 +1290,12 @@ class PipelineParser(object): pipeline.precedence = precedence pipeline.failure_message = conf.get('failure-message', "Build failed.") - # TODO: Remove in Zuul v6.0 - backwards_compat_merge_message = conf.get( - 'merge-failure-message', "Merge Failed.\n\nThis change or one " + pipeline.merge_conflict_message = conf.get( + 'merge-conflict-message', "Merge Failed.\n\nThis change or one " "of its cross-repo dependencies was unable to be " "automatically merged with the current state of its " "repository. Please rebase the change and upload a new " "patchset.") - pipeline.merge_conflict_message = conf.get( - 'merge-conflict-message', backwards_compat_merge_message) pipeline.success_message = conf.get('success-message', "Build succeeded.") @@ -1296,8 +1317,6 @@ class PipelineParser(object): # TODO: Remove in Zuul v6.0 # Make a copy to manipulate for backwards compat. conf_copy = conf.copy() - if 'merge-failure' in conf_copy and 'merge-conflict' not in conf_copy: - conf_copy['merge-conflict'] = conf_copy['merge-failure'] for conf_key, action in self.reporter_actions.items(): reporter_set = [] @@ -1318,6 +1337,10 @@ class PipelineParser(object): if not pipeline.merge_conflict_actions: pipeline.merge_conflict_actions = pipeline.failure_actions + # If config-error actions aren't explicit, use the failure actions + if not pipeline.config_error_actions: + pipeline.config_error_actions = pipeline.failure_actions + pipeline.disable_at = conf.get( 'disable-after-consecutive-failures', None) @@ -2395,6 +2418,10 @@ class TenantParser(object): # Now that all the jobs are loaded, verify references to other # config objects. + for nodeset in layout.nodesets.values(): + with reference_exceptions('nodeset', nodeset, + layout.loading_errors): + nodeset.validateReferences(layout) for jobs in layout.jobs.values(): for job in jobs: with reference_exceptions('job', job, layout.loading_errors): diff --git a/zuul/driver/elasticsearch/connection.py b/zuul/driver/elasticsearch/connection.py index e1ba50b37..eda15089f 100644 --- a/zuul/driver/elasticsearch/connection.py +++ b/zuul/driver/elasticsearch/connection.py @@ -95,7 +95,7 @@ class ElasticsearchConnection(BaseConnection): def setIndex(self, index): settings = { 'mappings': { - 'zuul': { + '_doc': { "properties": self.properties } } @@ -113,7 +113,6 @@ class ElasticsearchConnection(BaseConnection): source['@timestamp'] = datetime.utcfromtimestamp( int(source['start_time'])).strftime("%Y-%m-%dT%H:%M:%S.%fZ") d['_index'] = index - d['_type'] = 'zuul' d['_op_type'] = 'index' d['_source'] = source yield d diff --git a/zuul/driver/gerrit/gerritreporter.py b/zuul/driver/gerrit/gerritreporter.py index b99133dce..c38a9484a 100644 --- a/zuul/driver/gerrit/gerritreporter.py +++ b/zuul/driver/gerrit/gerritreporter.py @@ -36,6 +36,9 @@ class GerritReporter(BaseReporter): self._checks_api = action.pop('checks-api', None) self._labels = action + def __repr__(self): + return f"<GerritReporter: {self._action}>" + def report(self, item, phase1=True, phase2=True): """Send a message to gerrit.""" log = get_annotated_logger(self.log, item.event) diff --git a/zuul/driver/mqtt/mqttreporter.py b/zuul/driver/mqtt/mqttreporter.py index 4090bb082..5c95a19ea 100644 --- a/zuul/driver/mqtt/mqttreporter.py +++ b/zuul/driver/mqtt/mqttreporter.py @@ -30,6 +30,7 @@ class MQTTReporter(BaseReporter): def report(self, item, phase1=True, phase2=True): if not phase1: return + include_returned_data = self.config.get('include-returned-data') log = get_annotated_logger(self.log, item.event) log.debug("Report change %s, params %s", item.change, self.config) message = { @@ -80,6 +81,10 @@ class MQTTReporter(BaseReporter): 'artifacts': get_artifacts_from_result_data( build.result_data, logger=log) }) + if include_returned_data: + rdata = build.result_data.copy() + rdata.pop('zuul', None) + job_informations['returned_data'] = rdata # Report build data of retried builds if available retry_builds = item.current_build_set.getRetryBuildsForJob( @@ -144,4 +149,8 @@ def qosValue(value): def getSchema(): - return v.Schema({v.Required('topic'): topicValue, 'qos': qosValue}) + return v.Schema({ + v.Required('topic'): topicValue, + 'qos': qosValue, + 'include-returned-data': bool, + }) diff --git a/zuul/driver/sql/sqlreporter.py b/zuul/driver/sql/sqlreporter.py index cf75a7495..d16f50fcb 100644 --- a/zuul/driver/sql/sqlreporter.py +++ b/zuul/driver/sql/sqlreporter.py @@ -176,6 +176,8 @@ class SQLReporter(BaseReporter): start = datetime.datetime.fromtimestamp(start_time, tz=datetime.timezone.utc) buildset = build.build_set + if not buildset: + return db_buildset = db.getBuildset( tenant=buildset.item.pipeline.tenant.name, uuid=buildset.uuid) if not db_buildset: diff --git a/zuul/executor/server.py b/zuul/executor/server.py index e00612e9e..89f93b8c5 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -1837,6 +1837,11 @@ class AnsibleJob(object): if not self.jobdir.cleanup_playbooks: return + if not self.frozen_hostvars: + # Job failed before we could load the frozen hostvars. + # This means we can't run any cleanup playbooks. + return + # TODO: make this configurable cleanup_timeout = 300 @@ -2443,6 +2448,7 @@ class AnsibleJob(object): work_root=self.jobdir.work_root, result_data_file=self.jobdir.result_data_file, inventory_file=self.jobdir.inventory) + zuul_vars['ansible_version'] = self.ansible_version # Add playbook_context info zuul_vars['playbook_context'] = dict( diff --git a/zuul/lib/ansible-config.conf b/zuul/lib/ansible-config.conf index 5bc3bd325..35a9f59fe 100644 --- a/zuul/lib/ansible-config.conf +++ b/zuul/lib/ansible-config.conf @@ -1,6 +1,6 @@ # This file describes the currently supported ansible versions [common] -default_version = 2.9 +default_version = 5 # OpenStackSDK 0.99.0 coincides with CORS header problems in some providers requirements = openstacksdk<0.99 openshift jmespath google-cloud-storage pywinrm boto3 azure-storage-blob ibm-cos-sdk netaddr passlib @@ -8,11 +8,16 @@ requirements = openstacksdk<0.99 openshift jmespath google-cloud-storage pywinrm # Ansible 2.8.16 breaks the k8s connection plugin # Jinja 3.1.1 is incompatible with 2.8 requirements = ansible>=2.8,<2.9,!=2.8.16 Jinja2<3.1.0 +deprecated = true [2.9] # Ansible 2.9.14 breaks the k8s connection plugin # https://github.com/ansible/ansible/issues/72171 requirements = ansible>=2.9,<2.10,!=2.9.14 +deprecated = true [5] requirements = ansible>=5.0,<6.0 + +[6] +requirements = ansible>=6.0,<7.0 diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py index 31c4b54d7..365435f3d 100644 --- a/zuul/manager/__init__.py +++ b/zuul/manager/__init__.py @@ -133,26 +133,19 @@ class PipelineManager(metaclass=ABCMeta): for project_name, project_configs in layout_project_configs.items(): (trusted, project) = tenant.getProject(project_name) - project_queue_name = None - pipeline_queue_name = None + queue_name = None project_in_pipeline = False for project_config in layout.getAllProjectConfigs(project_name): project_pipeline_config = project_config.pipelines.get( self.pipeline.name) - if not project_queue_name: - project_queue_name = project_config.queue_name + if not queue_name: + queue_name = project_config.queue_name if project_pipeline_config is None: continue project_in_pipeline = True - if not pipeline_queue_name: - pipeline_queue_name = project_pipeline_config.queue_name if not project_in_pipeline: continue - # Note: we currently support queue name per pipeline and per - # project while project has precedence. - queue_name = project_queue_name or pipeline_queue_name - if not queue_name: continue if queue_name in change_queues: @@ -322,9 +315,10 @@ class PipelineManager(metaclass=ABCMeta): (item, ret)) def reportNormalBuildsetEnd(self, build_set, action, final, result=None): - # Report a buildset end, but only if there are jobs - if (build_set.job_graph and - len(build_set.job_graph.jobs) > 0): + # Report a buildset end if there are jobs or errors + if ((build_set.job_graph and len(build_set.job_graph.jobs) > 0) or + build_set.config_errors or + build_set.unable_to_merge): self.sql.reportBuildsetEnd(build_set, action, final, result) @@ -657,13 +651,12 @@ class PipelineManager(metaclass=ABCMeta): def getQueueConfig(self, project): layout = self.pipeline.tenant.layout - pipeline_queue_name = None - project_queue_name = None + queue_name = None for project_config in layout.getAllProjectConfigs( project.canonical_name ): - if not project_queue_name: - project_queue_name = project_config.queue_name + if not queue_name: + queue_name = project_config.queue_name project_pipeline_config = project_config.pipelines.get( self.pipeline.name) @@ -671,16 +664,6 @@ class PipelineManager(metaclass=ABCMeta): if project_pipeline_config is None: continue - # TODO(simonw): Remove pipeline_queue_name after deprecation - if not pipeline_queue_name: - pipeline_queue_name = project_pipeline_config.queue_name - - # Note: we currently support queue name per pipeline and per - # project while project has precedence. - queue_name = project_queue_name or pipeline_queue_name - if queue_name is None: - return None - return layout.queues.get(queue_name) def canProcessCycle(self, project): @@ -879,23 +862,28 @@ class PipelineManager(metaclass=ABCMeta): else: relative_priority = 0 for job in jobs: - provider = self._getPausedParentProvider(build_set, job) - priority = self._calculateNodeRequestPriority(build_set, job) - tenant_name = build_set.item.pipeline.tenant.name - pipeline_name = build_set.item.pipeline.name - req = self.sched.nodepool.requestNodes( - build_set.uuid, job, tenant_name, pipeline_name, provider, - priority, relative_priority, event=item.event) - log.debug("Adding node request %s for job %s to item %s", - req, job, item) - build_set.setJobNodeRequestID(job.name, req.id) - if req.fulfilled: - nodeset = self.sched.nodepool.getNodeSet(req, job.nodeset) - build_set.jobNodeRequestComplete(req.job_name, nodeset) - else: - job.setWaitingStatus(f'node request: {req.id}') + self._makeNodepoolRequest(log, build_set, job, relative_priority) return True + def _makeNodepoolRequest(self, log, build_set, job, relative_priority, + alternative=0): + provider = self._getPausedParentProvider(build_set, job) + priority = self._calculateNodeRequestPriority(build_set, job) + tenant_name = build_set.item.pipeline.tenant.name + pipeline_name = build_set.item.pipeline.name + item = build_set.item + req = self.sched.nodepool.requestNodes( + build_set.uuid, job, tenant_name, pipeline_name, provider, + priority, relative_priority, event=item.event) + log.debug("Adding node request %s for job %s to item %s", + req, job, item) + build_set.setJobNodeRequestID(job.name, req.id) + if req.fulfilled: + nodeset = self.sched.nodepool.getNodeSet(req, job.nodeset) + build_set.jobNodeRequestComplete(req.job_name, nodeset) + else: + job.setWaitingStatus(f'node request: {req.id}') + def _getPausedParent(self, build_set, job): job_graph = build_set.job_graph if job_graph: @@ -1908,17 +1896,46 @@ class PipelineManager(metaclass=ABCMeta): build_set.setExtraRepoState(event.repo_state) build_set.repo_state_state = build_set.COMPLETE + def _handleNodeRequestFallback(self, log, build_set, job, old_request): + if len(job.nodeset_alternatives) <= job.nodeset_index + 1: + # No alternatives to fall back upon + return False + + # Increment the nodeset index and remove the old request + with job.activeContext(self.current_context): + job.nodeset_index = job.nodeset_index + 1 + + log.info("Re-attempting node request for job " + f"{job.name} of item {build_set.item} " + f"with nodeset alternative {job.nodeset_index}") + + build_set.removeJobNodeRequestID(job.name) + + # Make a new request + if self.sched.globals.use_relative_priority: + relative_priority = build_set.item.getNodePriority() + else: + relative_priority = 0 + log = build_set.item.annotateLogger(self.log) + self._makeNodepoolRequest(log, build_set, job, relative_priority) + return True + def onNodesProvisioned(self, request, nodeset, build_set): - # TODOv3(jeblair): handle provisioning failure here log = get_annotated_logger(self.log, request.event_id) self.reportPipelineTiming('node_request_time', request.created_time) - if nodeset is not None: - build_set.jobNodeRequestComplete(request.job_name, nodeset) + job = build_set.item.getJob(request.job_name) + # First see if we need to retry the request if not request.fulfilled: log.info("Node request %s: failure for %s", request, request.job_name) - job = build_set.item.getJob(request.job_name) + if self._handleNodeRequestFallback(log, build_set, job, request): + return + # No more fallbacks -- tell the buildset the request is complete + if nodeset is not None: + build_set.jobNodeRequestComplete(request.job_name, nodeset) + # Put a fake build through the cycle to clean it up. + if not request.fulfilled: fakebuild = build_set.item.setNodeRequestFailure(job) try: self.sql.reportBuildEnd( @@ -2032,9 +2049,8 @@ class PipelineManager(metaclass=ABCMeta): item.setReportedResult('NO_JOBS') elif item.getConfigErrors(): log.debug("Invalid config for change %s", item.change) - # TODOv3(jeblair): consider a new reporter action for this - action = 'merge-conflict' - actions = self.pipeline.merge_conflict_actions + action = 'config-error' + actions = self.pipeline.config_error_actions item.setReportedResult('CONFIG_ERROR') elif item.didMergerFail(): log.debug("Merge conflict") diff --git a/zuul/manager/shared.py b/zuul/manager/shared.py index db8735289..4be107db9 100644 --- a/zuul/manager/shared.py +++ b/zuul/manager/shared.py @@ -71,27 +71,19 @@ class SharedQueuePipelineManager(PipelineManager, metaclass=ABCMeta): for project_name, project_configs in layout_project_configs.items(): (trusted, project) = tenant.getProject(project_name) - project_queue_name = None - pipeline_queue_name = None + queue_name = None project_in_pipeline = False for project_config in layout.getAllProjectConfigs(project_name): project_pipeline_config = project_config.pipelines.get( self.pipeline.name) - if not project_queue_name: - project_queue_name = project_config.queue_name + if not queue_name: + queue_name = project_config.queue_name if project_pipeline_config is None: continue project_in_pipeline = True - # TODO(tobiash): Remove pipeline_queue_name after deprecation - if not pipeline_queue_name: - pipeline_queue_name = project_pipeline_config.queue_name if not project_in_pipeline: continue - # Note: we currently support queue name per pipeline and per - # project while project has precedence. - queue_name = project_queue_name or pipeline_queue_name - # Check if the queue is global or per branch queue = layout.queues.get(queue_name) per_branch = queue and queue.per_branch diff --git a/zuul/model.py b/zuul/model.py index 0d889f557..f66a5e875 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -1383,6 +1383,7 @@ class NodeSet(ConfigObject): self.name = name or '' self.nodes = OrderedDict() self.groups = OrderedDict() + self.alternatives = [] def __ne__(self, other): return not self.__eq__(other) @@ -1391,7 +1392,9 @@ class NodeSet(ConfigObject): if not isinstance(other, NodeSet): return False return (self.name == other.name and - self.nodes == other.nodes) + self.nodes == other.nodes and + self.groups == other.groups and + self.alternatives == other.alternatives) def toDict(self): d = {} @@ -1402,6 +1405,12 @@ class NodeSet(ConfigObject): d['groups'] = [] for group in self.groups.values(): d['groups'].append(group.toDict()) + d['alternatives'] = [] + for alt in self.alternatives: + if isinstance(alt, NodeSet): + d['alternatives'].append(alt.toDict()) + else: + d['alternatives'].append(alt) return d @classmethod @@ -1411,6 +1420,12 @@ class NodeSet(ConfigObject): nodeset.addNode(Node.fromDict(node)) for group in data["groups"]: nodeset.addGroup(Group.fromDict(group)) + for alt in data.get('alternatives', []): + if isinstance(alt, str): + if isinstance(alt, str): + nodeset.addAlternative(alt) + else: + nodeset.addAlternative(NodeSet.fromDict(alt)) return nodeset def copy(self): @@ -1419,6 +1434,11 @@ class NodeSet(ConfigObject): n.addNode(Node(node.name, node.label)) for name, group in self.groups.items(): n.addGroup(Group(group.name, group.nodes[:])) + for alt in self.alternatives: + if isinstance(alt, str): + n.addAlternative(alt) + else: + n.addAlternative(alt.copy()) return n def addNode(self, node): @@ -1438,6 +1458,36 @@ class NodeSet(ConfigObject): def getGroups(self): return list(self.groups.values()) + def addAlternative(self, alt): + self.alternatives.append(alt) + + def flattenAlternatives(self, layout): + alts = [] + history = [] + self._flattenAlternatives(layout, self, alts, history) + return alts + + def _flattenAlternatives(self, layout, nodeset, + alternatives, history): + if isinstance(nodeset, str): + # This references an existing named nodeset in the layout. + ns = layout.nodesets.get(nodeset) + if ns is None: + raise Exception(f'The nodeset "{nodeset}" was not found.') + else: + ns = nodeset + if ns in history: + raise Exception(f'Nodeset cycle detected on "{nodeset}"') + history.append(ns) + if ns.alternatives: + for alt in ns.alternatives: + self._flattenAlternatives(layout, alt, alternatives, history) + else: + alternatives.append(ns) + + def validateReferences(self, layout): + self.flattenAlternatives(layout) + def __repr__(self): if self.name: name = self.name + ' ' @@ -2038,7 +2088,8 @@ class FrozenJob(zkobject.ZKObject): 'dependencies', 'inheritance_path', 'name', - 'nodeset', + 'nodeset_alternatives', + 'nodeset_index', 'override_branch', 'override_checkout', 'post_timeout', @@ -2149,8 +2200,8 @@ class FrozenJob(zkobject.ZKObject): if not hasattr(self, k): continue v = getattr(self, k) - if k == 'nodeset': - v = v.toDict() + if k == 'nodeset_alternatives': + v = [alt.toDict() for alt in v] elif k == 'dependencies': # frozenset of JobDependency v = [dep.toDict() for dep in v] @@ -2173,6 +2224,9 @@ class FrozenJob(zkobject.ZKObject): v = {'storage': 'local', 'data': v} data[k] = v + if (COMPONENT_REGISTRY.model_api < 9): + data['nodeset'] = data['nodeset_alternatives'][0] + # Use json_dumps to strip any ZuulMark entries return json_dumps(data, sort_keys=True).encode("utf8") @@ -2183,13 +2237,18 @@ class FrozenJob(zkobject.ZKObject): if 'deduplicate' not in data: data['deduplicate'] = 'auto' - if hasattr(self, 'nodeset'): - nodeset = self.nodeset + # MODEL_API < 9 + if data.get('nodeset'): + data['nodeset_alternatives'] = [data['nodeset']] + data['nodeset_index'] = 0 + del data['nodeset'] + + if hasattr(self, 'nodeset_alternatives'): + alts = self.nodeset_alternatives else: - nodeset = data.get('nodeset') - if nodeset: - nodeset = NodeSet.fromDict(nodeset) - data['nodeset'] = nodeset + alts = data.get('nodeset_alternatives', []) + alts = [NodeSet.fromDict(alt) for alt in alts] + data['nodeset_alternatives'] = alts if hasattr(self, 'dependencies'): data['dependencies'] = self.dependencies @@ -2250,6 +2309,12 @@ class FrozenJob(zkobject.ZKObject): return val @property + def nodeset(self): + if self.nodeset_alternatives: + return self.nodeset_alternatives[self.nodeset_index] + return None + + @property def parent_data(self): return self._getJobData('_parent_data') @@ -2459,12 +2524,11 @@ class Job(ConfigObject): d['parent'] = self.parent else: d['parent'] = tenant.default_base_job - if isinstance(self.nodeset, str): - ns = tenant.layout.nodesets.get(self.nodeset) - else: - ns = self.nodeset - if ns: - d['nodeset'] = ns.toDict() + alts = self.flattenNodesetAlternatives(tenant.layout) + if len(alts) == 1 and len(alts[0]): + d['nodeset'] = alts[0].toDict() + elif len(alts) > 1: + d['nodeset_alternatives'] = [x.toDict() for x in alts] if self.ansible_version: d['ansible_version'] = self.ansible_version else: @@ -2629,6 +2693,17 @@ class Job(ConfigObject): secrets.append(secret_value) playbook['secrets'][secret_key] = len(secrets) - 1 + def flattenNodesetAlternatives(self, layout): + nodeset = self.nodeset + if isinstance(nodeset, str): + # This references an existing named nodeset in the layout. + ns = layout.nodesets.get(nodeset) + if ns is None: + raise Exception(f'The nodeset "{nodeset}" was not found.') + else: + ns = nodeset + return ns.flattenAlternatives(layout) + def freezeJob(self, context, tenant, layout, item, redact_secrets_and_keys): buildset = item.current_build_set @@ -2640,6 +2715,9 @@ class Job(ConfigObject): attributes.discard('secrets') attributes.discard('affected_projects') attributes.discard('config_hash') + # Nodeset alternatives are flattened at this point + attributes.discard('nodeset_alternatives') + attributes.discard('nodeset_index') secrets = [] for k in attributes: # If this is a config object, it's frozen, so it's @@ -2663,6 +2741,8 @@ class Job(ConfigObject): for pb in v: self._deduplicateSecrets(context, secrets, pb) kw[k] = v + kw['nodeset_alternatives'] = self.flattenNodesetAlternatives(layout) + kw['nodeset_index'] = 0 kw['secrets'] = secrets kw['affected_projects'] = self._getAffectedProjects(tenant) kw['config_hash'] = self.getConfigHash(tenant) @@ -2735,7 +2815,7 @@ class Job(ConfigObject): if self._get('cleanup_run') is not None: self.cleanup_run = self.freezePlaybooks(self.cleanup_run, layout) - def getNodeSet(self, layout): + def getNodeset(self, layout): if isinstance(self.nodeset, str): # This references an existing named nodeset in the layout. ns = layout.nodesets.get(self.nodeset) @@ -2752,14 +2832,14 @@ class Job(ConfigObject): if not self.isBase() and self.parent: layout.getJob(self.parent) - ns = self.getNodeSet(layout) - if layout.tenant.max_nodes_per_job != -1 and \ - len(ns) > layout.tenant.max_nodes_per_job: - raise Exception( - 'The job "{job}" exceeds tenant ' - 'max-nodes-per-job {maxnodes}.'.format( - job=self.name, - maxnodes=layout.tenant.max_nodes_per_job)) + for ns in self.flattenNodesetAlternatives(layout): + if layout.tenant.max_nodes_per_job != -1 and \ + len(ns) > layout.tenant.max_nodes_per_job: + raise Exception( + 'The job "{job}" exceeds tenant ' + 'max-nodes-per-job {maxnodes}.'.format( + job=self.name, + maxnodes=layout.tenant.max_nodes_per_job)) for dependency in self.dependencies: layout.getJob(dependency.name) @@ -2956,7 +3036,7 @@ class Job(ConfigObject): self.addRoles(other.roles) # Freeze the nodeset - self.nodeset = self.getNodeSet(layout) + self.nodeset = self.getNodeset(layout) # Pass secrets to parents secrets_for_parents = [s for s in other.secrets if s.pass_to_parent] @@ -6619,7 +6699,6 @@ class ProjectPipelineConfig(ConfigObject): def __init__(self): super(ProjectPipelineConfig, self).__init__() self.job_list = JobList() - self.queue_name = None self.debug = False self.debug_messages = [] self.fail_fast = None @@ -6631,8 +6710,6 @@ class ProjectPipelineConfig(ConfigObject): def update(self, other): if not isinstance(other, ProjectPipelineConfig): raise Exception("Unable to update from %s" % (other,)) - if self.queue_name is None: - self.queue_name = other.queue_name if other.debug: self.debug = other.debug if self.fail_fast is None: @@ -6647,7 +6724,6 @@ class ProjectPipelineConfig(ConfigObject): def toDict(self): d = {} - d['queue_name'] = self.queue_name return d diff --git a/zuul/model_api.py b/zuul/model_api.py index 0534ee9c4..27d4a2b07 100644 --- a/zuul/model_api.py +++ b/zuul/model_api.py @@ -14,4 +14,4 @@ # When making ZK schema changes, increment this and add a record to # docs/developer/model-changelog.rst -MODEL_API = 8 +MODEL_API = 9 diff --git a/zuul/reporter/__init__.py b/zuul/reporter/__init__.py index 9cd60bff6..5af48abc6 100644 --- a/zuul/reporter/__init__.py +++ b/zuul/reporter/__init__.py @@ -135,6 +135,7 @@ class BaseReporter(object, metaclass=abc.ABCMeta): 'failure': self._formatItemReportFailure, 'merge-conflict': self._formatItemReportMergeConflict, 'merge-failure': self._formatItemReportMergeFailure, + 'config-error': self._formatItemReportConfigError, 'no-jobs': self._formatItemReportNoJobs, 'disabled': self._formatItemReportDisabled, 'dequeue': self._formatItemReportDequeue, @@ -226,6 +227,13 @@ class BaseReporter(object, metaclass=abc.ABCMeta): def _formatItemReportMergeFailure(self, item, with_jobs=True): return 'This change was not merged by the code review system.\n' + def _formatItemReportConfigError(self, item, with_jobs=True): + if item.getConfigErrors(): + msg = str(item.getConfigErrors()[0].error) + else: + msg = "Unknown configuration error" + return msg + def _formatItemReportNoJobs(self, item, with_jobs=True): status_url = get_default(self.connection.sched.config, 'web', 'status_url', '') |