diff options
author | Jenkins <jenkins@review.openstack.org> | 2014-08-15 18:16:08 +0000 |
---|---|---|
committer | Gerrit Code Review <review@openstack.org> | 2014-08-15 18:16:08 +0000 |
commit | c878c98977a5531af206f41bd3b34385d9306d78 (patch) | |
tree | 38cbaa0517029159a2e845f0c50132f7c65625a7 | |
parent | 4e0cdeb2db2e2c8d6f2b89bf1c7edfc7c31b034c (diff) | |
parent | c494d5456bd691c988ae02572612615e358c1e7d (diff) | |
download | zuul-c878c98977a5531af206f41bd3b34385d9306d78.tar.gz |
Merge "Add a Zuul trigger"
-rw-r--r-- | doc/source/triggers.rst | 8 | ||||
-rw-r--r-- | doc/source/zuul.rst | 27 | ||||
-rwxr-xr-x | tests/base.py | 8 | ||||
-rw-r--r-- | tests/fixtures/layout-zuultrigger-enqueued.yaml | 53 | ||||
-rw-r--r-- | tests/fixtures/layout-zuultrigger-merged.yaml | 53 | ||||
-rwxr-xr-x | tests/test_scheduler.py | 1 | ||||
-rw-r--r-- | tests/test_zuultrigger.py | 104 | ||||
-rwxr-xr-x | zuul/cmd/server.py | 4 | ||||
-rw-r--r-- | zuul/layoutvalidator.py | 12 | ||||
-rw-r--r-- | zuul/lib/gerrit.py | 17 | ||||
-rw-r--r-- | zuul/model.py | 16 | ||||
-rw-r--r-- | zuul/scheduler.py | 14 | ||||
-rw-r--r-- | zuul/trigger/gerrit.py | 8 | ||||
-rw-r--r-- | zuul/trigger/zuultrigger.py | 117 |
14 files changed, 434 insertions, 8 deletions
diff --git a/doc/source/triggers.rst b/doc/source/triggers.rst index c4485bf5d..dd650f2ff 100644 --- a/doc/source/triggers.rst +++ b/doc/source/triggers.rst @@ -4,8 +4,7 @@ Triggers ======== The process of merging a change starts with proposing a change to be -merged. Primarily, Zuul supports Gerrit as a triggering system, as -well as a facility for triggering jobs based on a timer. +merged. Primarily, Zuul supports Gerrit as a triggering system. Zuul's design is modular, so alternate triggering and reporting systems can be supported. @@ -40,3 +39,8 @@ Timer A simple timer trigger is available as well. It supports triggering jobs in a pipeline based on cron-style time instructions. + +Zuul +---- + +The Zuul trigger generates events based on internal actions in Zuul. diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst index 03b00b311..cdfa4df6d 100644 --- a/doc/source/zuul.rst +++ b/doc/source/zuul.rst @@ -389,7 +389,7 @@ explanation of each of the parameters:: DependentPipelineManager, see: :doc:`gating`. **trigger** - Exactly one trigger source must be supplied for each pipeline. + At least one trigger source must be supplied for each pipeline. Triggers are not exclusive -- matching events may be placed in multiple pipelines, and they will behave independently in each of the pipelines they match. You may select from the following: @@ -475,6 +475,31 @@ explanation of each of the parameters:: supported, not the symbolic names. Example: ``0 0 * * *`` runs at midnight. + **zuul** + This trigger supplies events generated internally by Zuul. + Multiple events may be listed. + + *event* + The event name. Currently supported: + + *project-change-merged* when Zuul merges a change to a project, + it generates this event for every open change in the project. + + *parent-change-enqueued* when Zuul enqueues a change into any + pipeline, it generates this event for every child of that + change. + + *pipeline* + Only available for ``parent-change-enqueued`` events. This is the + name of the pipeline in which the parent change was enqueued. + + *require-approval* + This may be used for any event. It requires that a certain kind + of approval be present for the current patchset of the change (the + approval could be added by the event in question). It follows the + same syntax as the "approval" pipeline requirement below. + + **require** If this section is present, it established pre-requisites for any kind of item entering the Pipeline. Regardless of how the item is diff --git a/tests/base.py b/tests/base.py index a86de82d7..1b8294486 100755 --- a/tests/base.py +++ b/tests/base.py @@ -52,6 +52,7 @@ import zuul.reporter.gerrit import zuul.reporter.smtp import zuul.trigger.gerrit import zuul.trigger.timer +import zuul.trigger.zuultrigger FIXTURE_DIR = os.path.join(os.path.dirname(__file__), 'fixtures') @@ -401,6 +402,11 @@ class FakeGerrit(object): return change.query() return {} + def simpleQuery(self, query): + # This is currently only used to return all open changes for a + # project + return [change.query() for change in self.changes.values()] + def startWatching(self, *args, **kw): pass @@ -906,6 +912,8 @@ class ZuulTestCase(testtools.TestCase): self.sched.registerTrigger(self.gerrit) self.timer = zuul.trigger.timer.Timer(self.config, self.sched) self.sched.registerTrigger(self.timer) + self.zuultrigger = zuul.trigger.zuultrigger.ZuulTrigger(self.config, self.sched) + self.sched.registerTrigger(self.zuultrigger) self.sched.registerReporter( zuul.reporter.gerrit.Reporter(self.gerrit)) diff --git a/tests/fixtures/layout-zuultrigger-enqueued.yaml b/tests/fixtures/layout-zuultrigger-enqueued.yaml new file mode 100644 index 000000000..8babd9e71 --- /dev/null +++ b/tests/fixtures/layout-zuultrigger-enqueued.yaml @@ -0,0 +1,53 @@ +pipelines: + - name: check + manager: IndependentPipelineManager + source: gerrit + require: + approval: + - verified: -1 + trigger: + gerrit: + - event: patchset-created + zuul: + - event: parent-change-enqueued + pipeline: gate + success: + gerrit: + verified: 1 + failure: + gerrit: + verified: -1 + + - name: gate + manager: DependentPipelineManager + failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures + source: gerrit + require: + approval: + - verified: 1 + trigger: + gerrit: + - event: comment-added + approval: + - approved: 1 + zuul: + - event: parent-change-enqueued + pipeline: gate + success: + gerrit: + verified: 2 + submit: true + failure: + gerrit: + verified: -2 + start: + gerrit: + verified: 0 + precedence: high + +projects: + - name: org/project + check: + - project-check + gate: + - project-gate diff --git a/tests/fixtures/layout-zuultrigger-merged.yaml b/tests/fixtures/layout-zuultrigger-merged.yaml new file mode 100644 index 000000000..657700dfe --- /dev/null +++ b/tests/fixtures/layout-zuultrigger-merged.yaml @@ -0,0 +1,53 @@ +pipelines: + - name: check + manager: IndependentPipelineManager + source: gerrit + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + verified: 1 + failure: + gerrit: + verified: -1 + + - name: gate + manager: DependentPipelineManager + failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures + source: gerrit + trigger: + gerrit: + - event: comment-added + approval: + - approved: 1 + success: + gerrit: + verified: 2 + submit: true + failure: + gerrit: + verified: -2 + start: + gerrit: + verified: 0 + precedence: high + + - name: merge-check + manager: IndependentPipelineManager + source: gerrit + trigger: + zuul: + - event: project-change-merged + merge-failure: + gerrit: + verified: -1 + +projects: + - name: org/project + check: + - project-check + gate: + - project-gate + merge-check: + - noop diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 247bde1e9..9baa824cc 100755 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1742,6 +1742,7 @@ class TestScheduler(ZuulTestCase): sched = zuul.scheduler.Scheduler() sched.registerTrigger(None, 'gerrit') sched.registerTrigger(None, 'timer') + sched.registerTrigger(None, 'zuul') sched.testConfig(self.config.get('zuul', 'layout_config')) def test_build_description(self): diff --git a/tests/test_zuultrigger.py b/tests/test_zuultrigger.py new file mode 100644 index 000000000..eb8fdc533 --- /dev/null +++ b/tests/test_zuultrigger.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python + +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import time + +from tests.base import ZuulTestCase + +logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s %(name)-32s ' + '%(levelname)-8s %(message)s') + + +class TestZuulTrigger(ZuulTestCase): + """Test Zuul Trigger""" + + def test_zuul_trigger_parent_change_enqueued(self): + "Test Zuul trigger event: parent-change-enqueued" + self.config.set('zuul', 'layout_config', + 'tests/fixtures/layout-zuultrigger-enqueued.yaml') + self.sched.reconfigure(self.config) + self.registerJobs() + + # This test has the following three changes: + # B1 -> A; B2 -> A + # When A is enqueued in the gate, B1 and B2 should both attempt + # to be enqueued in both pipelines. B1 should end up in check + # and B2 in gate because of differing pipeline requirements. + self.worker.hold_jobs_in_build = True + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + B1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'B1') + B2 = self.fake_gerrit.addFakeChange('org/project', 'master', 'B2') + A.addApproval('CRVW', 2) + B1.addApproval('CRVW', 2) + B2.addApproval('CRVW', 2) + A.addApproval('VRFY', 1) # required by gate + B1.addApproval('VRFY', -1) # should go to check + B2.addApproval('VRFY', 1) # should go to gate + B1.addApproval('APRV', 1) + B2.addApproval('APRV', 1) + B1.setDependsOn(A, 1) + B2.setDependsOn(A, 1) + self.fake_gerrit.addEvent(A.addApproval('APRV', 1)) + # Jobs are being held in build to make sure that 3,1 has time + # to enqueue behind 1,1 so that the test is more + # deterministic. + self.waitUntilSettled() + self.worker.hold_jobs_in_build = False + self.worker.release() + self.waitUntilSettled() + + self.assertEqual(len(self.history), 3) + for job in self.history: + if job.changes == '1,1': + self.assertEqual(job.name, 'project-gate') + elif job.changes == '2,1': + self.assertEqual(job.name, 'project-check') + elif job.changes == '1,1 3,1': + self.assertEqual(job.name, 'project-gate') + else: + raise Exception("Unknown job") + + def test_zuul_trigger_project_change_merged(self): + "Test Zuul trigger event: project-change-merged" + self.config.set('zuul', 'layout_config', + 'tests/fixtures/layout-zuultrigger-merged.yaml') + self.sched.reconfigure(self.config) + self.registerJobs() + + # This test has the following three changes: + # A, B, C; B conflicts with A, but C does not. + # When A is merged, B and C should be checked for conflicts, + # and B should receive a -1. + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') + C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') + A.addPatchset(['conflict']) + B.addPatchset(['conflict']) + A.addApproval('CRVW', 2) + self.fake_gerrit.addEvent(A.addApproval('APRV', 1)) + self.waitUntilSettled() + + self.assertEqual(len(self.history), 1) + self.assertEqual(self.history[0].name, 'project-gate') + self.assertEqual(A.reported, 2) + self.assertEqual(B.reported, 1) + self.assertEqual(C.reported, 0) + self.assertEqual(B.messages[0], + "Merge Failed.\n\nThis change was unable to be automatically " + "merged with the current state of the repository. Please rebase " + "your change and upload a new patchset.") diff --git a/zuul/cmd/server.py b/zuul/cmd/server.py index d7de85a45..25dab6f42 100755 --- a/zuul/cmd/server.py +++ b/zuul/cmd/server.py @@ -87,6 +87,7 @@ class Server(zuul.cmd.ZuulApp): self.sched.registerReporter(None, 'smtp') self.sched.registerTrigger(None, 'gerrit') self.sched.registerTrigger(None, 'timer') + self.sched.registerTrigger(None, 'zuul') layout = self.sched.testConfig(self.config.get('zuul', 'layout_config')) if not job_list_path: @@ -145,6 +146,7 @@ class Server(zuul.cmd.ZuulApp): import zuul.reporter.smtp import zuul.trigger.gerrit import zuul.trigger.timer + import zuul.trigger.zuultrigger import zuul.webapp import zuul.rpclistener @@ -163,6 +165,7 @@ class Server(zuul.cmd.ZuulApp): merger = zuul.merger.client.MergeClient(self.config, self.sched) gerrit = zuul.trigger.gerrit.Gerrit(self.config, self.sched) timer = zuul.trigger.timer.Timer(self.config, self.sched) + zuultrigger = zuul.trigger.zuultrigger.ZuulTrigger(self.config, self.sched) if self.config.has_option('zuul', 'status_expiry'): cache_expiry = self.config.getint('zuul', 'status_expiry') else: @@ -185,6 +188,7 @@ class Server(zuul.cmd.ZuulApp): self.sched.setMerger(merger) self.sched.registerTrigger(gerrit) self.sched.registerTrigger(timer) + self.sched.registerTrigger(zuultrigger) self.sched.registerReporter(gerrit_reporter) self.sched.registerReporter(smtp_reporter) diff --git a/zuul/layoutvalidator.py b/zuul/layoutvalidator.py index 0adc2c220..9a4e00f5d 100644 --- a/zuul/layoutvalidator.py +++ b/zuul/layoutvalidator.py @@ -65,8 +65,16 @@ class LayoutSchema(object): timer_trigger = {v.Required('time'): str} - trigger = v.Required(v.Any({'gerrit': toList(gerrit_trigger)}, - {'timer': toList(timer_trigger)})) + zuul_trigger = {v.Required('event'): + toList(v.Any('parent-change-enqueued', + 'project-change-merged')), + 'pipeline': toList(str), + 'require-approval': toList(require_approval), + } + + trigger = v.Required({'gerrit': toList(gerrit_trigger), + 'timer': toList(timer_trigger), + 'zuul': toList(zuul_trigger)}) report_actions = {'gerrit': variable_dict, 'smtp': {'to': str, diff --git a/zuul/lib/gerrit.py b/zuul/lib/gerrit.py index 30fb6fed3..52e60578a 100644 --- a/zuul/lib/gerrit.py +++ b/zuul/lib/gerrit.py @@ -144,6 +144,23 @@ class Gerrit(object): (pprint.pformat(data))) return data + def simpleQuery(self, query): + args = '--current-patch-set' + cmd = 'gerrit query --format json %s %s' % ( + args, query) + out, err = self._ssh(cmd) + if not out: + return False + lines = out.split('\n') + if not lines: + return False + data = [json.loads(line) for line in lines[:-1]] + if not data: + return False + self.log.debug("Received data from Gerrit query: \n%s" % + (pprint.pformat(data))) + return data + def _open(self): client = paramiko.SSHClient() client.load_system_host_keys() diff --git a/zuul/model.py b/zuul/model.py index 8b9724170..77ab68bdf 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -947,6 +947,8 @@ class TriggerEvent(object): self.newrev = None # timer self.timespec = None + # zuultrigger + self.pipeline_name = None # For events that arrive with a destination pipeline (eg, from # an admin command, etc): self.forced_pipeline = None @@ -1026,7 +1028,7 @@ class BaseFilter(object): class EventFilter(BaseFilter): def __init__(self, trigger, types=[], branches=[], refs=[], event_approvals={}, comments=[], emails=[], usernames=[], - timespecs=[], required_approvals=[]): + timespecs=[], required_approvals=[], pipelines=[]): super(EventFilter, self).__init__( required_approvals=required_approvals) self.trigger = trigger @@ -1036,12 +1038,14 @@ class EventFilter(BaseFilter): self._comments = comments self._emails = emails self._usernames = usernames + self._pipelines = pipelines self.types = [re.compile(x) for x in types] self.branches = [re.compile(x) for x in branches] self.refs = [re.compile(x) for x in refs] self.comments = [re.compile(x) for x in comments] self.emails = [re.compile(x) for x in emails] self.usernames = [re.compile(x) for x in usernames] + self.pipelines = [re.compile(x) for x in pipelines] self.event_approvals = event_approvals self.timespecs = timespecs @@ -1050,6 +1054,8 @@ class EventFilter(BaseFilter): if self._types: ret += ' types: %s' % ', '.join(self._types) + if self._pipelines: + ret += ' pipelines: %s' % ', '.join(self._pipelines) if self._branches: ret += ' branches: %s' % ', '.join(self._branches) if self._refs: @@ -1081,6 +1087,14 @@ class EventFilter(BaseFilter): if self.types and not matches_type: return False + # pipelines are ORed + matches_pipeline = False + for epipe in self.pipelines: + if epipe.match(event.pipeline_name): + matches_pipeline = True + if self.pipelines and not matches_pipeline: + return False + # branches are ORed matches_branch = False for branch in self.branches: diff --git a/zuul/scheduler.py b/zuul/scheduler.py index 60a22ce0a..a2e07cd0f 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -1,4 +1,4 @@ -# Copyright 2012 Hewlett-Packard Development Company, L.P. +# Copyright 2012-2014 Hewlett-Packard Development Company, L.P. # Copyright 2013 OpenStack Foundation # Copyright 2013 Antoine "hashar" Musso # Copyright 2013 Wikimedia Foundation Inc. @@ -326,12 +326,20 @@ class Scheduler(threading.Thread): required_approvals= toList(trigger.get('require-approval'))) manager.event_filters.append(f) - elif 'timer' in conf_pipeline['trigger']: + if 'timer' in conf_pipeline['trigger']: for trigger in toList(conf_pipeline['trigger']['timer']): f = EventFilter(trigger=self.triggers['timer'], types=['timer'], timespecs=toList(trigger['time'])) manager.event_filters.append(f) + if 'zuul' in conf_pipeline['trigger']: + for trigger in toList(conf_pipeline['trigger']['zuul']): + f = EventFilter(trigger=self.triggers['zuul'], + types=toList(trigger['event']), + pipelines=toList(trigger.get('pipeline')), + required_approvals= + toList(trigger.get('require-approval'))) + manager.event_filters.append(f) for project_template in data.get('project-templates', []): # Make sure the template only contains valid pipelines @@ -1153,6 +1161,7 @@ class BasePipelineManager(object): item.enqueue_time = enqueue_time self.reportStats(item) self.enqueueChangesBehind(change, quiet, ignore_requirements) + self.sched.triggers['zuul'].onChangeEnqueued(item.change, self.pipeline) else: self.log.error("Unable to find change queue for project %s" % change.project) @@ -1427,6 +1436,7 @@ class BasePipelineManager(object): change_queue.increaseWindowSize() self.log.debug("%s window size increased to %s" % (change_queue, change_queue.window)) + self.sched.triggers['zuul'].onChangeMerged(item.change) def _reportItem(self, item): self.log.debug("Reporting change %s" % item.change) diff --git a/zuul/trigger/gerrit.py b/zuul/trigger/gerrit.py index d2cd7fc9c..6a2c362eb 100644 --- a/zuul/trigger/gerrit.py +++ b/zuul/trigger/gerrit.py @@ -323,6 +323,14 @@ class Gerrit(object): raise return change + def getProjectOpenChanges(self, project): + data = self.gerrit.simpleQuery("project:%s status:open" % project.name) + changes = [] + for record in data: + changes.append(self._getChange(record['number'], + record['currentPatchSet']['number'])) + return changes + def updateChange(self, change): self.log.info("Updating information for %s,%s" % (change.number, change.patchset)) diff --git a/zuul/trigger/zuultrigger.py b/zuul/trigger/zuultrigger.py new file mode 100644 index 000000000..436311bbd --- /dev/null +++ b/zuul/trigger/zuultrigger.py @@ -0,0 +1,117 @@ +# Copyright 2012-2014 Hewlett-Packard Development Company, L.P. +# Copyright 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +from zuul.model import TriggerEvent + + +class ZuulTrigger(object): + name = 'zuul' + log = logging.getLogger("zuul.ZuulTrigger") + + def __init__(self, config, sched): + self.sched = sched + self.config = config + self._handle_parent_change_enqueued_events = False + self._handle_project_change_merged_events = False + + def stop(self): + pass + + def isMerged(self, change, head=None): + raise Exception("Zuul trigger does not support checking if " + "a change is merged.") + + def canMerge(self, change, allow_needs): + raise Exception("Zuul trigger does not support checking if " + "a change can merge.") + + def maintainCache(self, relevant): + return + + def onChangeMerged(self, change): + # Called each time zuul merges a change + if self._handle_project_change_merged_events: + try: + self._createProjectChangeMergedEvents(change) + except Exception: + self.log.exception("Unable to create project-change-merged events for %s" % (change,)) + + def onChangeEnqueued(self, change, pipeline): + # Called each time a change is enqueued in a pipeline + if self._handle_parent_change_enqueued_events: + try: + self._createParentChangeEnqueuedEvents(change, pipeline) + except Exception: + self.log.exception("Unable to create parent-change-enqueued events for %s in %s" % (change, pipeline)) + + def _createProjectChangeMergedEvents(self, change): + changes = self.sched.triggers['gerrit'].getProjectOpenChanges(change.project) + for change in changes: + self._createProjectChangeMergedEvent(change) + + def _createProjectChangeMergedEvent(self, change): + event = TriggerEvent() + event.type = 'project-change-merged' + event.trigger_name = self.name + event.project_name = change.project.name + event.change_number = change.number + event.branch = change.branch + event.change_url = change.url + event.patch_number = change.patchset + event.refspec = change.refspec + self.sched.addEvent(event) + + def _createParentChangeEnqueuedEvents(self, change, pipeline): + self.log.debug("Checking for changes needing %s:" % change) + if not hasattr(change, 'needed_by_changes'): + self.log.debug(" Changeish does not support dependencies") + return + for needs in change.needed_by_changes: + self._createParentChangeEnqueuedEvent(needs, pipeline) + + def _createParentChangeEnqueuedEvent(self, change, pipeline): + event = TriggerEvent() + event.type = 'parent-change-enqueued' + event.trigger_name = self.name + event.pipeline_name = pipeline.name + event.project_name = change.project.name + event.change_number = change.number + event.branch = change.branch + event.change_url = change.url + event.patch_number = change.patchset + event.refspec = change.refspec + self.sched.addEvent(event) + + def postConfig(self): + self._handle_parent_change_enqueued_events = False + self._handle_project_change_merged_events = False + for pipeline in self.sched.layout.pipelines.values(): + for ef in pipeline.manager.event_filters: + if ef.trigger != self: + continue + if 'parent-change-enqueued' in ef._types: + self._handle_parent_change_enqueued_events = True + elif 'project-change-merged' in ef._types: + self._handle_project_change_merged_events = True + + def getChange(self, number, patchset, refresh=False): + raise Exception("Zuul trigger does not support changes.") + + def getGitUrl(self, project): + raise Exception("Zuul trigger does not support changes.") + + def getGitwebUrl(self, project, sha=None): + raise Exception("Zuul trigger does not support changes.") |