Merge "Report dequeued changes via Github checks API"

12 files changed, 348 insertions, 10 deletions

diff --git a/doc/source/reference/drivers/github.rst b/doc/source/reference/drivers/github.rst
index c2c085eca..df649d3eb 100644
--- a/doc/source/reference/drivers/github.rst
+++ b/doc/source/reference/drivers/github.rst
@@ -366,8 +366,8 @@ itself. Status name, description, and context is taken from the pipeline.
    .. attr:: check
 
       If the reporter should utilize github's checks API to set the commit
-      status, this must be set to ``in_progress``, ``success`` or ``failure``
-      (depending on which status the reporter should report).
+      status, this must be set to ``in_progress``, ``success``, ``failure``
+      or ``cancelled`` (depending on which status the reporter should report).
 
    .. attr:: comment
       :default: true
diff --git a/doc/source/reference/pipeline_def.rst b/doc/source/reference/pipeline_def.rst
index e2c2d9587..da13570f7 100644
--- a/doc/source/reference/pipeline_def.rst
+++ b/doc/source/reference/pipeline_def.rst
@@ -219,6 +219,13 @@ success, the pipeline reports back to Gerrit with ``Verified`` vote of
       The introductory text in reports when an item is dequeued
       without running any jobs.  Empty by default.
 
+   .. attr:: dequeue-message
+      :default: Build canceled.
+
+      The introductory text in reports when an item is dequeued.
+      The dequeue message only applies if the item was dequeued without
+      a result.
+
    .. attr:: footer-message
 
       Supplies additional information after test results.  Useful for
@@ -354,6 +361,12 @@ success, the pipeline reports back to Gerrit with ``Verified`` vote of
       These reporters describe what Zuul should do when a pipeline is
       disabled.  See ``disable-after-consecutive-failures``.
 
+   .. attr:: dequeue
+
+      These reporters describe what Zuul should do if an item is
+      dequeued. The dequeue reporters will only apply, if the item
+      was dequeued without a result.
+
    The following options can be used to alter Zuul's behavior to
    mitigate situations in which jobs are failing frequently (perhaps
    due to a problem with an external dependency, or unusually high
diff --git a/releasenotes/notes/dequeue-reporting-620f364309587304.yaml b/releasenotes/notes/dequeue-reporting-620f364309587304.yaml
new file mode 100644
index 000000000..9cd28ad52
--- /dev/null
+++ b/releasenotes/notes/dequeue-reporting-620f364309587304.yaml
@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Pipelines now provide a :attr:`pipeline.dequeue` reporter action so that
+    reporters may run whenever an item is dequeued. The dequeue reporters will
+    only apply if the item wasn't a success or failure.
diff --git a/tests/fixtures/layouts/dequeue-reporting.yaml b/tests/fixtures/layouts/dequeue-reporting.yaml
new file mode 100644
index 000000000..52afe9a5a
--- /dev/null
+++ b/tests/fixtures/layouts/dequeue-reporting.yaml
@@ -0,0 +1,85 @@
+- pipeline:
+    name: check
+    manager: independent
+    failure-message: Build failed (check)
+    success-message: Build succeeded (check)
+    dequeue-message: Build canceled (check)
+    start-message: Build started (check)
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+    start:
+      gerrit:
+        Verified: 0
+    dequeue:
+      gerrit:
+        Verified: 0
+
+- pipeline:
+    name: gate
+    manager: dependent
+    supercedes: check
+    failure-message: Build failed (gate)
+    success-message: Build succeeded (gate)
+    dequeue-message: Build canceled (gate)
+    start-message: Build started (gate)
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - Approved: 1
+    success:
+      gerrit:
+        Verified: 2
+        submit: true
+    failure:
+      gerrit:
+        Verified: -2
+    start:
+      gerrit:
+        Verified: 0
+    dequeue:
+      gerrit:
+        Verified: 0
+    precedence: high
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
+
+- job:
+    name: project-test1
+    run: playbooks/project-test1.yaml
+
+- job:
+    name: project-test2
+    run: playbooks/project-test2.yaml
+
+- job:
+    name: project-merge
+    hold-following-changes: true
+    run: playbooks/project-merge.yaml
+
+- project:
+    name: org/project
+    check:
+      jobs:
+        - project-merge
+        - project-test1:
+            dependencies: project-merge
+        - project-test2:
+            dependencies: project-merge
+    gate:
+      jobs:
+        - project-merge
+        - project-test1:
+            dependencies: project-merge
+        - project-test2:
+            dependencies: project-merge
diff --git a/tests/fixtures/layouts/reporting-github.yaml b/tests/fixtures/layouts/reporting-github.yaml
index c35207af1..a9cbf8fcb 100644
--- a/tests/fixtures/layouts/reporting-github.yaml
+++ b/tests/fixtures/layouts/reporting-github.yaml
@@ -96,6 +96,9 @@
     failure:
       github:
         check: failure
+    dequeue:
+      github:
+        check: cancelled
 
 - pipeline:
     name: gate
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
index 0558c9994..39c33440a 100644
--- a/tests/unit/test_github_driver.py
+++ b/tests/unit/test_github_driver.py
@@ -1685,6 +1685,61 @@ class TestGithubAppDriver(ZuulGithubAppTestCase):
         self.assertTrue(A.is_merged)
 
     @simple_layout("layouts/reporting-github.yaml", driver="github")
+    def test_reporting_checks_api_dequeue(self):
+        "Test that a dequeued change will be reported back to the check run"
+        project = "org/project3"
+        github = self.fake_github.getGithubClient(None)
+
+        client = zuul.rpcclient.RPCClient(
+            "127.0.0.1", self.gearman_server.port
+        )
+        self.addCleanup(client.shutdown)
+
+        self.executor_server.hold_jobs_in_build = True
+        A = self.fake_github.openFakePullRequest(project, "master", "A")
+        self.fake_github.emitEvent(A.getPullRequestOpenedEvent())
+        self.waitUntilSettled()
+
+        # We should have a pending check for the head sha
+        self.assertIn(
+            A.head_sha, github.repo_from_project(project)._commits.keys())
+        check_runs = self.fake_github.getCommitChecks(project, A.head_sha)
+
+        self.assertEqual(1, len(check_runs))
+        check_run = check_runs[0]
+
+        self.assertEqual("tenant-one/checks-api-reporting", check_run["name"])
+        self.assertEqual("in_progress", check_run["status"])
+        self.assertThat(
+            check_run["output"]["summary"],
+            MatchesRegex(r'.*Starting checks-api-reporting jobs.*', re.DOTALL)
+        )
+
+        # Use the client to dequeue the pending change
+        client.dequeue(
+            tenant="tenant-one",
+            pipeline="checks-api-reporting",
+            project="org/project3",
+            change="{},{}".format(A.number, A.head_sha),
+            ref=None,
+        )
+        self.waitUntilSettled()
+
+        # We should now have a cancelled check run for the head sha
+        check_runs = self.fake_github.getCommitChecks(project, A.head_sha)
+        self.assertEqual(1, len(check_runs))
+        check_run = check_runs[0]
+
+        self.assertEqual("tenant-one/checks-api-reporting", check_run["name"])
+        self.assertEqual("completed", check_run["status"])
+        self.assertEqual("cancelled", check_run["conclusion"])
+        self.assertThat(
+            check_run["output"]["summary"],
+            MatchesRegex(r'.*Build canceled.*', re.DOTALL)
+        )
+        self.assertIsNotNone(check_run["completed_at"])
+
+    @simple_layout("layouts/reporting-github.yaml", driver="github")
     def test_update_non_existing_check_run(self):
         project = "org/project3"
         github = self.fake_github.getGithubClient(None)
diff --git a/tests/unit/test_reporting.py b/tests/unit/test_reporting.py
new file mode 100644
index 000000000..dab041244
--- /dev/null
+++ b/tests/unit/test_reporting.py
@@ -0,0 +1,137 @@
+# Copyright 2020 BMW Group
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import zuul.rpcclient
+
+from tests.base import ZuulTestCase, simple_layout
+
+
+class TestReporting(ZuulTestCase):
+    tenant_config_file = "config/single-tenant/main.yaml"
+
+    @simple_layout("layouts/dequeue-reporting.yaml")
+    def test_dequeue_reporting(self):
+        """Check that explicitly dequeued items are reported as dequeued"""
+
+        # We use the rpcclient to explicitly dequeue the item
+        client = zuul.rpcclient.RPCClient(
+            "127.0.0.1", self.gearman_server.port
+        )
+        self.addCleanup(client.shutdown)
+
+        self.executor_server.hold_jobs_in_build = True
+        A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
+        A.addApproval("Code-Review", 2)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        client.dequeue(
+            tenant="tenant-one",
+            pipeline="check",
+            project="org/project",
+            change="1,1",
+            ref=None,
+        )
+        self.waitUntilSettled()
+
+        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
+        check_pipeline = tenant.layout.pipelines['check']
+
+        # A should have been reported two times: start, cancel
+        self.assertEqual(2, A.reported)
+        self.assertEqual(2, len(A.messages))
+        self.assertIn("Build started (check)", A.messages[0])
+        self.assertIn("Build canceled (check)", A.messages[1])
+        # There shouldn't be any successful items
+        self.assertEqual(len(check_pipeline.getAllItems()), 0)
+        # But one canceled
+        self.assertEqual(self.countJobResults(self.history, "ABORTED"), 1)
+
+    @simple_layout("layouts/dequeue-reporting.yaml")
+    def test_dequeue_reporting_gate_reset(self):
+        """Check that a gate reset is not reported as dequeued"""
+
+        A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
+        B = self.fake_gerrit.addFakeChange("org/project", "master", "B")
+        A.addApproval("Code-Review", 2)
+        B.addApproval("Code-Review", 2)
+
+        self.executor_server.failJob("project-test1", A)
+
+        self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
+        self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
+        self.waitUntilSettled()
+
+        # None of the items should be reported as dequeued, only success or
+        # failure
+        self.assertEqual(A.data["status"], "NEW")
+        self.assertEqual(B.data["status"], "MERGED")
+        self.assertEqual(A.reported, 2)
+        self.assertEqual(B.reported, 2)
+
+        self.assertIn("Build started (gate)", A.messages[0])
+        self.assertIn("Build failed (gate)", A.messages[1])
+        self.assertIn("Build started (gate)", B.messages[0])
+        self.assertIn("Build succeeded (gate)", B.messages[1])
+
+    @simple_layout("layouts/dequeue-reporting.yaml")
+    def test_dequeue_reporting_supercedes(self):
+        """Test that a superceeded change is reported as dequeued"""
+
+        self.executor_server.hold_jobs_in_build = True
+
+        A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        A.addApproval("Code-Review", 2)
+        self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
+
+        self.executor_server.hold_jobs_in_build = False
+        self.executor_server.release()
+        self.waitUntilSettled()
+
+        self.assertEqual(4, A.reported)
+
+        self.assertIn("Build started (check)", A.messages[0])
+        self.assertIn("Build canceled (check)", A.messages[1])
+        self.assertIn("Build started (gate)", A.messages[2])
+        self.assertIn("Build succeeded (gate)", A.messages[3])
+
+    @simple_layout("layouts/dequeue-reporting.yaml")
+    def test_dequeue_reporting_new_patchset(self):
+        "Test that change superceeded by a new patchset is reported as deqeued"
+
+        self.executor_server.hold_jobs_in_build = True
+
+        A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertEqual(1, len(self.builds))
+
+        A.addPatchset()
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
+        self.waitUntilSettled()
+
+        self.executor_server.hold_jobs_in_build = False
+        self.executor_server.release()
+        self.waitUntilSettled()
+
+        self.assertEqual(4, A.reported)
+
+        self.assertIn("Build started (check)", A.messages[0])
+        self.assertIn("Build canceled (check)", A.messages[1])
+        self.assertIn("Build started (check)", A.messages[2])
+        self.assertIn("Build succeeded (check)", A.messages[3])
diff --git a/zuul/configloader.py b/zuul/configloader.py
index f985804c6..b999bd0c4 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -1151,6 +1151,7 @@ class PipelineParser(object):
         'merge-failure': 'merge_failure_actions',
         'no-jobs': 'no_jobs_actions',
         'disabled': 'disabled_actions',
+        'dequeue': 'dequeue_actions',
     }
 
     def __init__(self, pcontext):
@@ -1200,6 +1201,7 @@ class PipelineParser(object):
                     'merge-failure-message': str,
                     'no-jobs-message': str,
                     'footer-message': str,
+                    'dequeue-message': str,
                     'dequeue-on-new-patchset': bool,
                     'ignore-dependencies': bool,
                     'post-review': bool,
@@ -1218,7 +1220,7 @@ class PipelineParser(object):
         pipeline['reject'] = self.getDriverSchema('reject')
         pipeline['trigger'] = vs.Required(self.getDriverSchema('trigger'))
         for action in ['enqueue', 'start', 'success', 'failure',
-                       'merge-failure', 'no-jobs', 'disabled']:
+                       'merge-failure', 'no-jobs', 'disabled', 'dequeue']:
             pipeline[action] = self.getDriverSchema('reporter')
         return vs.Schema(pipeline)
 
@@ -1247,6 +1249,9 @@ class PipelineParser(object):
                                           "Starting {pipeline.name} jobs.")
         pipeline.enqueue_message = conf.get('enqueue-message', "")
         pipeline.no_jobs_message = conf.get('no-jobs-message', "")
+        pipeline.dequeue_message = conf.get(
+            "dequeue-message", "Build canceled."
+        )
         pipeline.dequeue_on_new_patchset = conf.get(
             'dequeue-on-new-patchset', True)
         pipeline.ignore_dependencies = conf.get(
diff --git a/zuul/driver/github/githubreporter.py b/zuul/driver/github/githubreporter.py
index 2c9ccec34..a5a509623 100644
--- a/zuul/driver/github/githubreporter.py
+++ b/zuul/driver/github/githubreporter.py
@@ -213,11 +213,16 @@ class GithubReporter(BaseReporter):
         pr_number = item.change.number
         sha = item.change.patchset
 
-        # Check if the buildset is finished or not. In case it's finished, we
-        # must provide additional parameters when updating the check_run via
-        # the Github API later on.
-        completed = item.current_build_set.result is not None
         status = self._check
+        # We declare a item as completed if it either has a result
+        # (success|failure) or a dequeue reporter is called (cancelled in case
+        # of Github checks API). For the latter one, the item might or might
+        # not have a result, but we still must set a conclusion on the check
+        # run. Thus, we cannot rely on the buildset's result only, but also
+        # check the state the reporter is going to report.
+        completed = (
+            item.current_build_set.result is not None or status == "cancelled"
+        )
 
         log.debug(
             "Updating check for change %s, params %s, context %s, message: %s",
@@ -313,6 +318,6 @@ def getSchema():
         'unlabel': scalar_or_list(str),
         'review': v.Any('approve', 'request-changes', 'comment'),
         'review-body': str,
-        'check': v.Any("in_progress", "success", "failure"),
+        'check': v.Any("in_progress", "success", "failure", "cancelled"),
     })
     return github_reporter
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index ee0f2c241..fdce0c7b7 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -169,6 +169,19 @@ class PipelineManager(metaclass=ABCMeta):
                 self.log.error("Reporting item start %s received: %s" %
                                (item, ret))
 
+    def reportDequeue(self, item):
+        if not self.pipeline._disabled:
+            self.log.info(
+                "Reporting dequeue, action %s item%s",
+                self.pipeline.dequeue_actions,
+                item,
+            )
+            ret = self.sendReport(self.pipeline.dequeue_actions, item)
+            if ret:
+                self.log.error(
+                    "Reporting item dequeue %s received: %s", item, ret
+                )
+
     def sendReport(self, action_reporters, item, message=None):
         """Sends the built message off to configured reporters.
 
@@ -371,6 +384,12 @@ class PipelineManager(metaclass=ABCMeta):
         log = get_annotated_logger(self.log, item.event)
         log.debug("Removing change %s from queue", item.change)
         item.queue.dequeueItem(item)
+        # In case a item is dequeued that doesn't have a result yet
+        # (success/failed/...) we report it as dequeued.
+        # Without this check, all items with a valid result would be reported
+        # twice.
+        if not item.current_build_set.result and item.live:
+            self.reportDequeue(item)
 
     def removeItem(self, item):
         log = get_annotated_logger(self.log, item.event)
diff --git a/zuul/model.py b/zuul/model.py
index a5dd8827d..a1e3d3dc4 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -261,6 +261,7 @@ class Pipeline(object):
         self.footer_message = None
         self.enqueue_message = None
         self.start_message = None
+        self.dequeue_message = None
         self.post_review = False
         self.dequeue_on_new_patchset = True
         self.ignore_dependencies = False
@@ -276,6 +277,7 @@ class Pipeline(object):
         self.merge_failure_actions = []
         self.no_jobs_actions = []
         self.disabled_actions = []
+        self.dequeue_actions = []
         self.disable_at = None
         self._consecutive_failures = 0
         self._disabled = False
@@ -295,7 +297,8 @@ class Pipeline(object):
             self.failure_actions +
             self.merge_failure_actions +
             self.no_jobs_actions +
-            self.disabled_actions
+            self.disabled_actions +
+            self.dequeue_actions
         )
 
     def __repr__(self):
diff --git a/zuul/reporter/__init__.py b/zuul/reporter/__init__.py
index ac5f371d6..da3c606b4 100644
--- a/zuul/reporter/__init__.py
+++ b/zuul/reporter/__init__.py
@@ -123,7 +123,8 @@ class BaseReporter(object, metaclass=abc.ABCMeta):
             'failure': self._formatItemReportFailure,
             'merge-failure': self._formatItemReportMergeFailure,
             'no-jobs': self._formatItemReportNoJobs,
-            'disabled': self._formatItemReportDisabled
+            'disabled': self._formatItemReportDisabled,
+            'dequeue': self._formatItemReportDequeue,
         }
         return format_methods[self._action]
 
@@ -208,6 +209,12 @@ class BaseReporter(object, metaclass=abc.ABCMeta):
         else:
             return self._formatItemReport(item)
 
+    def _formatItemReportDequeue(self, item, with_jobs=True):
+        msg = item.pipeline.dequeue_message
+        if with_jobs:
+            msg += '\n\n' + self._formatItemReportJobs(item)
+        return msg
+
     def _getItemReportJobsFields(self, item):
         # Extract the report elements from an item
         config = self.connection.sched.config