Report dequeued changes via Github checks API

This patch provides the general functionality to allow reporting of
dequeued items and makes use of that in the Github checks API.
This reporting will only apply if the item wasn't a success or failure.

Change-Id: I1297da4d1708908c6b179110479fe0450e5550fe

4 files changed, 280 insertions, 0 deletions

diff --git a/tests/fixtures/layouts/dequeue-reporting.yaml b/tests/fixtures/layouts/dequeue-reporting.yaml
new file mode 100644
index 000000000..52afe9a5a
--- /dev/null
+++ b/tests/fixtures/layouts/dequeue-reporting.yaml
@@ -0,0 +1,85 @@
+- pipeline:
+    name: check
+    manager: independent
+    failure-message: Build failed (check)
+    success-message: Build succeeded (check)
+    dequeue-message: Build canceled (check)
+    start-message: Build started (check)
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+    start:
+      gerrit:
+        Verified: 0
+    dequeue:
+      gerrit:
+        Verified: 0
+
+- pipeline:
+    name: gate
+    manager: dependent
+    supercedes: check
+    failure-message: Build failed (gate)
+    success-message: Build succeeded (gate)
+    dequeue-message: Build canceled (gate)
+    start-message: Build started (gate)
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - Approved: 1
+    success:
+      gerrit:
+        Verified: 2
+        submit: true
+    failure:
+      gerrit:
+        Verified: -2
+    start:
+      gerrit:
+        Verified: 0
+    dequeue:
+      gerrit:
+        Verified: 0
+    precedence: high
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
+
+- job:
+    name: project-test1
+    run: playbooks/project-test1.yaml
+
+- job:
+    name: project-test2
+    run: playbooks/project-test2.yaml
+
+- job:
+    name: project-merge
+    hold-following-changes: true
+    run: playbooks/project-merge.yaml
+
+- project:
+    name: org/project
+    check:
+      jobs:
+        - project-merge
+        - project-test1:
+            dependencies: project-merge
+        - project-test2:
+            dependencies: project-merge
+    gate:
+      jobs:
+        - project-merge
+        - project-test1:
+            dependencies: project-merge
+        - project-test2:
+            dependencies: project-merge
diff --git a/tests/fixtures/layouts/reporting-github.yaml b/tests/fixtures/layouts/reporting-github.yaml
index c35207af1..a9cbf8fcb 100644
--- a/tests/fixtures/layouts/reporting-github.yaml
+++ b/tests/fixtures/layouts/reporting-github.yaml
@@ -96,6 +96,9 @@
     failure:
       github:
         check: failure
+    dequeue:
+      github:
+        check: cancelled
 
 - pipeline:
     name: gate
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
index 0558c9994..39c33440a 100644
--- a/tests/unit/test_github_driver.py
+++ b/tests/unit/test_github_driver.py
@@ -1685,6 +1685,61 @@ class TestGithubAppDriver(ZuulGithubAppTestCase):
         self.assertTrue(A.is_merged)
 
     @simple_layout("layouts/reporting-github.yaml", driver="github")
+    def test_reporting_checks_api_dequeue(self):
+        "Test that a dequeued change will be reported back to the check run"
+        project = "org/project3"
+        github = self.fake_github.getGithubClient(None)
+
+        client = zuul.rpcclient.RPCClient(
+            "127.0.0.1", self.gearman_server.port
+        )
+        self.addCleanup(client.shutdown)
+
+        self.executor_server.hold_jobs_in_build = True
+        A = self.fake_github.openFakePullRequest(project, "master", "A")
+        self.fake_github.emitEvent(A.getPullRequestOpenedEvent())
+        self.waitUntilSettled()
+
+        # We should have a pending check for the head sha
+        self.assertIn(
+            A.head_sha, github.repo_from_project(project)._commits.keys())
+        check_runs = self.fake_github.getCommitChecks(project, A.head_sha)
+
+        self.assertEqual(1, len(check_runs))
+        check_run = check_runs[0]
+
+        self.assertEqual("tenant-one/checks-api-reporting", check_run["name"])
+        self.assertEqual("in_progress", check_run["status"])
+        self.assertThat(
+            check_run["output"]["summary"],
+            MatchesRegex(r'.*Starting checks-api-reporting jobs.*', re.DOTALL)
+        )
+
+        # Use the client to dequeue the pending change
+        client.dequeue(
+            tenant="tenant-one",
+            pipeline="checks-api-reporting",
+            project="org/project3",
+            change="{},{}".format(A.number, A.head_sha),
+            ref=None,
+        )
+        self.waitUntilSettled()
+
+        # We should now have a cancelled check run for the head sha
+        check_runs = self.fake_github.getCommitChecks(project, A.head_sha)
+        self.assertEqual(1, len(check_runs))
+        check_run = check_runs[0]
+
+        self.assertEqual("tenant-one/checks-api-reporting", check_run["name"])
+        self.assertEqual("completed", check_run["status"])
+        self.assertEqual("cancelled", check_run["conclusion"])
+        self.assertThat(
+            check_run["output"]["summary"],
+            MatchesRegex(r'.*Build canceled.*', re.DOTALL)
+        )
+        self.assertIsNotNone(check_run["completed_at"])
+
+    @simple_layout("layouts/reporting-github.yaml", driver="github")
     def test_update_non_existing_check_run(self):
         project = "org/project3"
         github = self.fake_github.getGithubClient(None)
diff --git a/tests/unit/test_reporting.py b/tests/unit/test_reporting.py
new file mode 100644
index 000000000..dab041244
--- /dev/null
+++ b/tests/unit/test_reporting.py
@@ -0,0 +1,137 @@
+# Copyright 2020 BMW Group
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import zuul.rpcclient
+
+from tests.base import ZuulTestCase, simple_layout
+
+
+class TestReporting(ZuulTestCase):
+    tenant_config_file = "config/single-tenant/main.yaml"
+
+    @simple_layout("layouts/dequeue-reporting.yaml")
+    def test_dequeue_reporting(self):
+        """Check that explicitly dequeued items are reported as dequeued"""
+
+        # We use the rpcclient to explicitly dequeue the item
+        client = zuul.rpcclient.RPCClient(
+            "127.0.0.1", self.gearman_server.port
+        )
+        self.addCleanup(client.shutdown)
+
+        self.executor_server.hold_jobs_in_build = True
+        A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
+        A.addApproval("Code-Review", 2)
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        client.dequeue(
+            tenant="tenant-one",
+            pipeline="check",
+            project="org/project",
+            change="1,1",
+            ref=None,
+        )
+        self.waitUntilSettled()
+
+        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
+        check_pipeline = tenant.layout.pipelines['check']
+
+        # A should have been reported two times: start, cancel
+        self.assertEqual(2, A.reported)
+        self.assertEqual(2, len(A.messages))
+        self.assertIn("Build started (check)", A.messages[0])
+        self.assertIn("Build canceled (check)", A.messages[1])
+        # There shouldn't be any successful items
+        self.assertEqual(len(check_pipeline.getAllItems()), 0)
+        # But one canceled
+        self.assertEqual(self.countJobResults(self.history, "ABORTED"), 1)
+
+    @simple_layout("layouts/dequeue-reporting.yaml")
+    def test_dequeue_reporting_gate_reset(self):
+        """Check that a gate reset is not reported as dequeued"""
+
+        A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
+        B = self.fake_gerrit.addFakeChange("org/project", "master", "B")
+        A.addApproval("Code-Review", 2)
+        B.addApproval("Code-Review", 2)
+
+        self.executor_server.failJob("project-test1", A)
+
+        self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
+        self.fake_gerrit.addEvent(B.addApproval("Approved", 1))
+        self.waitUntilSettled()
+
+        # None of the items should be reported as dequeued, only success or
+        # failure
+        self.assertEqual(A.data["status"], "NEW")
+        self.assertEqual(B.data["status"], "MERGED")
+        self.assertEqual(A.reported, 2)
+        self.assertEqual(B.reported, 2)
+
+        self.assertIn("Build started (gate)", A.messages[0])
+        self.assertIn("Build failed (gate)", A.messages[1])
+        self.assertIn("Build started (gate)", B.messages[0])
+        self.assertIn("Build succeeded (gate)", B.messages[1])
+
+    @simple_layout("layouts/dequeue-reporting.yaml")
+    def test_dequeue_reporting_supercedes(self):
+        """Test that a superceeded change is reported as dequeued"""
+
+        self.executor_server.hold_jobs_in_build = True
+
+        A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        A.addApproval("Code-Review", 2)
+        self.fake_gerrit.addEvent(A.addApproval("Approved", 1))
+
+        self.executor_server.hold_jobs_in_build = False
+        self.executor_server.release()
+        self.waitUntilSettled()
+
+        self.assertEqual(4, A.reported)
+
+        self.assertIn("Build started (check)", A.messages[0])
+        self.assertIn("Build canceled (check)", A.messages[1])
+        self.assertIn("Build started (gate)", A.messages[2])
+        self.assertIn("Build succeeded (gate)", A.messages[3])
+
+    @simple_layout("layouts/dequeue-reporting.yaml")
+    def test_dequeue_reporting_new_patchset(self):
+        "Test that change superceeded by a new patchset is reported as deqeued"
+
+        self.executor_server.hold_jobs_in_build = True
+
+        A = self.fake_gerrit.addFakeChange("org/project", "master", "A")
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertEqual(1, len(self.builds))
+
+        A.addPatchset()
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
+        self.waitUntilSettled()
+
+        self.executor_server.hold_jobs_in_build = False
+        self.executor_server.release()
+        self.waitUntilSettled()
+
+        self.assertEqual(4, A.reported)
+
+        self.assertIn("Build started (check)", A.messages[0])
+        self.assertIn("Build canceled (check)", A.messages[1])
+        self.assertIn("Build started (check)", A.messages[2])
+        self.assertIn("Build succeeded (check)", A.messages[3])