Trigger pipeline runs when semaphores release

The recent optimization to avoid processing pipelines if no events are
waiting did not account for semaphores which may be held by jobs in
different pipelines.  In that case, a job completing in one pipeline
needs to generate an event in another pipeline in order to prompt it
to begin processing.

We have no easy way of knowing which pipelines may have jobs which are
waiting for a semaphore, so this change broadcasts an event to every
pipeline in the tenant when a semaphore is released.  Hopefully this
shouldn't generate that much more traffic (how much depends on how
frequently semaphores are released).  If desired, we can further
optimize this by storing semaphore pipeline waiters in ZK in a later
change.

Change-Id: Ide381279b0442d11535c00746e4baf19f32f3cd7

3 files changed, 107 insertions, 1 deletions

diff --git a/tests/fixtures/layouts/semaphore-multi-pipeline.yaml b/tests/fixtures/layouts/semaphore-multi-pipeline.yaml
new file mode 100644
index 000000000..2f582684c
--- /dev/null
+++ b/tests/fixtures/layouts/semaphore-multi-pipeline.yaml
@@ -0,0 +1,65 @@
+- pipeline:
+    name: check
+    manager: independent
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- pipeline:
+    name: gate
+    manager: dependent
+    success-message: Build succeeded (gate).
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - Approved: 1
+    success:
+      gerrit:
+        Verified: 2
+        submit: true
+    failure:
+      gerrit:
+        Verified: -2
+    start:
+      gerrit:
+        Verified: 0
+    precedence: high
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
+    nodeset:
+      nodes:
+        - label: ubuntu-xenial
+          name: controller
+
+- semaphore:
+    name: test-semaphore
+    max: 1
+
+- job:
+    name: check-job
+    run: playbooks/check.yaml
+    semaphores: test-semaphore
+
+- job:
+    name: gate-job
+    run: playbooks/gate.yaml
+    semaphores: test-semaphore
+
+- project:
+    name: org/project
+    check:
+      jobs:
+        - check-job
+    gate:
+      jobs:
+        - gate-job
diff --git a/tests/unit/test_model_upgrade.py b/tests/unit/test_model_upgrade.py
index 9bbaf12f7..94c2d307e 100644
--- a/tests/unit/test_model_upgrade.py
+++ b/tests/unit/test_model_upgrade.py
@@ -173,7 +173,7 @@ class TestSemaphoreModelUpgrade(ZuulTestCase):
             1)
 
         # Try to release the old-style semaphore after the model API upgrade.
-        tenant.semaphore_handler.release(item, job)
+        tenant.semaphore_handler.release(self.scheds.first.sched, item, job)
         self.assertEqual(
             len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
             0)
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 6c0c79bde..037a519b2 100644
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -7727,6 +7727,47 @@ class TestSemaphore(ZuulTestCase):
         # least one should be able to start on each pass through the
         # loop).
 
+    @simple_layout('layouts/semaphore-multi-pipeline.yaml')
+    def test_semaphore_multi_pipeline(self):
+        "Test semaphores in multiple pipelines"
+        tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
+
+        self.executor_server.hold_jobs_in_build = True
+
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        self.assertEqual(
+            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
+            0)
+
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 1)
+        self.assertEqual(
+            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
+            1)
+
+        # Start a second change in a different pipeline
+        B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
+        B.addApproval('Code-Review', 2)
+        self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
+        self.waitUntilSettled()
+
+        # Still just the first change holds the lock
+        self.assertEqual(len(self.builds), 1)
+        self.assertEqual(
+            len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")),
+            1)
+
+        # Now the second should run
+        self.executor_server.hold_jobs_in_build = False
+        self.executor_server.release()
+        self.waitUntilSettled()
+        self.assertHistory([
+            dict(name='check-job', result='SUCCESS', changes='1,1'),
+            dict(name='gate-job', result='SUCCESS', changes='2,1'),
+        ])
+
 
 class TestSemaphoreMultiTenant(ZuulTestCase):
     tenant_config_file = 'config/multi-tenant-semaphore/main.yaml'