Refresh cached branches in timer driver

The cache maintenance has an inherent data race as it is only
considering changes as relevent that are currently in any pipeline. To
prevent garbage collection of changes for in-flight events, we only
clean up items older than 2h.

Usually the driver will refresh a change when receiving a connection
event. However, this wasn't the case for trigger events created by the
timer driver.

This can lead to a race condition where a cached branch is cleaned up
while a timer triggered item is enqueued.

For consistency all non-change objects (Branch, Tag, Ref) will now be
refreshed in case the refresh flag of `getChange()` is set to True.

2022-01-24 11:31:50,815 ERROR zuul.Scheduler: Exception processing pipeline periodic-xy in tenant foobar
Traceback (most recent call last):
  File "/opt/zuul/lib/python3.8/site-packages/zuul/scheduler.py", line 1786, in process_pipelines
    pipeline.state.refresh(ctx)
  File "/opt/zuul/lib/python3.8/site-packages/zuul/zk/zkobject.py", line 153, in refresh
    self._load(context)
  File "/opt/zuul/lib/python3.8/site-packages/zuul/zk/zkobject.py", line 205, in _load
    self._set(**self.deserialize(data, context))
  File "/opt/zuul/lib/python3.8/site-packages/zuul/model.py", line 690, in deserialize
    queue = ChangeQueue.fromZK(context, queue_path,
  File "/opt/zuul/lib/python3.8/site-packages/zuul/zk/zkobject.py", line 148, in fromZK
    obj._load(context, path=path)
  File "/opt/zuul/lib/python3.8/site-packages/zuul/zk/zkobject.py", line 205, in _load
    self._set(**self.deserialize(data, context))
  File "/opt/zuul/lib/python3.8/site-packages/zuul/model.py", line 944, in deserialize
    item = QueueItem.fromZK(context, item_path,
  File "/opt/zuul/lib/python3.8/site-packages/zuul/zk/zkobject.py", line 148, in fromZK
    obj._load(context, path=path)
  File "/opt/zuul/lib/python3.8/site-packages/zuul/zk/zkobject.py", line 205, in _load
    self._set(**self.deserialize(data, context))
  File "/opt/zuul/lib/python3.8/site-packages/zuul/model.py", line 4093, in deserialize
    change = self.pipeline.manager.resolveChangeReferences(
  File "/opt/zuul/lib/python3.8/site-packages/zuul/manager/__init__.py", line 199, in resolveChangeReferences
    return self.resolveChangeKeys(
  File "/opt/zuul/lib/python3.8/site-packages/zuul/manager/__init__.py", line 211, in resolveChangeKeys
    self._change_cache[change.cache_key] = change
AttributeError: 'NoneType' object has no attribute 'cache_key'
2022-01-24 11:31:50,811 ERROR zuul.Pipeline.foobar.periodic-xy: Unable to resolve change from key <ChangeKey github org/project Branch master None hash=da4e62f669e51a7fbef5db1a9b480b0bd42693a7febffdb47a4eb794faa300a9>

Change-Id: I62f5b816780e244e1426ab8a8871f09379293f3e

1 files changed, 32 insertions, 0 deletions

diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 8e42e47b7..d7f6e36be 100644
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -23,6 +23,7 @@ import threading
 import time
 from collections import namedtuple
 from unittest import mock, skip
+from uuid import uuid4
 from kazoo.exceptions import NoNodeError
 
 import git
@@ -4241,6 +4242,17 @@ class TestScheduler(ZuulTestCase):
         self.commitConfigUpdate('common-config', config_file)
         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
 
+        # Collect the currently cached branches in order to later check,
+        # that the timer driver refreshes the cache.
+        cached_versions = {}
+        tenant = self.scheds.first.sched.abide.tenants['tenant-one']
+        for project_name in tenant.layout.project_configs:
+            _, project = tenant.getProject('org/project')
+            for branch in project.source.getProjectBranches(project, tenant):
+                event = self._create_dummy_event(project, branch)
+                change = project.source.getChange(event)
+                cached_versions[branch] = change.cache_version
+
         # The pipeline triggers every second, so we should have seen
         # several by now.
         for _ in iterate_timeout(60, 'jobs started'):
@@ -4282,6 +4294,26 @@ class TestScheduler(ZuulTestCase):
             self.assertEqual(job.name, 'project-bitrot')
             self.assertIn(job.ref, ('refs/heads/stable', 'refs/heads/master'))
 
+        for project_name in tenant.layout.project_configs:
+            _, project = tenant.getProject('org/project')
+            for branch in project.source.getProjectBranches(project, tenant):
+                event = self._create_dummy_event(project, branch)
+                change = project.source.getChange(event)
+                # Make sure the timer driver refreshed the cache
+                self.assertGreater(change.cache_version,
+                                   cached_versions[branch])
+
+    def _create_dummy_event(self, project, branch):
+        event = zuul.model.TriggerEvent()
+        event.type = 'test'
+        event.project_hostname = project.canonical_hostname
+        event.project_name = project.name
+        event.ref = f'refs/heads/{branch}'
+        event.branch = branch
+        event.zuul_event_id = str(uuid4().hex)
+        event.timestamp = time.time()
+        return event
+
     def test_timer(self):
         "Test that a periodic job is triggered"
         self._test_timer('layouts/timer.yaml')