Merge "Don't clear connection caches during full reconfig"

12 files changed, 54 insertions, 44 deletions

diff --git a/tests/unit/test_circular_dependencies.py b/tests/unit/test_circular_dependencies.py
index 846513821..40c98920e 100644
--- a/tests/unit/test_circular_dependencies.py
+++ b/tests/unit/test_circular_dependencies.py
@@ -1388,15 +1388,21 @@ class TestGithubCircularDependencies(ZuulTestCase):
             'master', True)
         github.repo_from_project('gh/project')._set_branch_protection(
             'stable/foo', True)
+        pevent = self.fake_github.getPushEvent(project='gh/project',
+                                               ref='refs/heads/stable/foo')
+        self.fake_github.emitEvent(pevent)
 
         self.create_branch('gh/project1', 'stable/bar')
         github.repo_from_project('gh/project1')._set_branch_protection(
             'master', True)
         github.repo_from_project('gh/project1')._set_branch_protection(
             'stable/bar', True)
+        pevent = self.fake_github.getPushEvent(project='gh/project',
+                                               ref='refs/heads/stable/bar')
+        self.fake_github.emitEvent(pevent)
 
-        # Reconfigure to pick up branch protection settings
-        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
+        # Wait until push events are processed to pick up branch
+        # protection settings
         self.waitUntilSettled()
 
         A = self.fake_github.openFakePullRequest(
diff --git a/tests/unit/test_configloader.py b/tests/unit/test_configloader.py
index bf8d2fd61..a0b4bfad0 100644
--- a/tests/unit/test_configloader.py
+++ b/tests/unit/test_configloader.py
@@ -434,8 +434,13 @@ class TestTenantConfigBranches(ZuulTestCase):
         self.log.debug('Creating branches')
         self.create_branch('common-config', 'stable')
         self.create_branch('common-config', 'feat_x')
-
-        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'common-config', 'stable'))
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'common-config', 'feat_x'))
+        self.waitUntilSettled()
 
         # Job must be defined in master
         self._validate_job(common_job, 'master')
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
index 757601fdd..f6df5eb22 100644
--- a/tests/unit/test_github_driver.py
+++ b/tests/unit/test_github_driver.py
@@ -1339,7 +1339,10 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
         github = self.fake_github.getGithubClient()
         repo = github.repo_from_project('org/project2')
         repo._set_branch_protection('master', True)
-        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
+
+        pevent = self.fake_github.getPushEvent(project='org/project2',
+                                               ref='refs/heads/master')
+        self.fake_github.emitEvent(pevent)
         self.waitUntilSettled()
 
         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
@@ -1470,6 +1473,9 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
         github = self.fake_github.getGithubClient()
         repo = github.repo_from_project('org/project2')
         repo._set_branch_protection('master', True)
+        self.fake_github.emitEvent(
+            self.fake_github.getPushEvent(
+                project='org/project2', ref='refs/heads/master'))
 
         A = self.fake_github.openFakePullRequest('org/project2', 'master', 'A')
         A.setMerged("merging A")
@@ -1477,8 +1483,10 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
         # add a spare branch so that the project is not empty after master gets
         # deleted.
         repo._create_branch('feat-x')
+        self.fake_github.emitEvent(
+            self.fake_github.getPushEvent(
+                project='org/project2', ref='refs/heads/feat-x'))
 
-        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
         self.waitUntilSettled()
 
         # record previous tenant reconfiguration time, which may not be set
diff --git a/tests/unit/test_gitlab_driver.py b/tests/unit/test_gitlab_driver.py
index 8458769ad..7a6d5e5ba 100644
--- a/tests/unit/test_gitlab_driver.py
+++ b/tests/unit/test_gitlab_driver.py
@@ -726,7 +726,8 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
 
         # now enable branch protection and trigger reload
         self.fake_gitlab.protectBranch('org', 'project2', 'master')
-        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
+        pevent = self.fake_gitlab.getPushEvent(project='org/project2')
+        self.fake_gitlab.emitEvent(pevent)
         self.waitUntilSettled()
 
         tenant = self.scheds.first.sched.abide.tenants.get('tenant-one')
@@ -869,6 +870,9 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
 
         # Prepare repo with an initial commit and enable branch protection
         self.fake_gitlab.protectBranch('org', 'project2', 'master')
+        self.fake_gitlab.emitEvent(
+            self.fake_gitlab.getPushEvent(
+                project='org/project2', branch='refs/heads/master'))
 
         A = self.fake_gitlab.openFakeMergeRequest('org/project2', 'master',
                                                   'A')
@@ -879,8 +883,9 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
         self.create_branch('org/project2', 'feat-x')
         self.fake_gitlab.protectBranch('org', 'project2', 'feat-x',
                                        protected=False)
-
-        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
+        self.fake_gitlab.emitEvent(
+            self.fake_gitlab.getPushEvent(
+                project='org/project2', branch='refs/heads/feat-x'))
         self.waitUntilSettled()
 
         # record previous tenant reconfiguration time, which may not be set
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index eddab51fc..0ed05c914 100644
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -4072,6 +4072,9 @@ class TestScheduler(ZuulTestCase):
         # timer-triggered job so that we have an opportunity to set
         # the hold flag before the first job.
         self.create_branch('org/project', 'stable')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project', 'stable'))
         self.executor_server.hold_jobs_in_build = True
         self.commitConfigUpdate('common-config', 'layouts/timer-template.yaml')
         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
@@ -4122,6 +4125,9 @@ class TestScheduler(ZuulTestCase):
         # timer-triggered job so that we have an opportunity to set
         # the hold flag before the first job.
         self.create_branch('org/project', 'stable')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project', 'stable'))
         self.executor_server.hold_jobs_in_build = True
         self.commitConfigUpdate('common-config', config_file)
         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 6b3d7ce24..652a793d9 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -2541,6 +2541,8 @@ class TestGlobalRepoState(AnsibleZuulTestCase):
         github.repo_from_project(
             'org/requiringproject-github')._set_branch_protection(
             'master', True)
+        self.fake_github.emitEvent(self.fake_github.getPushEvent(
+            'org/requiringproject-github', ref='refs/heads/master'))
 
         # Create unprotected branch feat-x. This branch will be the target
         # of override-checkout
@@ -2548,9 +2550,11 @@ class TestGlobalRepoState(AnsibleZuulTestCase):
         repo._set_branch_protection('master', True)
         repo._create_branch('feat-x')
         self.create_branch('org/requiredproject-github', 'feat-x')
+        self.fake_github.emitEvent(self.fake_github.getPushEvent(
+            'org/requiredproject-github', ref='refs/heads/feat-x'))
 
-        # Reconfigure to ensure zuul knows about the branch protection
-        self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
+        # Wait until Zuul has processed the push events and knows about
+        # the branch protection
         self.waitUntilSettled()
 
         A = self.fake_github.openFakePullRequest(
diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py
index c282f869f..99e0eaf8a 100644
--- a/tests/unit/test_web.py
+++ b/tests/unit/test_web.py
@@ -1794,6 +1794,9 @@ class TestTenantScopedWebApi(BaseTestWeb):
         """Test that the admin web interface can dequeue a change"""
         start_builds = len(self.builds)
         self.create_branch('org/project', 'stable')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project', 'stable'))
         self.executor_server.hold_jobs_in_build = True
         self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
@@ -2538,6 +2541,9 @@ class TestCLIViaWebApi(BaseTestWeb):
         """Test that the CLI can dequeue a change via REST"""
         start_builds = len(self.builds)
         self.create_branch('org/project', 'stable')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project', 'stable'))
         self.executor_server.hold_jobs_in_build = True
         self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
diff --git a/tests/zuul_client/test_zuulclient.py b/tests/zuul_client/test_zuulclient.py
index 7a51becd9..51dd80eae 100644
--- a/tests/zuul_client/test_zuulclient.py
+++ b/tests/zuul_client/test_zuulclient.py
@@ -291,6 +291,9 @@ class TestZuulClientAdmin(BaseTestWeb):
         self.executor_server.hold_jobs_in_build = True
         start_builds = len(self.builds)
         self.create_branch('org/project', 'stable')
+        self.fake_gerrit.addEvent(
+            self.fake_gerrit.getFakeBranchCreatedEvent(
+                'org/project', 'stable'))
         self.executor_server.hold_jobs_in_build = True
         self.commitConfigUpdate('common-config', 'layouts/timer.yaml')
         self.scheds.execute(lambda app: app.sched.reconfigure(app.config))
diff --git a/zuul/connection/__init__.py b/zuul/connection/__init__.py
index 1d83772fd..480ee7daa 100644
--- a/zuul/connection/__init__.py
+++ b/zuul/connection/__init__.py
@@ -77,17 +77,6 @@ class BaseConnection(object, metaclass=abc.ABCMeta):
     def registerScheduler(self, sched) -> None:
         self.sched = sched
 
-    def clearCache(self):
-        """Clear the cache for this connection.
-
-        This is called immediately prior to performing a full
-        reconfiguration. The cache should be cleared so that a
-        full reconfiguration can be used to correct any errors in
-        cached data.
-
-        """
-        pass
-
     def maintainCache(self, relevant):
 
         """Make cache contain relevant changes.
@@ -236,17 +225,6 @@ class CachedBranchConnection(BaseConnection):
         cache[project.name] = branches
         return branches
 
-    def clearCache(self) -> None:
-        """Clear the connection cache for all projects.
-
-        This method is called by the scheduler in order to perform a full
-        reconfiguration.
-        """
-        self.log.debug("Clearing branch cache for all branches: %s",
-                       self.connection_name)
-        self._project_branch_cache_exclude_unprotected = {}
-        self._project_branch_cache_include_unprotected = {}
-
     def clearProjectCache(self, project: Project) -> None:
         """Clear the connection cache for this project.
         """
diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py
index 078d24c99..97c4728f0 100644
--- a/zuul/driver/gerrit/gerritconnection.py
+++ b/zuul/driver/gerrit/gerritconnection.py
@@ -721,9 +721,6 @@ class GerritConnection(BaseConnection):
     def addProject(self, project: Project) -> None:
         self.projects[project.name] = project
 
-    def clearCache(self):
-        self._project_branch_cache = {}
-
     def _clearBranchCache(self, project):
         try:
             del self._project_branch_cache[project.name]
diff --git a/zuul/driver/pagure/pagureconnection.py b/zuul/driver/pagure/pagureconnection.py
index 01648aa8c..e65a329b8 100644
--- a/zuul/driver/pagure/pagureconnection.py
+++ b/zuul/driver/pagure/pagureconnection.py
@@ -534,9 +534,6 @@ class PagureConnection(BaseConnection):
         for key in remove:
             del self._change_cache[key]
 
-    def clearCache(self):
-        self.project_branch_cache = {}
-
     def getWebController(self, zuul_web):
         return PagureWebController(zuul_web, self)
 
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 70b2f0890..a5bd31b70 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -881,11 +881,6 @@ class Scheduler(threading.Thread):
             self.ansible_manager = AnsibleManager(
                 default_version=default_ansible_version)
 
-            if not event.smart:
-                for connection in self.connections.connections.values():
-                    self.log.debug("Clear cache for: %s" % connection)
-                    connection.clearCache()
-
             loader = configloader.ConfigLoader(
                 self.connections, self, self.merger, self.keystore)
             tenant_config, script = self._checkTenantSourceConf(self.config)