diff options
| -rw-r--r-- | tests/base.py | 1 | ||||
| -rwxr-xr-x | zuul/cmd/scheduler.py | 1 | ||||
| -rw-r--r-- | zuul/connection/__init__.py | 7 | ||||
| -rw-r--r-- | zuul/driver/gerrit/gerritconnection.py | 23 | ||||
| -rw-r--r-- | zuul/driver/git/gitconnection.py | 10 | ||||
| -rw-r--r-- | zuul/driver/github/githubconnection.py | 23 | ||||
| -rw-r--r-- | zuul/driver/gitlab/gitlabconnection.py | 23 | ||||
| -rw-r--r-- | zuul/driver/mqtt/mqttconnection.py | 2 | ||||
| -rw-r--r-- | zuul/driver/pagure/pagureconnection.py | 23 | ||||
| -rw-r--r-- | zuul/driver/sql/sqlconnection.py | 2 | ||||
| -rw-r--r-- | zuul/lib/connections.py | 8 |
11 files changed, 93 insertions, 30 deletions
diff --git a/tests/base.py b/tests/base.py index ad6e53226..80b44c0c7 100644 --- a/tests/base.py +++ b/tests/base.py @@ -4366,6 +4366,7 @@ class SchedulerTestApp: if validate_tenants is None: self.connections.registerScheduler(self.sched) + self.connections.load(self.sched.zk_client) # TODO (swestphahl): Can be removed when we no longer use global # management events. diff --git a/zuul/cmd/scheduler.py b/zuul/cmd/scheduler.py index 93c093ff4..7954e302c 100755 --- a/zuul/cmd/scheduler.py +++ b/zuul/cmd/scheduler.py @@ -141,6 +141,7 @@ class Scheduler(zuul.cmd.ZuulDaemonApp): self.connections, self) if self.args.validate_tenants is None: self.connections.registerScheduler(self.sched) + self.connections.load(self.sched.zk_client) self.log.info('Starting scheduler') try: diff --git a/zuul/connection/__init__.py b/zuul/connection/__init__.py index 5a69dbb55..03eaee300 100644 --- a/zuul/connection/__init__.py +++ b/zuul/connection/__init__.py @@ -68,7 +68,7 @@ class BaseConnection(object, metaclass=abc.ABCMeta): except Exception: self.log.exception("Exception reporting event stats") - def onLoad(self): + def onLoad(self, zk_client): pass def onStop(self): @@ -136,6 +136,7 @@ class ZKBranchCacheMixin: # Expected to be defined by the connection and to be an instance # of BranchCache _branch_cache = None + read_only = False @abc.abstractmethod def isBranchProtected(self, project_name: str, branch_name: str, @@ -243,6 +244,10 @@ class ZKBranchCacheMixin: if branches is not None: return sorted(branches) + if self.read_only: + raise RuntimeError( + "Won't fetch project branches as read-only is set.") + # We need to perform a query branches = self._fetchProjectBranches(project, exclude_unprotected) self.log.info("Got branches for %s" % project.name) diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py index a004da11f..6ab6e8a63 100644 --- a/zuul/driver/gerrit/gerritconnection.py +++ b/zuul/driver/gerrit/gerritconnection.py @@ -1580,7 +1580,7 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection): } self.addEvent(event) - def onLoad(self): + def onLoad(self, zk_client): self.log.debug("Starting Gerrit Connection/Watchers") try: if self.session: @@ -1588,15 +1588,24 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection): except Exception: self.log.exception("Unable to determine remote Gerrit version") + # Set the project branch cache to read only if no scheduler is + # provided to prevent fetching the branches from the connection. + self.read_only = not self.sched + + self.log.debug('Creating Zookeeper branch cache') + self._branch_cache = BranchCache(zk_client, self) + self.log.info("Creating Zookeeper event queue") - self.event_queue = ConnectionEventQueue(self.sched.zk_client, - self.connection_name) + self.event_queue = ConnectionEventQueue( + zk_client, self.connection_name) - self.log.debug('Creating Zookeeper change cache') - self._change_cache = GerritChangeCache(self.sched.zk_client, self) + # If the connection was not loaded by a scheduler, but by e.g. + # zuul-web, we want to stop here. + if not self.sched: + return - self.log.debug('Creating Zookeeper branch cache') - self._branch_cache = BranchCache(self.sched.zk_client, self) + self.log.debug('Creating Zookeeper change cache') + self._change_cache = GerritChangeCache(zk_client, self) if self.enable_stream_events: self._start_watcher_thread() diff --git a/zuul/driver/git/gitconnection.py b/zuul/driver/git/gitconnection.py index 6346f676b..8e52e798f 100644 --- a/zuul/driver/git/gitconnection.py +++ b/zuul/driver/git/gitconnection.py @@ -165,9 +165,15 @@ class GitConnection(ZKChangeCacheMixin, BaseConnection): # Pass the event to the scheduler self.sched.addTriggerEvent(self.driver_name, event) - def onLoad(self): + def onLoad(self, zk_client): self.log.debug("Creating Zookeeper change cache") - self._change_cache = GitChangeCache(self.sched.zk_client, self) + self._change_cache = GitChangeCache(zk_client, self) + + # If the connection was not loaded by a scheduler, but by e.g. + # zuul-web, we want to stop here. + if not self.sched: + return + self.log.debug("Starting Git Watcher") self._start_watcher_thread() diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py index 9a85cbba8..0aea99505 100644 --- a/zuul/driver/github/githubconnection.py +++ b/zuul/driver/github/githubconnection.py @@ -1229,17 +1229,30 @@ class GithubConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection): }) return d - def onLoad(self): + def onLoad(self, zk_client): self.log.info('Starting GitHub connection: %s', self.connection_name) self._github_client_manager.initialize() + + # Set the project branch cache to read only if no scheduler is + # provided to prevent fetching the branches from the connection. + self.read_only = not self.sched + + self.log.debug('Creating Zookeeper branch cache') + self._branch_cache = BranchCache(zk_client, self) + self.log.debug('Creating Zookeeper event queue') self.event_queue = ConnectionEventQueue( - self.sched.zk_client, self.connection_name + zk_client, self.connection_name ) + + # If the connection was not loaded by a scheduler, but by e.g. + # zuul-web, we want to stop here. + if not self.sched: + return + self.log.debug('Creating Zookeeper change cache') - self._change_cache = GithubChangeCache(self.sched.zk_client, self) - self.log.debug('Creating Zookeeper branch cache') - self._branch_cache = BranchCache(self.sched.zk_client, self) + self._change_cache = GithubChangeCache(zk_client, self) + self.log.info('Starting event connector') self._start_event_connector() diff --git a/zuul/driver/gitlab/gitlabconnection.py b/zuul/driver/gitlab/gitlabconnection.py index cb8b56b03..3a057b7e4 100644 --- a/zuul/driver/gitlab/gitlabconnection.py +++ b/zuul/driver/gitlab/gitlabconnection.py @@ -479,16 +479,29 @@ class GitlabConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection): self.gitlab_event_connector.stop() self.gitlab_event_connector.join() - def onLoad(self): + def onLoad(self, zk_client): self.log.info('Starting Gitlab connection: %s', self.connection_name) + + # Set the project branch cache to read only if no scheduler is + # provided to prevent fetching the branches from the connection. + self.read_only = not self.sched + + self.log.debug('Creating Zookeeper branch cache') + self._branch_cache = BranchCache(zk_client, self) + self.log.info('Creating Zookeeper event queue') self.event_queue = ConnectionEventQueue( - self.sched.zk_client, self.connection_name + zk_client, self.connection_name ) + + # If the connection was not loaded by a scheduler, but by e.g. + # zuul-web, we want to stop here. + if not self.sched: + return + self.log.debug('Creating Zookeeper change cache') - self._change_cache = GitlabChangeCache(self.sched.zk_client, self) - self.log.debug('Creating Zookeeper branch cache') - self._branch_cache = BranchCache(self.sched.zk_client, self) + self._change_cache = GitlabChangeCache(zk_client, self) + self.log.info('Starting event connector') self._start_event_connector() diff --git a/zuul/driver/mqtt/mqttconnection.py b/zuul/driver/mqtt/mqttconnection.py index 3990177ee..d75b7a3a2 100644 --- a/zuul/driver/mqtt/mqttconnection.py +++ b/zuul/driver/mqtt/mqttconnection.py @@ -62,7 +62,7 @@ class MQTTConnection(BaseConnection): def _on_disconnect(self, client, userdata, rc): self.connected = False - def onLoad(self): + def onLoad(self, zk_client): self.log.debug("Starting MQTT Connection") try: self.client.connect( diff --git a/zuul/driver/pagure/pagureconnection.py b/zuul/driver/pagure/pagureconnection.py index 798de70f3..0e3d9a776 100644 --- a/zuul/driver/pagure/pagureconnection.py +++ b/zuul/driver/pagure/pagureconnection.py @@ -492,16 +492,29 @@ class PagureConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection): r"^\*\*Metadata Update", re.MULTILINE) self.sched = None - def onLoad(self): + def onLoad(self, zk_client): self.log.info('Starting Pagure connection: %s', self.connection_name) + + # Set the project branch cache to read only if no scheduler is + # provided to prevent fetching the branches from the connection. + self.read_only = not self.sched + + self.log.debug('Creating Zookeeper branch cache') + self._branch_cache = BranchCache(zk_client, self) + self.log.info('Creating Zookeeper event queue') self.event_queue = ConnectionEventQueue( - self.sched.zk_client, self.connection_name + zk_client, self.connection_name ) + + # If the connection was not loaded by a scheduler, but by e.g. + # zuul-web, we want to stop here. + if not self.sched: + return + self.log.debug('Creating Zookeeper change cache') - self._change_cache = PagureChangeCache(self.sched.zk_client, self) - self.log.debug('Creating Zookeeper branch cache') - self._branch_cache = BranchCache(self.sched.zk_client, self) + self._change_cache = PagureChangeCache(zk_client, self) + self.log.info('Starting event connector') self._start_event_connector() diff --git a/zuul/driver/sql/sqlconnection.py b/zuul/driver/sql/sqlconnection.py index 186b71005..79293dda8 100644 --- a/zuul/driver/sql/sqlconnection.py +++ b/zuul/driver/sql/sqlconnection.py @@ -290,7 +290,7 @@ class SQLConnection(BaseConnection): else: alembic.command.upgrade(config, 'head', tag=tag) - def onLoad(self): + def onLoad(self, zk_client=None): while True: try: self._migrate() diff --git a/zuul/lib/connections.py b/zuul/lib/connections.py index 608313fc0..08051fe23 100644 --- a/zuul/lib/connections.py +++ b/zuul/lib/connections.py @@ -69,13 +69,15 @@ class ConnectionRegistry(object): raise Exception("Driver %s already registered" % driver.name) self.drivers[driver.name] = driver - def registerScheduler(self, sched, load=True): + def registerScheduler(self, sched): for driver_name, driver in self.drivers.items(): driver.registerScheduler(sched) for connection_name, connection in self.connections.items(): connection.registerScheduler(sched) - if load: - connection.onLoad() + + def load(self, zk_client): + for connection in self.connections.values(): + connection.onLoad(zk_client) def reconfigureDrivers(self, tenant): for driver in self.drivers.values(): |