# Copyright 2014 Rackspace Australia # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import abc import logging from zuul.lib.logutil import get_annotated_logger from zuul import model class ReadOnlyBranchCacheError(RuntimeError): pass class BaseConnection(object, metaclass=abc.ABCMeta): """Base class for connections. A connection is a shared object that sources, triggers and reporters can use to speak with a remote API without needing to establish a new connection each time or without having to authenticate each time. Multiple instances of the same connection may exist with different credentials, for example, thus allowing for different pipelines to operate on different Gerrit installations or post back as a different user etc. Connections can implement their own public methods. Required connection methods are validated by the {trigger, source, reporter} they are loaded into. For example, a trigger will likely require some kind of query method while a reporter may need a review method.""" log = logging.getLogger('zuul.BaseConnection') def __init__(self, driver, connection_name, connection_config): # connection_name is the name given to this connection in zuul.ini # connection_config is a dictionary of config_section from zuul.ini for # this connection. # __init__ shouldn't make the actual connection in case this connection # isn't used in the layout. self.driver = driver self.connection_name = connection_name self.connection_config = connection_config self.sched = None def logEvent(self, event): log = get_annotated_logger(self.log, event.zuul_event_id) log.debug('Scheduling event from {connection}: {event}'.format( connection=self.connection_name, event=event)) try: if self.sched.statsd: self.sched.statsd.incr( 'zuul.event.{driver}.{event}'.format( driver=self.driver.name, event=event.type)) self.sched.statsd.incr( 'zuul.event.{driver}.{connection}.{event}'.format( driver=self.driver.name, connection=self.connection_name, event=event.type)) except Exception: self.log.exception("Exception reporting event stats") def onLoad(self, zk_client, component_registry): pass def onStop(self): pass def registerScheduler(self, sched) -> None: self.sched = sched def cleanupCache(self): """Clean up the connection cache. This allows a connection to perform periodic cleanup actions of the cache, e.g. garbage collection. """ pass def maintainCache(self, relevant, max_age): """Remove stale changes from the cache. This lets the user supply a list of change cache keys that are still in use. Anything in our cache that isn't in the supplied list and is older than the given max. age (in seconds) should be safe to remove from the cache.""" pass def getWebController(self, zuul_web): """Return a cherrypy web controller to register with zuul-web. :param zuul.web.ZuulWeb zuul_web: Zuul Web instance. :returns: A `zuul.web.handler.BaseWebController` instance. """ return None def getEventQueue(self): """Return the event queue for this connection. :returns: A `zuul.zk.event_queues.ConnectionEventQueue` instance or None. """ return None def validateWebConfig(self, config, connections): """Validate web config. If there is a fatal error, the method should raise an exception. :param config: The parsed config object. :param zuul.lib.connections.ConnectionRegistry connections: Registry of all configured connections. """ return False def toDict(self): """Return public information about the connection """ return { "name": self.connection_name, "driver": self.driver.name, } class ZKBranchCacheMixin: # Expected to be defined by the connection and to be an instance # of BranchCache _branch_cache = None read_only = False @abc.abstractmethod def isBranchProtected(self, project_name, branch_name, zuul_event_id): """Return if the branch is protected or None if the branch is unknown. :param str project_name: The name of the project. :param str branch_name: The name of the branch. :param zuul_event_id: Unique id associated to the handled event. """ pass @abc.abstractmethod def _fetchProjectBranches(self, project, exclude_unprotected): """Perform a remote query to determine the project's branches. Connection subclasses should implement this method. :param model.Project project: The project. :param bool exclude_unprotected: Whether the query should exclude unprotected branches from the response. :returns: A list of branch names. """ def _fetchProjectMergeModes(self, project): """Perform a remote query to determine the project's supported merge modes. Connection subclasses should implement this method if they are able to determine which merge modes apply for a project. The default implemantion returns that all merge modes are valid. :param model.Project project: The project. :returns: A list of merge modes as model IDs. """ return model.ALL_MERGE_MODES def clearConnectionCacheOnBranchEvent(self, event): """Update event and clear connection cache if needed. This checks whether the event created or deleted a branch so that Zuul may know to perform a reconfiguration on the project. Drivers must call this method when a branch event is received. :param event: The event, inherit from `zuul.model.TriggerEvent` class. """ if event.oldrev == '0' * 40: event.branch_created = True elif event.newrev == '0' * 40: event.branch_deleted = True else: event.branch_updated = True project = self.source.getProject(event.project_name) if event.branch: if event.branch_deleted: # We currently cannot determine if a deleted branch was # protected so we need to assume it was. GitHub/GitLab don't # allow deletion of protected branches but we don't get a # notification about branch protection settings. Thus we don't # know if branch protection has been disabled before deletion # of the branch. # FIXME(tobiash): Find a way to handle that case self.updateProjectBranches(project) elif event.branch_created: # In GitHub, a new branch never can be protected # because that needs to be configured after it has # been created. Other drivers could optimize this, # but for the moment, implement the lowest common # denominator and clear the cache so that we query. self.updateProjectBranches(project) event.branch_cache_ltime = self._branch_cache.ltime return event def updateProjectBranches(self, project): """Update the branch cache for the project. :param zuul.model.Project project: The project for which the branches are returned. """ # Figure out which queries we have a cache for protected_branches = self._branch_cache.getProjectBranches( project.name, True, default=None) all_branches = self._branch_cache.getProjectBranches( project.name, False, default=None) # Update them if we have them if protected_branches is not None: protected_branches = self._fetchProjectBranches(project, True) self._branch_cache.setProjectBranches( project.name, True, protected_branches) if all_branches is not None: all_branches = self._fetchProjectBranches(project, False) self._branch_cache.setProjectBranches( project.name, False, all_branches) merge_modes = self._fetchProjectMergeModes(project) self._branch_cache.setProjectMergeModes( project.name, merge_modes) self.log.info("Got branches for %s" % project.name) def getProjectBranches(self, project, tenant, min_ltime=-1): """Get the branch names for the given project. :param zuul.model.Project project: The project for which the branches are returned. :param zuul.model.Tenant tenant: The related tenant. :param int min_ltime: The minimum ltime to determine if we need to refresh the cache. :returns: The list of branch names. """ exclude_unprotected = tenant.getExcludeUnprotectedBranches(project) branches = None if self._branch_cache: try: branches = self._branch_cache.getProjectBranches( project.name, exclude_unprotected, min_ltime) except LookupError: if self.read_only: # A scheduler hasn't attempted to fetch them yet raise ReadOnlyBranchCacheError( "Will not fetch project branches as read-only is set") else: branches = None if branches is not None: return sorted(branches) elif self.read_only: # A scheduler has previously attempted a fetch, but got # the None due to an error; we can't retry since we're # read-only. raise RuntimeError( "Will not fetch project branches as read-only is set") # We need to perform a query try: branches = self._fetchProjectBranches(project, exclude_unprotected) except Exception: # We weren't able to get the branches. We need to tell # future schedulers to try again but tell zuul-web that we # tried and failed. Set the branches to None to indicate # that we have performed a fetch and retrieved no data. Any # time we encounter None in the cache, we will try again. if self._branch_cache: self._branch_cache.setProjectBranches( project.name, exclude_unprotected, None) raise self.log.info("Got branches for %s" % project.name) if self._branch_cache: self._branch_cache.setProjectBranches( project.name, exclude_unprotected, branches) return sorted(branches) def getProjectMergeModes(self, project, tenant, min_ltime=-1): """Get the merge modes for the given project. :param zuul.model.Project project: The project for which the merge modes are returned. :param zuul.model.Tenant tenant: The related tenant. :param int min_ltime: The minimum ltime to determine if we need to refresh the cache. :returns: The list of merge modes by model id. """ merge_modes = None if self._branch_cache: try: merge_modes = self._branch_cache.getProjectMergeModes( project.name, min_ltime) except LookupError: if self.read_only: # A scheduler hasn't attempted to fetch them yet raise ReadOnlyBranchCacheError( "Will not fetch merge modes as read-only is set") else: merge_modes = None if merge_modes is not None: return merge_modes elif self.read_only: # A scheduler has previously attempted a fetch, but got # the None due to an error; we can't retry since we're # read-only. raise RuntimeError( "Will not fetch merge_modes as read-only is set") # We need to perform a query try: merge_modes = self._fetchProjectMergeModes(project) except Exception: # We weren't able to get the merge modes. We need to tell # future schedulers to try again but tell zuul-web that we # tried and failed. Set the merge modes to None to indicate # that we have performed a fetch and retrieved no data. Any # time we encounter None in the cache, we will try again. if self._branch_cache: self._branch_cache.setProjectMergeModes( project.name, None) raise self.log.info("Got merge modes for %s" % project.name) if self._branch_cache: self._branch_cache.setProjectMergeModes( project.name, merge_modes) return merge_modes def checkBranchCache(self, project_name: str, event, protected: bool = None) -> None: """Clear the cache for a project when a branch event is processed This method must be called when a branch event is processed: if the event references a branch and the unprotected branches are excluded, the branch protection status could have been changed. :params str project_name: The project name. :params event: The event, inherit from `zuul.model.TriggerEvent` class. :params protected: If defined the caller already knows if the branch is protected so the query can be skipped. """ if protected is None: protected = self.isBranchProtected(project_name, event.branch, zuul_event_id=event) if protected is not None: # If the branch appears in the exclude_unprotected cache but # is unprotected, clear the exclude cache. # If the branch does not appear in the exclude_unprotected # cache but is protected, clear the exclude cache. # All branches should always appear in the include_unprotected # cache, so we never clear it. branches = self._branch_cache.getProjectBranches( project_name, True, default=None) if not branches: branches = [] update = False if (event.branch in branches) and (not protected): update = True if (event.branch not in branches) and (protected): update = True if update: self.log.info("Project %s branch %s protected state " "changed to %s", project_name, event.branch, protected) self._branch_cache.setProtected(project_name, event.branch, protected) event.branch_cache_ltime = self._branch_cache.ltime event.branch_protected = protected else: # This can happen if the branch was deleted in GitHub/GitLab. # In this case we assume that the branch COULD have # been protected before. The cache update is handled by # the push event, so we don't touch the cache here # again. event.branch_protected = True def clearBranchCache(self, projects=None): """Clear the branch cache In case the branch cache gets out of sync with the source, this method can be called to clear it and force querying the source the next time the cache is used. """ self._branch_cache.clear(projects) class ZKChangeCacheMixin: # Expected to be defined by the connection and to be an instance # that implements the AbstractChangeCache API. _change_cache = None def cleanupCache(self): self._change_cache.cleanup() def maintainCache(self, relevant, max_age): self._change_cache.prune(relevant, max_age) def updateChangeAttributes(self, change, **attrs): def _update_attrs(c): for name, value in attrs.items(): setattr(c, name, value) self._change_cache.updateChangeWithRetry(change.cache_stat.key, change, _update_attrs) def estimateCacheDataSize(self): return self._change_cache.estimateDataSize()