Use structured change cache keys

This adds a ChangeKey class which is essentially a structured universal
identifier for a change-like object (Ref, Branch, Change, PR, whatever).

We can use this in ZK objects to reference changes, and by doing so, we
can in many cases avoid actually referencing the change objects
themselves.

This also updates the actual keys in ZK to be sha256sums of the structured
key (for brevity and simplicity of encoding).

Change-Id: I6cd62973d48ad3515f6aa8a8172b9e9c19fcda55

5 files changed, 62 insertions, 20 deletions

diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py
index 7b7107b92..bf71aba1f 100644
--- a/zuul/driver/gerrit/gerritconnection.py
+++ b/zuul/driver/gerrit/gerritconnection.py
@@ -42,7 +42,11 @@ from zuul.driver.gerrit.gerritmodel import GerritChange, GerritTriggerEvent
 from zuul.driver.git.gitwatcher import GitWatcher
 from zuul.lib.logutil import get_annotated_logger
 from zuul.model import Ref, Tag, Branch, Project
-from zuul.zk.change_cache import AbstractChangeCache, ConcurrentUpdateError
+from zuul.zk.change_cache import (
+    AbstractChangeCache,
+    ChangeKey,
+    ConcurrentUpdateError,
+)
 from zuul.zk.event_queues import ConnectionEventQueue, EventReceiverElection
 
 # HTTP timeout in seconds
@@ -311,7 +315,9 @@ class GerritEventConnector(threading.Thread):
         # cache as it may be a dependency
         if event.change_number:
             refresh = True
-            key = str((event.change_number, event.patch_number))
+            key = ChangeKey(self.connection.connection_name, None,
+                            'GerritChange', str(event.change_number),
+                            str(event.patch_number))
             if self.connection._change_cache.get(key) is None:
                 refresh = False
                 for tenant in self.connection.sched.abide.tenants.values():
@@ -760,7 +766,8 @@ class GerritConnection(ZKChangeCacheMixin, BaseConnection):
         # Ensure number and patchset are str
         number = str(number)
         patchset = str(patchset)
-        key = str((number, patchset))
+        key = ChangeKey(self.connection_name, None,
+                        'GerritChange', number, patchset)
         change = self._change_cache.get(key)
         if change and not refresh:
             return change
@@ -776,7 +783,8 @@ class GerritConnection(ZKChangeCacheMixin, BaseConnection):
 
     def _getTag(self, event):
         tag = event.ref[len('refs/tags/'):]
-        key = str((event.project_name, tag, event.newrev))
+        key = ChangeKey(self.connection_name, None,
+                        'Tag', tag, event.newrev)
         change = self._change_cache.get(key)
         if change:
             return change
@@ -794,7 +802,8 @@ class GerritConnection(ZKChangeCacheMixin, BaseConnection):
         return change
 
     def _getBranch(self, event, branch, ref):
-        key = str((event.project_name, branch, event.newrev))
+        key = ChangeKey(self.connection_name, None,
+                        'Branch', branch, event.newrev)
         change = self._change_cache.get(key)
         if change:
             return change
@@ -812,7 +821,8 @@ class GerritConnection(ZKChangeCacheMixin, BaseConnection):
         return change
 
     def _getRef(self, event):
-        key = str((event.project_name, event.ref, event.newrev))
+        key = ChangeKey(self.connection_name, None,
+                        'Ref', event.ref, event.newrev)
         change = self._change_cache.get(key)
         if change:
             return change
@@ -856,6 +866,7 @@ class GerritConnection(ZKChangeCacheMixin, BaseConnection):
                     result.message):
                 if match != change_id:
                     continue
+                # Note: This is not a ChangeCache ChangeKey
                 key = (result.number, result.current_patchset)
                 if key in seen:
                     continue
@@ -992,7 +1003,9 @@ class GerritConnection(ZKChangeCacheMixin, BaseConnection):
             return True
 
         data = self.queryChange(change.number)
-        key = str((change.number, change.patchset))
+        key = ChangeKey(self.connection_name, None,
+                        'GerritChange', str(change.number),
+                        str(change.patchset))
 
         def _update_change(c):
             c.update(data, self)
diff --git a/zuul/driver/git/gitconnection.py b/zuul/driver/git/gitconnection.py
index 83cc14d62..95dc0ecf8 100644
--- a/zuul/driver/git/gitconnection.py
+++ b/zuul/driver/git/gitconnection.py
@@ -23,7 +23,11 @@ from zuul.connection import BaseConnection, ZKChangeCacheMixin
 from zuul.driver.git.gitmodel import GitTriggerEvent
 from zuul.driver.git.gitwatcher import GitWatcher
 from zuul.model import Ref, Branch
-from zuul.zk.change_cache import AbstractChangeCache, ConcurrentUpdateError
+from zuul.zk.change_cache import (
+    AbstractChangeCache,
+    ChangeKey,
+    ConcurrentUpdateError,
+)
 
 
 class GitChangeCache(AbstractChangeCache):
@@ -98,7 +102,8 @@ class GitConnection(ZKChangeCacheMixin, BaseConnection):
         return refs
 
     def getChange(self, event, refresh=False):
-        key = str((event.project_name, event.ref, event.newrev))
+        key = ChangeKey(self.connection_name, event.project_name,
+                        'Ref', event.ref, event.newrev)
         change = self._change_cache.get(key)
         if change:
             return change
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index 334531cad..8dac0992b 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -48,7 +48,11 @@ from zuul.model import Ref, Branch, Tag, Project
 from zuul.exceptions import MergeFailure
 from zuul.driver.github.githubmodel import PullRequest, GithubTriggerEvent
 from zuul.model import DequeueEvent
-from zuul.zk.change_cache import AbstractChangeCache, ConcurrentUpdateError
+from zuul.zk.change_cache import (
+    AbstractChangeCache,
+    ChangeKey,
+    ConcurrentUpdateError,
+)
 from zuul.zk.event_queues import ConnectionEventQueue
 
 GITHUB_BASE_URL = 'https://api.github.com'
@@ -1281,7 +1285,9 @@ class GithubConnection(ZKChangeCacheMixin, CachedBranchConnection):
         # because it can originate from different sources (github event, manual
         # enqueue event) where some might just parse the string and forward it.
         number = int(number)
-        key = str((project.name, number, patchset))
+        key = ChangeKey(self.connection_name, project.name,
+                        'PullRequest', str(number),
+                        str(patchset))
         change = self._change_cache.get(key)
         if change and not refresh:
             return change
@@ -1330,7 +1336,8 @@ class GithubConnection(ZKChangeCacheMixin, CachedBranchConnection):
 
     def _getTag(self, project, event):
         tag = event.ref[len('refs/tags/'):]
-        key = str((event.project_name, tag, event.newrev))
+        key = ChangeKey(self.connection_name, project.name,
+                        'Tag', tag, event.newrev)
         change = self._change_cache.get(key)
         if change:
             return change
@@ -1351,7 +1358,8 @@ class GithubConnection(ZKChangeCacheMixin, CachedBranchConnection):
 
     def _getBranch(self, project, event):
         branch = event.ref[len('refs/heads/'):]
-        key = str((event.project_name, branch, event.newrev))
+        key = ChangeKey(self.connection_name, project.name,
+                        'Branch', branch, event.newrev)
         change = self._change_cache.get(key)
         if change:
             return change
@@ -1370,7 +1378,8 @@ class GithubConnection(ZKChangeCacheMixin, CachedBranchConnection):
         return change
 
     def _getRef(self, project, event):
-        key = str((event.project_name, event.ref, event.newrev))
+        key = ChangeKey(self.connection_name, project.name,
+                        'Ref', event.ref, event.newrev)
         change = self._change_cache.get(key)
         if change:
             return change
@@ -1421,6 +1430,7 @@ class GithubConnection(ZKChangeCacheMixin, CachedBranchConnection):
                 org, proj, _, num = pr.get('url').split('/')[-4:]
                 proj = pr.get('base').get('repo').get('full_name')
                 sha = pr.get('head').get('sha')
+                # This is not a ChangeKey
                 key = (proj, num, sha)
 
                 # A single tenant could have multiple projects with the same
diff --git a/zuul/driver/gitlab/gitlabconnection.py b/zuul/driver/gitlab/gitlabconnection.py
index 5c5675ad6..4b2e684fc 100644
--- a/zuul/driver/gitlab/gitlabconnection.py
+++ b/zuul/driver/gitlab/gitlabconnection.py
@@ -34,7 +34,11 @@ from zuul.lib.logutil import get_annotated_logger
 from zuul.exceptions import MergeFailure
 from zuul.model import Branch, Project, Ref, Tag
 from zuul.driver.gitlab.gitlabmodel import GitlabTriggerEvent, MergeRequest
-from zuul.zk.change_cache import AbstractChangeCache, ConcurrentUpdateError
+from zuul.zk.change_cache import (
+    AbstractChangeCache,
+    ChangeKey,
+    ConcurrentUpdateError,
+)
 from zuul.zk.event_queues import ConnectionEventQueue
 
 # HTTP timeout in seconds
@@ -520,7 +524,9 @@ class GitlabConnection(ZKChangeCacheMixin, CachedBranchConnection):
     def _getChange(self, project, number, patch_number=None,
                    refresh=False, url=None, event=None):
         log = get_annotated_logger(self.log, event)
-        key = str((project.name, number, patch_number))
+        key = ChangeKey(self.connection_name, project.name,
+                        'MergeRequest', str(number),
+                        str(patch_number))
         change = self._change_cache.get(key)
         if change and not refresh:
             log.debug("Getting change from cache %s" % str(key))
@@ -576,7 +582,8 @@ class GitlabConnection(ZKChangeCacheMixin, CachedBranchConnection):
         return change
 
     def _getNonMRRef(self, project, event):
-        key = str((project.name, event.ref, event.newrev))
+        key = ChangeKey(self.connection_name, project.name,
+                        'Ref', event.ref, event.newrev)
         change = self._change_cache.get(key)
         if change:
             return change
diff --git a/zuul/driver/pagure/pagureconnection.py b/zuul/driver/pagure/pagureconnection.py
index 4de5435ca..8d6edeeac 100644
--- a/zuul/driver/pagure/pagureconnection.py
+++ b/zuul/driver/pagure/pagureconnection.py
@@ -28,7 +28,11 @@ from zuul.lib.logutil import get_annotated_logger
 from zuul.web.handler import BaseWebController
 from zuul.model import Ref, Branch, Tag
 from zuul.lib import dependson
-from zuul.zk.change_cache import AbstractChangeCache, ConcurrentUpdateError
+from zuul.zk.change_cache import (
+    AbstractChangeCache,
+    ChangeKey,
+    ConcurrentUpdateError,
+)
 from zuul.zk.event_queues import ConnectionEventQueue
 
 from zuul.driver.pagure.paguremodel import PagureTriggerEvent, PullRequest
@@ -596,7 +600,9 @@ class PagureConnection(ZKChangeCacheMixin, BaseConnection):
 
     def _getChange(self, project, number, patchset=None,
                    refresh=False, url=None, event=None):
-        key = str((project.name, number, patchset))
+        key = ChangeKey(self.connection_name, project.name,
+                        'PullRequest', str(number),
+                        str(patchset))
         change = self._change_cache.get(key)
         if change and not refresh:
             self.log.debug("Getting change from cache %s" % str(key))
@@ -629,7 +635,8 @@ class PagureConnection(ZKChangeCacheMixin, BaseConnection):
         return change
 
     def _getNonPRRef(self, project, event):
-        key = str((project.name, event.ref, event.newrev))
+        key = ChangeKey(self.connection_name, project.name,
+                        'Ref', event.ref, event.newrev)
         change = self._change_cache.get(key)
         if change:
             return change