Trace received Github events

We'll create a span when zuul-web receives a Github webhook event which
is then linked to the span for the event pre-processing step.

The pre-processing span context will be added to the trigger events and
with Icd240712b86cc22e55fb67f6787a0974d5308043 complete tracing of the
whole chain from receiving a Github event until a change is enqueued.

Change-Id: I1734a3a9e44f0ae01f5ed3453f8218945c90db58

2 files changed, 36 insertions, 6 deletions

diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index 04759c15f..1e3af1e78 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -39,11 +39,13 @@ import github3
 import github3.exceptions
 import github3.pulls
 from github3.session import AppInstallationTokenAuth
+from opentelemetry import trace
 
 from zuul.connection import (
     BaseConnection, ZKChangeCacheMixin, ZKBranchCacheMixin
 )
 from zuul.driver.github.graphql import GraphQLClient
+from zuul.lib import tracing
 from zuul.web.handler import BaseWebController
 from zuul.lib.logutil import get_annotated_logger
 from zuul.model import Ref, Branch, Tag, Project
@@ -74,7 +76,9 @@ ANNOTATION_LEVELS = {
 }
 
 EventTuple = collections.namedtuple(
-    "EventTuple", ["timestamp", "body", "event_type", "delivery"]
+    "EventTuple", [
+        "timestamp", "span_context", "body", "event_type", "delivery"
+    ]
 )
 
 
@@ -326,10 +330,19 @@ class GithubShaCache(object):
 
 
 class GithubEventProcessor(object):
+    tracer = trace.get_tracer("zuul")
+
     def __init__(self, connector, event_tuple, connection_event):
         self.connector = connector
         self.connection = connector.connection
-        self.ts, self.body, self.event_type, self.delivery = event_tuple
+        (
+            self.ts,
+            span_context,
+            self.body,
+            self.event_type,
+            self.delivery
+        ) = event_tuple
+        self.event_span = tracing.restoreSpanContext(span_context)
         logger = logging.getLogger("zuul.GithubEventProcessor")
         self.zuul_event_id = self.delivery
         self.log = get_annotated_logger(logger, self.zuul_event_id)
@@ -341,7 +354,12 @@ class GithubEventProcessor(object):
     def run(self):
         self.log.debug("Starting event processing")
         try:
-            self._process_event()
+            attributes = {"rel": "GithubEvent"}
+            link = trace.Link(self.event_span.get_span_context(),
+                              attributes=attributes)
+            with self.tracer.start_as_current_span(
+                    "GithubEventProcessing", links=[link]):
+                self._process_event()
         except Exception:
             self.log.exception("Exception when processing event:")
         finally:
@@ -854,11 +872,13 @@ class GithubEventConnector:
 
     @staticmethod
     def _eventAsTuple(event):
+        span_context = event.get("span_context")
         body = event.get("body")
         headers = event.get("headers", {})
         event_type = headers.get('x-github-event')
         delivery = headers.get('x-github-delivery')
-        return EventTuple(time.time(), body, event_type, delivery)
+        return EventTuple(
+            time.time(), span_context, body, event_type, delivery)
 
 
 class GithubUser(Mapping):
@@ -2439,6 +2459,7 @@ class GithubConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection):
 class GithubWebController(BaseWebController):
 
     log = logging.getLogger("zuul.GithubWebController")
+    tracer = trace.get_tracer("zuul")
 
     def __init__(self, zuul_web, connection):
         self.connection = connection
@@ -2470,6 +2491,7 @@ class GithubWebController(BaseWebController):
 
     @cherrypy.expose
     @cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
+    @tracer.start_as_current_span("GithubEvent")
     def payload(self):
         # Note(tobiash): We need to normalize the headers. Otherwise we will
         # have trouble to get them from the dict afterwards.
@@ -2492,7 +2514,11 @@ class GithubWebController(BaseWebController):
         # encode it as json, after decoding it as utf-8
         json_body = json.loads(body.decode('utf-8'))
 
-        data = {'headers': headers, 'body': json_body}
+        data = {
+            'headers': headers,
+            'body': json_body,
+            'span_context': tracing.getSpanContext(trace.get_current_span()),
+        }
         self.event_queue.put(data)
         return data
 
diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py
index 644b82bec..68f39cdae 100755
--- a/zuul/web/__init__.py
+++ b/zuul/web/__init__.py
@@ -18,6 +18,7 @@ import socket
 from collections import defaultdict
 from contextlib import suppress
 
+from opentelemetry import trace
 from ws4py.server.cherrypyserver import WebSocketPlugin, WebSocketTool
 from ws4py.websocket import WebSocket
 import codecs
@@ -39,7 +40,7 @@ from zuul import exceptions
 from zuul.configloader import ConfigLoader
 from zuul.connection import BaseConnection, ReadOnlyBranchCacheError
 import zuul.lib.repl
-from zuul.lib import commandsocket, encryption, streamer_utils
+from zuul.lib import commandsocket, encryption, streamer_utils, tracing
 from zuul.lib.ansible import AnsibleManager
 from zuul.lib.jsonutil import ZuulJSONEncoder
 from zuul.lib.keystorage import KeyStorage
@@ -1741,6 +1742,7 @@ class StreamManager(object):
 
 class ZuulWeb(object):
     log = logging.getLogger("zuul.web")
+    tracer = trace.get_tracer("zuul")
 
     def __init__(self,
                  config,
@@ -1749,6 +1751,7 @@ class ZuulWeb(object):
                  info: WebInfo = None):
         self.start_time = time.time()
         self.config = config
+        self.tracing = tracing.Tracing(self.config)
         self.metrics = WebMetrics()
         self.statsd = get_statsd(config)
 
@@ -2037,6 +2040,7 @@ class ZuulWeb(object):
         self._command_running = False
         self.command_socket.stop()
         self.monitoring_server.stop()
+        self.tracing.stop()
 
     def join(self):
         self.command_thread.join()