7 files changed, 241 insertions, 1 deletions

diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 8bf8a3b53cd..85afbd85fe6 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -52,11 +52,18 @@ module Gitlab
           klass = stub_class(name)
           addr = stub_address(storage)
           creds = stub_creds(storage)
-          klass.new(addr, creds)
+          klass.new(addr, creds, interceptors: interceptors)
         end
       end
     end
 
+    def self.interceptors
+      return [] unless Gitlab::Tracing.enabled?
+
+      [Gitlab::Tracing::GRPCInterceptor.instance]
+    end
+    private_class_method :interceptors
+
     def self.stub_cert_paths
       cert_paths = Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*"]
       cert_paths << OpenSSL::X509::DEFAULT_CERT_FILE if File.exist? OpenSSL::X509::DEFAULT_CERT_FILE
diff --git a/lib/gitlab/tracing/common.rb b/lib/gitlab/tracing/common.rb
new file mode 100644
index 00000000000..5e2b12e3f90
--- /dev/null
+++ b/lib/gitlab/tracing/common.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Tracing
+    module Common
+      def tracer
+        OpenTracing.global_tracer
+      end
+
+      # Convience method for running a block with a span
+      def in_tracing_span(operation_name:, tags:, child_of: nil)
+        scope = tracer.start_active_span(
+          operation_name,
+          child_of: child_of,
+          tags: tags
+        )
+        span = scope.span
+
+        # Add correlation details to the span if we have them
+        correlation_id = Gitlab::CorrelationId.current_id
+        if correlation_id
+          span.set_tag('correlation_id', correlation_id)
+        end
+
+        begin
+          yield span
+        rescue => e
+          log_exception_on_span(span, e)
+          raise e
+        ensure
+          scope.close
+        end
+      end
+
+      def log_exception_on_span(span, exception)
+        span.set_tag('error', true)
+        span.log_kv(kv_tags_for_exception(exception))
+      end
+
+      def kv_tags_for_exception(exception)
+        case exception
+        when Exception
+          {
+            'event':      'error',
+            'error.kind': exception.class.to_s,
+            'message':    Gitlab::UrlSanitizer.sanitize(exception.message),
+            'stack':      exception.backtrace.join("\n")
+          }
+        else
+          {
+            'event':        'error',
+            'error.kind':   exception.class.to_s,
+            'error.object': Gitlab::UrlSanitizer.sanitize(exception.to_s)
+          }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/grpc_interceptor.rb b/lib/gitlab/tracing/grpc_interceptor.rb
new file mode 100644
index 00000000000..6c2aab73125
--- /dev/null
+++ b/lib/gitlab/tracing/grpc_interceptor.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'opentracing'
+require 'grpc'
+
+module Gitlab
+  module Tracing
+    class GRPCInterceptor < GRPC::ClientInterceptor
+      include Common
+      include Singleton
+
+      def request_response(request:, call:, method:, metadata:)
+        wrap_with_tracing(method, 'unary', metadata) do
+          yield
+        end
+      end
+
+      def client_streamer(requests:, call:, method:, metadata:)
+        wrap_with_tracing(method, 'client_stream', metadata) do
+          yield
+        end
+      end
+
+      def server_streamer(request:, call:, method:, metadata:)
+        wrap_with_tracing(method, 'server_stream', metadata) do
+          yield
+        end
+      end
+
+      def bidi_streamer(requests:, call:, method:, metadata:)
+        wrap_with_tracing(method, 'bidi_stream', metadata) do
+          yield
+        end
+      end
+
+      private
+
+      def wrap_with_tracing(method, grpc_type, metadata)
+        tags = {
+          'component' =>  'grpc',
+          'span.kind' =>  'client',
+          'grpc.method' => method,
+          'grpc.type' =>   grpc_type
+        }
+
+        in_tracing_span(operation_name: "grpc:#{method}", tags: tags) do |span|
+          OpenTracing.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, metadata)
+
+          yield
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/rack_middleware.rb b/lib/gitlab/tracing/rack_middleware.rb
new file mode 100644
index 00000000000..e6a31293f7b
--- /dev/null
+++ b/lib/gitlab/tracing/rack_middleware.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'opentracing'
+
+module Gitlab
+  module Tracing
+    class RackMiddleware
+      include Common
+
+      REQUEST_METHOD = 'REQUEST_METHOD'
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        method = env[REQUEST_METHOD]
+
+        context = tracer.extract(OpenTracing::FORMAT_RACK, env)
+        tags = {
+          'component' =>   'rack',
+          'span.kind' =>   'server',
+          'http.method' => method,
+          'http.url' =>    self.class.build_sanitized_url_from_env(env)
+        }
+
+        in_tracing_span(operation_name: "http:#{method}", child_of: context, tags: tags) do |span|
+          @app.call(env).tap do |status_code, _headers, _body|
+            span.set_tag('http.status_code', status_code)
+          end
+        end
+      end
+
+      # Generate a sanitized (safe) request URL from the rack environment
+      def self.build_sanitized_url_from_env(env)
+        request = ActionDispatch::Request.new(env)
+
+        original_url = request.original_url
+        uri = URI.parse(original_url)
+        uri.query = request.filtered_parameters.to_query if uri.query.present?
+
+        uri.to_s
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/sidekiq/client_middleware.rb b/lib/gitlab/tracing/sidekiq/client_middleware.rb
new file mode 100644
index 00000000000..2b71c1ea21e
--- /dev/null
+++ b/lib/gitlab/tracing/sidekiq/client_middleware.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'opentracing'
+
+module Gitlab
+  module Tracing
+    module Sidekiq
+      class ClientMiddleware
+        include SidekiqCommon
+
+        SPAN_KIND = 'client'
+
+        def call(worker_class, job, queue, redis_pool)
+          in_tracing_span(
+            operation_name: "sidekiq:#{job['class']}",
+            tags: tags_from_job(job, SPAN_KIND)) do |span|
+            # Inject the details directly into the job
+            tracer.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, job)
+
+            yield
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/sidekiq/server_middleware.rb b/lib/gitlab/tracing/sidekiq/server_middleware.rb
new file mode 100644
index 00000000000..5b43c4310e6
--- /dev/null
+++ b/lib/gitlab/tracing/sidekiq/server_middleware.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'opentracing'
+
+module Gitlab
+  module Tracing
+    module Sidekiq
+      class ServerMiddleware
+        include SidekiqCommon
+
+        SPAN_KIND = 'server'
+
+        def call(worker, job, queue)
+          context = tracer.extract(OpenTracing::FORMAT_TEXT_MAP, job)
+
+          in_tracing_span(
+            operation_name: "sidekiq:#{job['class']}",
+            child_of: context,
+            tags: tags_from_job(job, SPAN_KIND)) do |span|
+            yield
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/sidekiq/sidekiq_common.rb b/lib/gitlab/tracing/sidekiq/sidekiq_common.rb
new file mode 100644
index 00000000000..a911a29d773
--- /dev/null
+++ b/lib/gitlab/tracing/sidekiq/sidekiq_common.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Tracing
+    module Sidekiq
+      module SidekiqCommon
+        include Gitlab::Tracing::Common
+
+        def tags_from_job(job, kind)
+          {
+            'component' =>     'sidekiq',
+            'span.kind' =>     kind,
+            'sidekiq.queue' => job['queue'],
+            'sidekiq.jid' =>   job['jid'],
+            'sidekiq.retry' => job['retry'].to_s,
+            'sidekiq.args' =>  job['args']&.join(", ")
+          }
+        end
+      end
+    end
+  end
+end