Log and pass correlation-id between Unicorn, Sidekiq and Gitaly

The Correlation ID is taken or generated from received X-Request-ID.
Then it is being passed to all executed services (sidekiq workers
or gitaly calls).

The Correlation ID is logged in all structured logs as `correlation_id`.

10 files changed, 134 insertions, 9 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index a4bf0d77eb1..8abb24e6f69 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -20,7 +20,8 @@ module API
                     Gitlab::GrapeLogging::Loggers::RouteLogger.new,
                     Gitlab::GrapeLogging::Loggers::UserLogger.new,
                     Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new,
-                    Gitlab::GrapeLogging::Loggers::PerfLogger.new
+                    Gitlab::GrapeLogging::Loggers::PerfLogger.new,
+                    Gitlab::GrapeLogging::Loggers::CorrelationIdLogger.new
                   ]
 
     allow_access_with_scope :api
@@ -84,7 +85,6 @@ module API
     content_type :txt, "text/plain"
 
     # Ensure the namespace is right, otherwise we might load Grape::API::Helpers
-    helpers ::SentryHelper
     helpers ::API::Helpers
     helpers ::API::Helpers::CommonHelpers
 
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 9fda73d5b92..2cceb2ec798 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -368,10 +368,10 @@ module API
     end
 
     def handle_api_exception(exception)
-      if sentry_enabled? && report_exception?(exception)
+      if report_exception?(exception)
         define_params_for_grape_middleware
-        sentry_context
-        Raven.capture_exception(exception, extra: params)
+        Gitlab::Sentry.context(current_user)
+        Gitlab::Sentry.track_acceptable_exception(exception, extra: params)
       end
 
       # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60
diff --git a/lib/gitlab/correlation_id.rb b/lib/gitlab/correlation_id.rb
new file mode 100644
index 00000000000..0f9bde4390e
--- /dev/null
+++ b/lib/gitlab/correlation_id.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CorrelationId
+    LOG_KEY = 'correlation_id'.freeze
+
+    class << self
+      def use_id(correlation_id, &blk)
+        # always generate a id if null is passed
+        correlation_id ||= new_id
+
+        ids.push(correlation_id || new_id)
+
+        begin
+          yield(current_id)
+        ensure
+          ids.pop
+        end
+      end
+
+      def current_id
+        ids.last
+      end
+
+      def current_or_new_id
+        current_id || new_id
+      end
+
+      private
+
+      def ids
+        Thread.current[:correlation_id] ||= []
+      end
+
+      def new_id
+        SecureRandom.uuid
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 9be553a8b86..255601382b1 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -193,6 +193,7 @@ module Gitlab
       feature = feature_stack && feature_stack[0]
       metadata['call_site'] = feature.to_s if feature
       metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage
+      metadata['correlation_id'] = Gitlab::CorrelationId.current_id if Gitlab::CorrelationId.current_id
 
       metadata.merge!(server_feature_flags)
 
diff --git a/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb b/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb
new file mode 100644
index 00000000000..fa4c5d86d44
--- /dev/null
+++ b/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# This module adds additional correlation id the grape logger
+module Gitlab
+  module GrapeLogging
+    module Loggers
+      class CorrelationIdLogger < ::GrapeLogging::Loggers::Base
+        def parameters(_, _)
+          { Gitlab::CorrelationId::LOG_KEY => Gitlab::CorrelationId.current_id }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/json_logger.rb b/lib/gitlab/json_logger.rb
index 3bff77731f6..a5a5759cc89 100644
--- a/lib/gitlab/json_logger.rb
+++ b/lib/gitlab/json_logger.rb
@@ -10,6 +10,7 @@ module Gitlab
       data = {}
       data[:severity] = severity
       data[:time] = timestamp.utc.iso8601(3)
+      data[Gitlab::CorrelationId::LOG_KEY] = Gitlab::CorrelationId.current_id
 
       case message
       when String
diff --git a/lib/gitlab/middleware/correlation_id.rb b/lib/gitlab/middleware/correlation_id.rb
new file mode 100644
index 00000000000..4eafa09183c
--- /dev/null
+++ b/lib/gitlab/middleware/correlation_id.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# A dumb middleware that steals correlation id
+# and sets it as a global context for the request
+module Gitlab
+  module Middleware
+    class CorrelationId
+      include ActionView::Helpers::TagHelper
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        Gitlab::CorrelationId.use_id(correlation_id(env)) do
+          @app.call(env)
+        end
+      end
+
+      private
+
+      def correlation_id(env)
+        if Gitlab.rails5?
+          request(env).request_id
+        else
+          request(env).uuid
+        end
+      end
+
+      def request(env)
+        ActionDispatch::Request.new(env)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb
index 8079c5882c4..46d01964eac 100644
--- a/lib/gitlab/sentry.rb
+++ b/lib/gitlab/sentry.rb
@@ -3,7 +3,8 @@
 module Gitlab
   module Sentry
     def self.enabled?
-      Rails.env.production? && Gitlab::CurrentSettings.sentry_enabled?
+      (Rails.env.production? || Rails.env.development?) &&
+        Gitlab::CurrentSettings.sentry_enabled?
     end
 
     def self.context(current_user = nil)
@@ -31,7 +32,7 @@ module Gitlab
     def self.track_exception(exception, issue_url: nil, extra: {})
       track_acceptable_exception(exception, issue_url: issue_url, extra: extra)
 
-      raise exception if should_raise?
+      raise exception if should_raise_for_dev?
     end
 
     # This should be used when you do not want to raise an exception in
@@ -43,7 +44,11 @@ module Gitlab
         extra[:issue_url] = issue_url if issue_url
         context # Make sure we've set everything we know in the context
 
-        Raven.capture_exception(exception, extra: extra)
+        tags = {
+          Gitlab::CorrelationId::LOG_KEY.to_sym => Gitlab::CorrelationId.current_id
+        }
+
+        Raven.capture_exception(exception, tags: tags, extra: extra)
       end
     end
 
@@ -55,7 +60,7 @@ module Gitlab
       end
     end
 
-    def self.should_raise?
+    def self.should_raise_for_dev?
       Rails.env.development? || Rails.env.test?
     end
   end
diff --git a/lib/gitlab/sidekiq_middleware/correlation_injector.rb b/lib/gitlab/sidekiq_middleware/correlation_injector.rb
new file mode 100644
index 00000000000..b807b3a03ed
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/correlation_injector.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqMiddleware
+    class CorrelationInjector
+      def call(worker_class, job, queue, redis_pool)
+        job[Gitlab::CorrelationId::LOG_KEY] ||=
+          Gitlab::CorrelationId.current_or_new_id
+
+        yield
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_middleware/correlation_logger.rb b/lib/gitlab/sidekiq_middleware/correlation_logger.rb
new file mode 100644
index 00000000000..cb8ff4a6284
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/correlation_logger.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqMiddleware
+    class CorrelationLogger
+      def call(worker, job, queue)
+        correlation_id = job[Gitlab::CorrelationId::LOG_KEY]
+
+        Gitlab::CorrelationId.use_id(correlation_id) do
+          yield
+        end
+      end
+    end
+  end
+end