Add latest changes from gitlab-org/gitlab@13-1-stable-ee

10 files changed, 143 insertions, 10 deletions

diff --git a/lib/gitlab/metrics/dashboard/stages/url_validator.rb b/lib/gitlab/metrics/dashboard/stages/url_validator.rb
new file mode 100644
index 00000000000..ff36f7b605e
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/stages/url_validator.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Stages
+        class UrlValidator < BaseStage
+          def transform!
+            dashboard[:links]&.each do |link|
+              Gitlab::UrlBlocker.validate!(link[:url])
+            rescue Gitlab::UrlBlocker::BlockedUrlError
+              link[:url] = ''
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/elasticsearch_rack_middleware.rb b/lib/gitlab/metrics/elasticsearch_rack_middleware.rb
new file mode 100644
index 00000000000..6830eed68d5
--- /dev/null
+++ b/lib/gitlab/metrics/elasticsearch_rack_middleware.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    # Rack middleware for tracking Elasticsearch metrics from Grape and Web requests.
+    class ElasticsearchRackMiddleware
+      HISTOGRAM_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60].freeze
+
+      def initialize(app)
+        @app = app
+
+        @requests_total_counter = Gitlab::Metrics.counter(:http_elasticsearch_requests_total,
+                                                         'Amount of calls to Elasticsearch servers during web requests',
+                                                         Gitlab::Metrics::Transaction::BASE_LABELS)
+        @requests_duration_histogram = Gitlab::Metrics.histogram(:http_elasticsearch_requests_duration_seconds,
+                                                                 'Query time for Elasticsearch servers during web requests',
+                                                                 Gitlab::Metrics::Transaction::BASE_LABELS,
+                                                                 HISTOGRAM_BUCKETS)
+      end
+
+      def call(env)
+        transaction = Gitlab::Metrics.current_transaction
+
+        @app.call(env)
+      ensure
+        record_metrics(transaction)
+      end
+
+      private
+
+      def record_metrics(transaction)
+        labels = transaction.labels
+        query_time = ::Gitlab::Instrumentation::ElasticsearchTransport.query_time
+        request_count = ::Gitlab::Instrumentation::ElasticsearchTransport.get_request_count
+
+        @requests_total_counter.increment(labels, request_count)
+        @requests_duration_histogram.observe(labels, query_time)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb
index cee601ff14c..5955987541c 100644
--- a/lib/gitlab/metrics/methods.rb
+++ b/lib/gitlab/metrics/methods.rb
@@ -52,7 +52,7 @@ module Gitlab
         end
 
         def disabled_by_feature(options)
-          options.with_feature && !::Feature.get(options.with_feature).enabled?
+          options.with_feature && !::Feature.enabled?(options.with_feature)
         end
 
         def build_metric!(type, name, options)
diff --git a/lib/gitlab/metrics/redis_rack_middleware.rb b/lib/gitlab/metrics/redis_rack_middleware.rb
new file mode 100644
index 00000000000..f0f99c5f45d
--- /dev/null
+++ b/lib/gitlab/metrics/redis_rack_middleware.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    # Rack middleware for tracking Redis metrics from Grape and Web requests.
+    class RedisRackMiddleware
+      def initialize(app)
+        @app = app
+
+        @requests_total_counter = Gitlab::Metrics.counter(:http_redis_requests_total,
+                                                          'Amount of calls to Redis servers during web requests',
+                                                          Gitlab::Metrics::Transaction::BASE_LABELS)
+        @requests_duration_histogram = Gitlab::Metrics.histogram(:http_redis_requests_duration_seconds,
+                                                                 'Query time for Redis servers during web requests',
+                                                                 Gitlab::Metrics::Transaction::BASE_LABELS,
+                                                                 Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS)
+      end
+
+      def call(env)
+        transaction = Gitlab::Metrics.current_transaction
+
+        @app.call(env)
+      ensure
+        record_metrics(transaction)
+      end
+
+      private
+
+      def record_metrics(transaction)
+        labels = transaction.labels
+        query_time = Gitlab::Instrumentation::Redis.query_time
+        request_count = Gitlab::Instrumentation::Redis.get_request_count
+
+        @requests_total_counter.increment(labels, request_count)
+        @requests_duration_histogram.observe(labels, query_time)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb
index 90051f85f31..ff3e7be567f 100644
--- a/lib/gitlab/metrics/samplers/base_sampler.rb
+++ b/lib/gitlab/metrics/samplers/base_sampler.rb
@@ -6,8 +6,10 @@ module Gitlab
   module Metrics
     module Samplers
       class BaseSampler < Daemon
+        attr_reader :interval
+
         # interval - The sampling interval in seconds.
-        def initialize(interval)
+        def initialize(interval = self.class::SAMPLING_INTERVAL_SECONDS)
           interval_half = interval.to_f / 2
 
           @interval = interval
diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb
index 98dd517ee3b..b5343d5e66a 100644
--- a/lib/gitlab/metrics/samplers/puma_sampler.rb
+++ b/lib/gitlab/metrics/samplers/puma_sampler.rb
@@ -4,6 +4,8 @@ module Gitlab
   module Metrics
     module Samplers
       class PumaSampler < BaseSampler
+        SAMPLING_INTERVAL_SECONDS = 5
+
         def metrics
           @metrics ||= init_metrics
         end
diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb
index df59c06911b..dac9fbd1247 100644
--- a/lib/gitlab/metrics/samplers/ruby_sampler.rb
+++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb
@@ -6,9 +6,10 @@ module Gitlab
   module Metrics
     module Samplers
       class RubySampler < BaseSampler
+        SAMPLING_INTERVAL_SECONDS = 60
         GC_REPORT_BUCKETS = [0.005, 0.01, 0.02, 0.04, 0.07, 0.1, 0.5].freeze
 
-        def initialize(interval)
+        def initialize(*)
           GC::Profiler.clear
 
           metrics[:process_start_time_seconds].set(labels, Time.now.to_i)
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
index 8dfb61046c4..de8e1ca3256 100644
--- a/lib/gitlab/metrics/sidekiq_middleware.rb
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -6,19 +6,30 @@ module Gitlab
     #
     # This middleware is intended to be used as a server-side middleware.
     class SidekiqMiddleware
-      def call(worker, message, queue)
+      def call(worker, payload, queue)
         trans = BackgroundTransaction.new(worker.class)
 
         begin
           # Old gitlad-shell messages don't provide enqueued_at/created_at attributes
-          trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0))
+          enqueued_at = payload['enqueued_at'] || payload['created_at'] || 0
+          trans.set(:sidekiq_queue_duration, Time.current.to_f - enqueued_at)
           trans.run { yield }
         rescue Exception => error # rubocop: disable Lint/RescueException
           trans.add_event(:sidekiq_exception)
 
           raise error
+        ensure
+          add_info_to_payload(payload, trans)
         end
       end
+
+      private
+
+      def add_info_to_payload(payload, trans)
+        payload[:db_count] = trans.get(:db_count, :counter).to_i
+        payload[:db_write_count] = trans.get(:db_write_count, :counter).to_i
+        payload[:db_cached_count] = trans.get(:db_cached_count, :counter).to_i
+      end
     end
   end
 end
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index a02dd850582..1628eeb5a95 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -9,6 +9,7 @@ module Gitlab
         attach_to :active_record
 
         IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze
+        DB_COUNTERS = %i{db_count db_write_count db_cached_count}.freeze
 
         def sql(event)
           return unless current_transaction
@@ -19,8 +20,7 @@ module Gitlab
 
           self.class.gitlab_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0)
 
-          current_transaction.increment(:sql_duration, event.duration, false)
-          current_transaction.increment(:sql_count, 1, false)
+          increment_db_counters(payload)
         end
 
         private
@@ -31,6 +31,20 @@ module Gitlab
           buckets [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0]
         end
 
+        def select_sql_command?(payload)
+          payload[:sql].match(/\A((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i)
+        end
+
+        def increment_db_counters(payload)
+          current_transaction.increment(:db_count, 1)
+
+          if payload.fetch(:cached, payload[:name] == 'CACHE')
+            current_transaction.increment(:db_cached_count, 1)
+          end
+
+          current_transaction.increment(:db_write_count, 1) unless select_sql_command?(payload)
+        end
+
         def current_transaction
           Transaction.current
         end
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index b126efd2dd5..822f5243e9d 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -16,7 +16,7 @@ module Gitlab
       # The series to store events (e.g. Git pushes) in.
       EVENT_SERIES = 'events'
 
-      attr_reader :tags, :method
+      attr_reader :method
 
       def self.current
         Thread.current[THREAD_KEY]
@@ -28,8 +28,6 @@ module Gitlab
         @started_at = nil
         @finished_at = nil
 
-        @tags = {}
-
         @memory_before = 0
         @memory_after = 0
       end
@@ -94,6 +92,12 @@ module Gitlab
         self.class.transaction_metric(name, :gauge).set(labels, value) if use_prometheus
       end
 
+      def get(name, type, tags = {})
+        metric = self.class.transaction_metric(name, type)
+
+        metric.get(filter_tags(tags).merge(labels))
+      end
+
       def labels
         BASE_LABELS
       end