Add latest changes from gitlab-org/gitlab@14-4-stable-eev14.4.0-rc42

10 files changed, 204 insertions, 214 deletions

diff --git a/lib/gitlab/metrics/dashboard/service_selector.rb b/lib/gitlab/metrics/dashboard/service_selector.rb
index 641c0c76f8f..6d4b49676e5 100644
--- a/lib/gitlab/metrics/dashboard/service_selector.rb
+++ b/lib/gitlab/metrics/dashboard/service_selector.rb
@@ -30,7 +30,7 @@ module Gitlab
           # Returns a class which inherits from the BaseService
           # class that can be used to obtain a dashboard for
           # the provided params.
-          # @return [Gitlab::Metrics::Dashboard::Services::BaseService]
+          # @return [Metrics::Dashboard::BaseService]
           def call(params)
             service = services.find do |service_class|
               service_class.valid_params?(params)
diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb
index f378577f08e..c5fa1e545d7 100644
--- a/lib/gitlab/metrics/exporter/web_exporter.rb
+++ b/lib/gitlab/metrics/exporter/web_exporter.rb
@@ -15,6 +15,14 @@ module Gitlab
           end
         end
 
+        RailsMetricsInitializer = Struct.new(:app) do
+          def call(env)
+            Gitlab::Metrics::RailsSlis.initialize_request_slis_if_needed!
+
+            app.call(env)
+          end
+        end
+
         attr_reader :running
 
         # This exporter is always run on master process
@@ -45,6 +53,15 @@ module Gitlab
 
         private
 
+        def rack_app
+          app = super
+
+          Rack::Builder.app do
+            use RailsMetricsInitializer
+            run app
+          end
+        end
+
         def start_working
           @running = true
           super
diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
deleted file mode 100644
index ad45a037161..00000000000
--- a/lib/gitlab/metrics/instrumentation.rb
+++ /dev/null
@@ -1,194 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Metrics
-    # Module for instrumenting methods.
-    #
-    # This module allows instrumenting of methods without having to actually
-    # alter the target code (e.g. by including modules).
-    #
-    # Example usage:
-    #
-    #     Gitlab::Metrics::Instrumentation.instrument_method(User, :by_login)
-    module Instrumentation
-      PROXY_IVAR = :@__gitlab_instrumentation_proxy
-
-      def self.configure
-        yield self
-      end
-
-      # Returns the name of the series to use for storing method calls.
-      def self.series
-        @series ||= "#{::Gitlab::Metrics.series_prefix}method_calls"
-      end
-
-      # Instruments a class method.
-      #
-      # mod  - The module to instrument as a Module/Class.
-      # name - The name of the method to instrument.
-      def self.instrument_method(mod, name)
-        instrument(:class, mod, name)
-      end
-
-      # Instruments an instance method.
-      #
-      # mod  - The module to instrument as a Module/Class.
-      # name - The name of the method to instrument.
-      def self.instrument_instance_method(mod, name)
-        instrument(:instance, mod, name)
-      end
-
-      # Recursively instruments all subclasses of the given root module.
-      #
-      # This can be used to for example instrument all ActiveRecord models (as
-      # these all inherit from ActiveRecord::Base).
-      #
-      # This method can optionally take a block to pass to `instrument_methods`
-      # and `instrument_instance_methods`.
-      #
-      # root - The root module for which to instrument subclasses. The root
-      #        module itself is not instrumented.
-      def self.instrument_class_hierarchy(root, &block)
-        visit = root.subclasses
-
-        until visit.empty?
-          klass = visit.pop
-
-          instrument_methods(klass, &block)
-          instrument_instance_methods(klass, &block)
-
-          klass.subclasses.each { |c| visit << c }
-        end
-      end
-
-      # Instruments all public and private methods of a module.
-      #
-      # This method optionally takes a block that can be used to determine if a
-      # method should be instrumented or not. The block is passed the receiving
-      # module and an UnboundMethod. If the block returns a non truthy value the
-      # method is not instrumented.
-      #
-      # mod - The module to instrument.
-      def self.instrument_methods(mod)
-        methods = mod.methods(false) + mod.private_methods(false)
-        methods.each do |name|
-          method = mod.method(name)
-
-          if method.owner == mod.singleton_class
-            if !block_given? || block_given? && yield(mod, method)
-              instrument_method(mod, name)
-            end
-          end
-        end
-      end
-
-      # Instruments all public and private instance methods of a module.
-      #
-      # See `instrument_methods` for more information.
-      #
-      # mod - The module to instrument.
-      def self.instrument_instance_methods(mod)
-        methods = mod.instance_methods(false) + mod.private_instance_methods(false)
-        methods.each do |name|
-          method = mod.instance_method(name)
-
-          if method.owner == mod
-            if !block_given? || block_given? && yield(mod, method)
-              instrument_instance_method(mod, name)
-            end
-          end
-        end
-      end
-
-      # Returns true if a module is instrumented.
-      #
-      # mod - The module to check
-      def self.instrumented?(mod)
-        mod.instance_variable_defined?(PROXY_IVAR)
-      end
-
-      # Returns the proxy module (if any) of `mod`.
-      def self.proxy_module(mod)
-        mod.instance_variable_get(PROXY_IVAR)
-      end
-
-      # Instruments a method.
-      #
-      # type - The type (:class or :instance) of method to instrument.
-      # mod  - The module containing the method.
-      # name - The name of the method to instrument.
-      def self.instrument(type, mod, name)
-        return unless ::Gitlab::Metrics.enabled?
-
-        if type == :instance
-          target = mod
-          method_name = "##{name}"
-          method = mod.instance_method(name)
-        else
-          target = mod.singleton_class
-          method_name = ".#{name}"
-          method = mod.method(name)
-        end
-
-        label = "#{mod.name}#{method_name}"
-
-        unless instrumented?(target)
-          target.instance_variable_set(PROXY_IVAR, Module.new)
-        end
-
-        proxy_module = self.proxy_module(target)
-
-        # Some code out there (e.g. the "state_machine" Gem) checks the arity of
-        # a method to make sure it only passes arguments when the method expects
-        # any. If we were to always overwrite a method to take an `*args`
-        # signature this would break things. As a result we'll make sure the
-        # generated method _only_ accepts regular arguments if the underlying
-        # method also accepts them.
-        args_signature =
-          if method.arity == 0
-            ''
-          else
-            '*args'
-          end
-
-        method_visibility = method_visibility_for(target, name)
-
-        # We silence warnings to avoid such warnings:
-        # `Skipping set of ruby2_keywords flag for <...>
-        # (method accepts keywords or method does not accept argument splat)`
-        # as we apply ruby2_keywords 'blindly' for every instrumented method.
-        proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1
-          def #{name}(#{args_signature})
-            if trans = Gitlab::Metrics::Instrumentation.transaction
-              trans.method_call_for(#{label.to_sym.inspect}, #{mod.name.inspect}, "#{method_name}")
-                .measure { super }
-            else
-              super
-            end
-          end
-          silence_warnings { ruby2_keywords(:#{name}) if respond_to?(:ruby2_keywords, true) }
-          #{method_visibility} :#{name}
-        EOF
-
-        target.prepend(proxy_module)
-      end
-
-      def self.method_visibility_for(mod, name)
-        if mod.private_method_defined?(name)
-          :private
-        elsif mod.protected_method_defined?(name)
-          :protected
-        else
-          :public
-        end
-      end
-      private_class_method :method_visibility_for
-
-      # Small layer of indirection to make it easier to stub out the current
-      # transaction.
-      def self.transaction
-        Transaction.current
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/metrics/rails_slis.rb b/lib/gitlab/metrics/rails_slis.rb
new file mode 100644
index 00000000000..69e0c1e9fde
--- /dev/null
+++ b/lib/gitlab/metrics/rails_slis.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module RailsSlis
+      class << self
+        def request_apdex_counters_enabled?
+          Feature.enabled?(:request_apdex_counters)
+        end
+
+        def initialize_request_slis_if_needed!
+          return unless request_apdex_counters_enabled?
+          return if Gitlab::Metrics::Sli.initialized?(:rails_request_apdex)
+
+          Gitlab::Metrics::Sli.initialize_sli(:rails_request_apdex, possible_request_labels)
+        end
+
+        def request_apdex
+          Gitlab::Metrics::Sli[:rails_request_apdex]
+        end
+
+        private
+
+        def possible_request_labels
+          possible_controller_labels + possible_api_labels
+        end
+
+        def possible_api_labels
+          Gitlab::RequestEndpoints.all_api_endpoints.map do |route|
+            endpoint_id = API::Base.endpoint_id_for_route(route)
+            route_class = route.app.options[:for]
+            feature_category = route_class.feature_category_for_app(route.app)
+
+            {
+              endpoint_id: endpoint_id,
+              feature_category: feature_category
+            }
+          end
+        end
+
+        def possible_controller_labels
+          Gitlab::RequestEndpoints.all_controller_actions.map do |controller, action|
+            {
+              endpoint_id: controller.endpoint_id_for_action(action),
+              feature_category: controller.feature_category_for_action(action)
+            }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb
index 6ba336d37cd..3a0e34d5615 100644
--- a/lib/gitlab/metrics/requests_rack_middleware.rb
+++ b/lib/gitlab/metrics/requests_rack_middleware.rb
@@ -15,7 +15,8 @@ module Gitlab
 
       HEALTH_ENDPOINT = %r{^/-/(liveness|readiness|health|metrics)/?$}.freeze
 
-      FEATURE_CATEGORY_DEFAULT = 'unknown'
+      FEATURE_CATEGORY_DEFAULT = ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT
+      ENDPOINT_MISSING = 'unknown'
 
       # These were the top 5 categories at a point in time, chosen as a
       # reasonable default. If we initialize every category we'll end up
@@ -77,6 +78,8 @@ module Gitlab
 
           if !health_endpoint && ::Gitlab::Metrics.record_duration_for_status?(status)
             self.class.http_request_duration_seconds.observe({ method: method }, elapsed)
+
+            record_apdex_if_needed(env, elapsed)
           end
 
           [status, headers, body]
@@ -105,6 +108,39 @@ module Gitlab
       def feature_category
         ::Gitlab::ApplicationContext.current_context_attribute(:feature_category)
       end
+
+      def endpoint_id
+        ::Gitlab::ApplicationContext.current_context_attribute(:caller_id)
+      end
+
+      def record_apdex_if_needed(env, elapsed)
+        return unless Gitlab::Metrics::RailsSlis.request_apdex_counters_enabled?
+
+        Gitlab::Metrics::RailsSlis.request_apdex.increment(
+          labels: labels_from_context,
+          success: satisfactory?(env, elapsed)
+        )
+      end
+
+      def labels_from_context
+        {
+          feature_category: feature_category.presence || FEATURE_CATEGORY_DEFAULT,
+          endpoint_id: endpoint_id.presence || ENDPOINT_MISSING
+        }
+      end
+
+      def satisfactory?(env, elapsed)
+        target =
+          if env['api.endpoint'].present?
+            env['api.endpoint'].options[:for].try(:urgency_for_app, env['api.endpoint'])
+          elsif env['action_controller.instance'].present? && env['action_controller.instance'].respond_to?(:urgency)
+            env['action_controller.instance'].urgency
+          end
+
+        target ||= Gitlab::EndpointAttributes::DEFAULT_URGENCY
+
+        elapsed < target.duration
+      end
     end
   end
 end
diff --git a/lib/gitlab/metrics/sli.rb b/lib/gitlab/metrics/sli.rb
new file mode 100644
index 00000000000..de73db0755d
--- /dev/null
+++ b/lib/gitlab/metrics/sli.rb
@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    class Sli
+      SliNotInitializedError = Class.new(StandardError)
+
+      COUNTER_PREFIX = 'gitlab_sli'
+
+      class << self
+        INITIALIZATION_MUTEX = Mutex.new
+
+        def [](name)
+          known_slis[name] || initialize_sli(name, [])
+        end
+
+        def initialize_sli(name, possible_label_combinations)
+          INITIALIZATION_MUTEX.synchronize do
+            sli = new(name)
+            sli.initialize_counters(possible_label_combinations)
+            known_slis[name] = sli
+          end
+        end
+
+        def initialized?(name)
+          known_slis.key?(name) && known_slis[name].initialized?
+        end
+
+        private
+
+        def known_slis
+          @known_slis ||= {}
+        end
+      end
+
+      attr_reader :name
+
+      def initialize(name)
+        @name = name
+        @initialized_with_combinations = false
+      end
+
+      def initialize_counters(possible_label_combinations)
+        @initialized_with_combinations = possible_label_combinations.any?
+        possible_label_combinations.each do |label_combination|
+          total_counter.get(label_combination)
+          success_counter.get(label_combination)
+        end
+      end
+
+      def increment(labels:, success:)
+        total_counter.increment(labels)
+        success_counter.increment(labels) if success
+      end
+
+      def initialized?
+        @initialized_with_combinations
+      end
+
+      private
+
+      def total_counter
+        prometheus.counter(total_counter_name.to_sym, "Total number of measurements for #{name}")
+      end
+
+      def success_counter
+        prometheus.counter(success_counter_name.to_sym, "Number of successful measurements for #{name}")
+      end
+
+      def total_counter_name
+        "#{COUNTER_PREFIX}:#{name}:total"
+      end
+
+      def success_counter_name
+        "#{COUNTER_PREFIX}:#{name}:success_total"
+      end
+
+      def prometheus
+        Gitlab::Metrics
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index 59b2f88041f..df0582149a9 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -47,13 +47,11 @@ module Gitlab
             buckets SQL_DURATION_BUCKET
           end
 
-          if ::Gitlab::Database::LoadBalancing.enable?
-            db_role = ::Gitlab::Database::LoadBalancing.db_role_for_connection(payload[:connection])
-            return if db_role.blank?
+          db_role = ::Gitlab::Database::LoadBalancing.db_role_for_connection(payload[:connection])
+          return if db_role.blank?
 
-            increment_db_role_counters(db_role, payload)
-            observe_db_role_duration(db_role, event)
-          end
+          increment_db_role_counters(db_role, payload)
+          observe_db_role_duration(db_role, event)
         end
 
         def self.db_counter_payload
@@ -64,7 +62,7 @@ module Gitlab
               payload[key] = Gitlab::SafeRequestStore[key].to_i
             end
 
-            if ::Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable?
+            if ::Gitlab::SafeRequestStore.active?
               load_balancing_metric_counter_keys.each do |counter|
                 payload[counter] = ::Gitlab::SafeRequestStore[counter].to_i
               end
diff --git a/lib/gitlab/metrics/subscribers/load_balancing.rb b/lib/gitlab/metrics/subscribers/load_balancing.rb
index 333fc63ebef..bd77e8c3c3f 100644
--- a/lib/gitlab/metrics/subscribers/load_balancing.rb
+++ b/lib/gitlab/metrics/subscribers/load_balancing.rb
@@ -10,7 +10,7 @@ module Gitlab
         LOG_COUNTERS = { true => :caught_up_replica_pick_ok, false => :caught_up_replica_pick_fail }.freeze
 
         def caught_up_replica_pick(event)
-          return unless Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable?
+          return unless Gitlab::SafeRequestStore.active?
 
           result = event.payload[:result]
           counter_name = counter(result)
@@ -20,13 +20,13 @@ module Gitlab
 
         # we want to update Prometheus counter after the controller/action are set
         def web_transaction_completed(_event)
-          return unless Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable?
+          return unless Gitlab::SafeRequestStore.active?
 
           LOG_COUNTERS.keys.each { |result| increment_prometheus_for_result_label(result) }
         end
 
         def self.load_balancing_payload
-          return {} unless Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable?
+          return {} unless Gitlab::SafeRequestStore.active?
 
           {}.tap do |payload|
             LOG_COUNTERS.values.each do |counter|
diff --git a/lib/gitlab/metrics/subscribers/rack_attack.rb b/lib/gitlab/metrics/subscribers/rack_attack.rb
index ebd0d1634e7..d86c0f83c6c 100644
--- a/lib/gitlab/metrics/subscribers/rack_attack.rb
+++ b/lib/gitlab/metrics/subscribers/rack_attack.rb
@@ -22,7 +22,8 @@ module Gitlab
           :throttle_authenticated_protected_paths_web,
           :throttle_authenticated_packages_api,
           :throttle_authenticated_git_lfs,
-          :throttle_authenticated_files_api
+          :throttle_authenticated_files_api,
+          :throttle_authenticated_deprecated_api
         ].freeze
 
         PAYLOAD_KEYS = [
diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb
index 3ebfcc43b0b..544c142f7bb 100644
--- a/lib/gitlab/metrics/web_transaction.rb
+++ b/lib/gitlab/metrics/web_transaction.rb
@@ -57,10 +57,6 @@ module Gitlab
 
         action = "#{controller.action_name}"
 
-        # Try to get the feature category, but don't fail when the controller is
-        # not an ApplicationController.
-        feature_category = controller.class.try(:feature_category_for_action, action).to_s
-
         # Devise exposes a method called "request_format" that does the below.
         # However, this method is not available to all controllers (e.g. certain
         # Doorkeeper controllers). As such we use the underlying code directly.
@@ -91,9 +87,6 @@ module Gitlab
         if route
           path = endpoint_paths_cache[route.request_method][route.path]
 
-          grape_class = endpoint.options[:for]
-          feature_category = grape_class.try(:feature_category_for_app, endpoint).to_s
-
           { controller: 'Grape', action: "#{route.request_method} #{path}", feature_category: feature_category }
         end
       end
@@ -109,6 +102,10 @@ module Gitlab
       def endpoint_instrumentable_path(raw_path)
         raw_path.sub('(.:format)', '').sub('/:version', '')
       end
+
+      def feature_category
+        ::Gitlab::ApplicationContext.current_context_attribute(:feature_category) || ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT
+      end
     end
   end
 end