Add latest changes from gitlab-org/gitlab@13-4-stable-ee

15 files changed, 326 insertions, 32 deletions

diff --git a/lib/gitlab/metrics/dashboard/importer.rb b/lib/gitlab/metrics/dashboard/importer.rb
new file mode 100644
index 00000000000..ca835650648
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/importer.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      class Importer
+        def initialize(dashboard_path, project)
+          @dashboard_path = dashboard_path.to_s
+          @project = project
+        end
+
+        def execute
+          return false unless Dashboard::Validator.validate(dashboard_hash, project: project, dashboard_path: dashboard_path)
+
+          Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute
+        rescue Gitlab::Config::Loader::FormatError
+          false
+        end
+
+        def execute!
+          Dashboard::Validator.validate!(dashboard_hash, project: project, dashboard_path: dashboard_path)
+
+          Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute!
+        end
+
+        private
+
+        attr_accessor :dashboard_path, :project
+
+        def dashboard_hash
+          @dashboard_hash ||= begin
+            raw_dashboard = Dashboard::RepoDashboardFinder.read_dashboard(project, dashboard_path)
+            return unless raw_dashboard.present?
+
+            ::Gitlab::Config::Loader::Yaml.new(raw_dashboard).load_raw!
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb
new file mode 100644
index 00000000000..d1490d5d9b6
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Importers
+        class PrometheusMetrics
+          ALLOWED_ATTRIBUTES = %i(title query y_label unit legend group dashboard_path).freeze
+
+          # Takes a JSON schema validated dashboard hash and
+          # imports metrics to database
+          def initialize(dashboard_hash, project:, dashboard_path:)
+            @dashboard_hash = dashboard_hash
+            @project = project
+            @dashboard_path = dashboard_path
+          end
+
+          def execute
+            import
+          rescue ActiveRecord::RecordInvalid, ::Gitlab::Metrics::Dashboard::Transformers::TransformerError
+            false
+          end
+
+          def execute!
+            import
+          end
+
+          private
+
+          attr_reader :dashboard_hash, :project, :dashboard_path
+
+          def import
+            delete_stale_metrics
+            create_or_update_metrics
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def create_or_update_metrics
+            # TODO: use upsert and worker for callbacks?
+            prometheus_metrics_attributes.each do |attributes|
+              prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:identifier, :project))
+              prometheus_metric.update!(attributes.slice(*ALLOWED_ATTRIBUTES))
+            end
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+
+          def delete_stale_metrics
+            identifiers = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] }
+
+            stale_metrics = PrometheusMetric.for_project(project)
+              .for_dashboard_path(dashboard_path)
+              .for_group(Enums::PrometheusMetric.groups[:custom])
+              .not_identifier(identifiers)
+
+            # TODO: use destroy_all and worker for callbacks?
+            stale_metrics.each(&:destroy)
+          end
+
+          def prometheus_metrics_attributes
+            @prometheus_metrics_attributes ||= begin
+              Dashboard::Transformers::Yml::V1::PrometheusMetrics.new(
+                dashboard_hash,
+                project: project,
+                dashboard_path: dashboard_path
+              ).execute
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/stages/custom_metrics_details_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_metrics_details_inserter.rb
index cfec027155e..06cfa5cc58e 100644
--- a/lib/gitlab/metrics/dashboard/stages/custom_metrics_details_inserter.rb
+++ b/lib/gitlab/metrics/dashboard/stages/custom_metrics_details_inserter.rb
@@ -27,7 +27,7 @@ module Gitlab
           private
 
           def custom_group_titles
-            @custom_group_titles ||= PrometheusMetricEnums.custom_group_details.values.map { |group_details| group_details[:group_title] }
+            @custom_group_titles ||= Enums::PrometheusMetric.custom_group_details.values.map { |group_details| group_details[:group_title] }
           end
 
           def edit_path(metric)
diff --git a/lib/gitlab/metrics/dashboard/transformers/errors.rb b/lib/gitlab/metrics/dashboard/transformers/errors.rb
new file mode 100644
index 00000000000..4d94ab098ae
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/transformers/errors.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Transformers
+        TransformerError = Class.new(StandardError)
+
+        module Errors
+          class MissingAttribute < TransformerError
+            def initialize(attribute_name)
+              super("Missing attribute: '#{attribute_name}'")
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb
new file mode 100644
index 00000000000..4e46eec17d6
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Transformers
+        module Yml
+          module V1
+            # Takes a JSON schema validated dashboard hash and
+            # maps it to PrometheusMetric model attributes
+            class PrometheusMetrics
+              def initialize(dashboard_hash, project: nil, dashboard_path: nil)
+                @dashboard_hash = dashboard_hash.with_indifferent_access
+                @project = project
+                @dashboard_path = dashboard_path
+
+                @dashboard_hash.default_proc = -> (h, k) { raise Transformers::Errors::MissingAttribute, k.to_s }
+              end
+
+              def execute
+                prometheus_metrics = []
+
+                dashboard_hash[:panel_groups].each do |panel_group|
+                  panel_group[:panels].each do |panel|
+                    panel[:metrics].each do |metric|
+                      prometheus_metrics << {
+                        project:        project,
+                        title:          panel[:title],
+                        y_label:        panel[:y_label],
+                        query:          metric[:query_range] || metric[:query],
+                        unit:           metric[:unit],
+                        legend:         metric[:label],
+                        identifier:     metric[:id],
+                        group:          Enums::PrometheusMetric.groups[:custom],
+                        common:         false,
+                        dashboard_path: dashboard_path
+                      }.compact
+                    end
+                  end
+                end
+
+                prometheus_metrics
+              end
+
+              private
+
+              attr_reader :dashboard_hash, :project, :dashboard_path
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb
index 160ecfb85c9..6dcc73c0f6a 100644
--- a/lib/gitlab/metrics/dashboard/url.rb
+++ b/lib/gitlab/metrics/dashboard/url.rb
@@ -10,20 +10,23 @@ module Gitlab
 
           QUERY_PATTERN = '(?<query>\?[a-zA-Z0-9%.()+_=-]+(&[a-zA-Z0-9%.()+_=-]+)*)?'
           ANCHOR_PATTERN = '(?<anchor>\#[a-z0-9_-]+)?'
-          OPTIONAL_DASH_PATTERN = '(?:/-)?'
+          DASH_PATTERN = '(?:/-)'
 
-          # Matches urls for a metrics dashboard. This could be
-          # either the /metrics endpoint or the /metrics_dashboard
-          # endpoint.
+          # Matches urls for a metrics dashboard.
+          # This regex needs to match the old metrics URL, the new metrics URL,
+          # and the dashboard URL (inline_metrics_redactor_filter.rb
+          # uses this regex to match against the dashboard URL.)
           #
-          # EX - https://<host>/<namespace>/<project>/environments/<env_id>/metrics
+          # EX - Old URL: https://<host>/<namespace>/<project>/environments/<env_id>/metrics
+          # OR
+          # New URL: https://<host>/<namespace>/<project>/-/metrics?environment=<env_id>
+          # OR
+          # dashboard URL: https://<host>/<namespace>/<project>/environments/<env_id>/metrics_dashboard
           def metrics_regex
             strong_memoize(:metrics_regex) do
               regex_for_project_metrics(
                 %r{
-                    /environments
-                    /(?<environment>\d+)
-                    /(metrics_dashboard|metrics)
+                    ( #{environment_metrics_regex} ) | ( #{non_environment_metrics_regex} )
                 }x
               )
             end
@@ -36,6 +39,7 @@ module Gitlab
             strong_memoize(:grafana_regex) do
               regex_for_project_metrics(
                 %r{
+                  #{DASH_PATTERN}?
                   /grafana
                   /metrics_dashboard
                 }x
@@ -44,16 +48,22 @@ module Gitlab
           end
 
           # Matches dashboard urls for a metric chart embed
-          # for cluster metrics
+          # for cluster metrics.
+          # This regex needs to match the dashboard URL as well, not just the trigger URL.
+          # The inline_metrics_redactor_filter.rb uses this regex to match against
+          # the dashboard URL.
           #
           # EX - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB)
+          # dashboard URL - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/metrics_dashboard?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB)
           def clusters_regex
             strong_memoize(:clusters_regex) do
               regex_for_project_metrics(
                 %r{
+                  #{DASH_PATTERN}?
                   /clusters
                   /(?<cluster_id>\d+)
                   /?
+                  ( (/metrics) | ( /metrics_dashboard\.json ) )?
                 }x
               )
             end
@@ -67,10 +77,11 @@ module Gitlab
             strong_memoize(:alert_regex) do
               regex_for_project_metrics(
                 %r{
+                  #{DASH_PATTERN}?
                   /prometheus
                   /alerts
                   /(?<alert>\d+)
-                  /metrics_dashboard
+                  /metrics_dashboard(\.json)?
                 }x
               )
             end
@@ -95,16 +106,37 @@ module Gitlab
 
           private
 
+          def environment_metrics_regex
+            %r{
+              #{DASH_PATTERN}?
+              /environments
+              /(?<environment>\d+)
+              /(metrics_dashboard|metrics)
+            }x
+          end
+
+          def non_environment_metrics_regex
+            %r{
+              #{DASH_PATTERN}
+              /metrics
+              (?=                             # Lookahead to ensure there is an environment query param
+                \?
+                .*
+                environment=(?<environment>\d+)
+                .*
+              )
+            }x
+          end
+
           def regex_for_project_metrics(path_suffix_pattern)
             %r{
-              (?<url>
+              ^(?<url>
                 #{gitlab_host_pattern}
                 #{project_path_pattern}
-                #{OPTIONAL_DASH_PATTERN}
                 #{path_suffix_pattern}
                 #{QUERY_PATTERN}
                 #{ANCHOR_PATTERN}
-              )
+              )$
             }x
           end
 
diff --git a/lib/gitlab/metrics/dashboard/validator.rb b/lib/gitlab/metrics/dashboard/validator.rb
index 8edd9c397f9..1e8dc059968 100644
--- a/lib/gitlab/metrics/dashboard/validator.rb
+++ b/lib/gitlab/metrics/dashboard/validator.rb
@@ -4,24 +4,22 @@ module Gitlab
   module Metrics
     module Dashboard
       module Validator
-        DASHBOARD_SCHEMA_PATH = 'lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json'.freeze
+        DASHBOARD_SCHEMA_PATH = Rails.root.join(*%w[lib gitlab metrics dashboard validator schemas dashboard.json]).freeze
 
         class << self
           def validate(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil)
-            errors = _validate(content, schema_path, dashboard_path: dashboard_path, project: project)
-            errors.empty?
+            errors(content, schema_path, dashboard_path: dashboard_path, project: project).empty?
           end
 
           def validate!(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil)
-            errors = _validate(content, schema_path, dashboard_path: dashboard_path, project: project)
+            errors = errors(content, schema_path, dashboard_path: dashboard_path, project: project)
             errors.empty? || raise(errors.first)
           end
 
-          private
-
-          def _validate(content, schema_path, dashboard_path: nil, project: nil)
-            client = Validator::Client.new(content, schema_path, dashboard_path: dashboard_path, project: project)
-            client.execute
+          def errors(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil)
+            Validator::Client
+              .new(content, schema_path, dashboard_path: dashboard_path, project: project)
+              .execute
           end
         end
       end
diff --git a/lib/gitlab/metrics/dashboard/validator/client.rb b/lib/gitlab/metrics/dashboard/validator/client.rb
index c63415abcfc..588c677ca28 100644
--- a/lib/gitlab/metrics/dashboard/validator/client.rb
+++ b/lib/gitlab/metrics/dashboard/validator/client.rb
@@ -46,7 +46,7 @@ module Gitlab
 
           def validate_against_schema
             schemer.validate(content).map do |error|
-              Errors::SchemaValidationError.new(error)
+              ::Gitlab::Metrics::Dashboard::Validator::Errors::SchemaValidationError.new(error)
             end
           end
         end
diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/panel.json b/lib/gitlab/metrics/dashboard/validator/schemas/panel.json
index 011eef53e40..2ae9608036e 100644
--- a/lib/gitlab/metrics/dashboard/validator/schemas/panel.json
+++ b/lib/gitlab/metrics/dashboard/validator/schemas/panel.json
@@ -4,7 +4,7 @@
   "properties": {
     "type": {
       "type": "string",
-      "enum": ["area-chart", "anomaly-chart", "bar", "column", "stacked-column", "single-stat", "heatmap"],
+      "enum": ["area-chart", "line-chart", "anomaly-chart", "bar", "column", "stacked-column", "single-stat", "heatmap", "gauge"],
       "default": "area-chart"
     },
     "title": { "type": "string" },
diff --git a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb
index 054b4949dd6..36262b09b2d 100644
--- a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb
+++ b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb
@@ -12,7 +12,11 @@ module Gitlab
         end
 
         def log_filename
-          File.join(Rails.root, 'log', 'sidekiq_exporter.log')
+          if settings['log_enabled']
+            File.join(Rails.root, 'log', 'sidekiq_exporter.log')
+          else
+            File::NULL
+          end
         end
 
         private
diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index ff3fffe7b95..66361529546 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -120,9 +120,6 @@ module Gitlab
       def self.instrument(type, mod, name)
         return unless ::Gitlab::Metrics.enabled?
 
-        name = name.to_sym
-        target = type == :instance ? mod : mod.singleton_class
-
         if type == :instance
           target = mod
           method_name = "##{name}"
@@ -154,6 +151,8 @@ module Gitlab
             '*args'
           end
 
+        method_visibility = method_visibility_for(target, name)
+
         proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1
           def #{name}(#{args_signature})
             if trans = Gitlab::Metrics::Instrumentation.transaction
@@ -163,11 +162,23 @@ module Gitlab
               super
             end
           end
+          #{method_visibility} :#{name}
         EOF
 
         target.prepend(proxy_module)
       end
 
+      def self.method_visibility_for(mod, name)
+        if mod.private_method_defined?(name)
+          :private
+        elsif mod.protected_method_defined?(name)
+          :protected
+        else
+          :public
+        end
+      end
+      private_class_method :method_visibility_for
+
       # Small layer of indirection to make it easier to stub out the current
       # transaction.
       def self.transaction
diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb
index 2b5d1c710f6..3100450bc00 100644
--- a/lib/gitlab/metrics/methods.rb
+++ b/lib/gitlab/metrics/methods.rb
@@ -52,7 +52,7 @@ module Gitlab
         end
 
         def disabled_by_feature(options)
-          options.with_feature && !::Feature.enabled?(options.with_feature)
+          options.with_feature && !::Feature.enabled?(options.with_feature, type: :ops)
         end
 
         def build_metric!(type, name, options)
diff --git a/lib/gitlab/metrics/samplers/action_cable_sampler.rb b/lib/gitlab/metrics/samplers/action_cable_sampler.rb
new file mode 100644
index 00000000000..9f4979fa673
--- /dev/null
+++ b/lib/gitlab/metrics/samplers/action_cable_sampler.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Samplers
+      class ActionCableSampler < BaseSampler
+        SAMPLING_INTERVAL_SECONDS = 5
+
+        def initialize(interval = SAMPLING_INTERVAL_SECONDS, action_cable: ::ActionCable.server)
+          super(interval)
+          @action_cable = action_cable
+        end
+
+        def metrics
+          @metrics ||= {
+            active_connections: ::Gitlab::Metrics.gauge(
+              :action_cable_active_connections, 'Number of ActionCable WS clients currently connected'
+            ),
+            pool_min_size: ::Gitlab::Metrics.gauge(
+              :action_cable_pool_min_size, 'Minimum number of worker threads in ActionCable thread pool'
+            ),
+            pool_max_size: ::Gitlab::Metrics.gauge(
+              :action_cable_pool_max_size, 'Maximum number of worker threads in ActionCable thread pool'
+            ),
+            pool_current_size: ::Gitlab::Metrics.gauge(
+              :action_cable_pool_current_size, 'Current number of worker threads in ActionCable thread pool'
+            ),
+            pool_largest_size: ::Gitlab::Metrics.gauge(
+              :action_cable_pool_largest_size, 'Largest number of worker threads observed so far in ActionCable thread pool'
+            ),
+            pool_completed_tasks: ::Gitlab::Metrics.gauge(
+              :action_cable_pool_tasks_total, 'Total number of tasks executed in ActionCable thread pool'
+            ),
+            pool_pending_tasks: ::Gitlab::Metrics.gauge(
+              :action_cable_pool_pending_tasks, 'Number of tasks waiting to be executed in ActionCable thread pool'
+            )
+          }
+        end
+
+        def sample
+          pool = @action_cable.worker_pool.executor
+          labels = {
+            server_mode: server_mode
+          }
+
+          metrics[:active_connections].set(labels, @action_cable.connections.size)
+          metrics[:pool_min_size].set(labels, pool.min_length)
+          metrics[:pool_max_size].set(labels, pool.max_length)
+          metrics[:pool_current_size].set(labels, pool.length)
+          metrics[:pool_largest_size].set(labels, pool.largest_length)
+          metrics[:pool_completed_tasks].set(labels, pool.completed_task_count)
+          metrics[:pool_pending_tasks].set(labels, pool.queue_length)
+        end
+
+        private
+
+        def server_mode
+          Gitlab::ActionCable::Config.in_app? ? 'in-app' : 'standalone'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb
index ff3e7be567f..39a49187e45 100644
--- a/lib/gitlab/metrics/samplers/base_sampler.rb
+++ b/lib/gitlab/metrics/samplers/base_sampler.rb
@@ -21,7 +21,7 @@ module Gitlab
         def safe_sample
           sample
         rescue => e
-          Rails.logger.warn("#{self.class}: #{e}, stopping") # rubocop:disable Gitlab/RailsLogger
+          Gitlab::AppLogger.warn("#{self.class}: #{e}, stopping")
           stop
         end
 
diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb
index b5343d5e66a..d295beb59f1 100644
--- a/lib/gitlab/metrics/samplers/puma_sampler.rb
+++ b/lib/gitlab/metrics/samplers/puma_sampler.rb
@@ -42,7 +42,7 @@ module Gitlab
         def puma_stats
           Puma.stats
         rescue NoMethodError
-          Rails.logger.info "PumaSampler: stats are not available yet, waiting for Puma to boot" # rubocop:disable Gitlab/RailsLogger
+          Gitlab::AppLogger.info "PumaSampler: stats are not available yet, waiting for Puma to boot"
           nil
         end