Merge branch '60383-setup-dashboard-endpoint' into 'master'

Create dashboards endpoint & setup dashboard post-processing

Closes #60383

See merge request gitlab-org/gitlab-ce!27405

6 files changed, 302 insertions, 0 deletions

diff --git a/lib/gitlab/metrics/dashboard/processor.rb b/lib/gitlab/metrics/dashboard/processor.rb
new file mode 100644
index 00000000000..cc34ac53051
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/processor.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      # Responsible for processesing a dashboard hash, inserting
+      # relevant DB records & sorting for proper rendering in
+      # the UI. These includes shared metric info, custom metrics
+      # info, and alerts (only in EE).
+      class Processor
+        SEQUENCE = [
+          Stages::CommonMetricsInserter,
+          Stages::ProjectMetricsInserter,
+          Stages::Sorter
+        ].freeze
+
+        def initialize(project, environment, dashboard)
+          @project = project
+          @environment = environment
+          @dashboard = dashboard
+        end
+
+        # Returns a new dashboard hash with the results of
+        # running transforms on the dashboard.
+        def process
+          @dashboard.deep_symbolize_keys.tap do |dashboard|
+            sequence.each do |stage|
+              stage.new(@project, @environment, dashboard).transform!
+            end
+          end
+        end
+
+        private
+
+        def sequence
+          SEQUENCE
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/service.rb b/lib/gitlab/metrics/dashboard/service.rb
new file mode 100644
index 00000000000..79d563cce4f
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/service.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# Fetches the metrics dashboard layout and supplemented the output with DB info.
+module Gitlab
+  module Metrics
+    module Dashboard
+      class Service < ::BaseService
+        SYSTEM_DASHBOARD_NAME = 'common_metrics'
+        SYSTEM_DASHBOARD_PATH = Rails.root.join('config', 'prometheus', "#{SYSTEM_DASHBOARD_NAME}.yml")
+
+        # Returns a DB-supplemented json representation of a dashboard config file.
+        def get_dashboard
+          dashboard_string = Rails.cache.fetch(cache_key) { system_dashboard }
+
+          dashboard = process_dashboard(dashboard_string)
+
+          success(dashboard: dashboard)
+        rescue Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardLayoutError => e
+          error(e.message, :unprocessable_entity)
+        end
+
+        private
+
+        # Returns the base metrics shipped with every GitLab service.
+        def system_dashboard
+          YAML.safe_load(File.read(SYSTEM_DASHBOARD_PATH))
+        end
+
+        def cache_key
+          "metrics_dashboard_#{SYSTEM_DASHBOARD_NAME}"
+        end
+
+        # Returns a new dashboard Hash, supplemented with DB info
+        def process_dashboard(dashboard)
+          Gitlab::Metrics::Dashboard::Processor.new(project, params[:environment], dashboard).process
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb
new file mode 100644
index 00000000000..dd4aae6c115
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/stages/base_stage.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Stages
+        class BaseStage
+          DashboardLayoutError = Class.new(StandardError)
+
+          DEFAULT_PANEL_TYPE = 'area-chart'
+
+          attr_reader :project, :environment, :dashboard
+
+          def initialize(project, environment, dashboard)
+            @project = project
+            @environment = environment
+            @dashboard = dashboard
+          end
+
+          # Entry-point to the stage
+          def transform!
+            raise NotImplementedError
+          end
+
+          protected
+
+          def missing_panel_groups!
+            raise DashboardLayoutError.new('Top-level key :panel_groups must be an array')
+          end
+
+          def missing_panels!
+            raise DashboardLayoutError.new('Each "panel_group" must define an array :panels')
+          end
+
+          def missing_metrics!
+            raise DashboardLayoutError.new('Each "panel" must define an array :metrics')
+          end
+
+          def for_metrics(dashboard)
+            missing_panel_groups! unless dashboard[:panel_groups].is_a?(Array)
+
+            dashboard[:panel_groups].each do |panel_group|
+              missing_panels! unless panel_group[:panels].is_a?(Array)
+
+              panel_group[:panels].each do |panel|
+                missing_metrics! unless panel[:metrics].is_a?(Array)
+
+                panel[:metrics].each do |metric|
+                  yield metric
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb
new file mode 100644
index 00000000000..3406021bbea
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Stages
+        class CommonMetricsInserter < BaseStage
+          # For each metric in the dashboard config, attempts to
+          # find a corresponding database record. If found,
+          # includes the record's id in the dashboard config.
+          def transform!
+            common_metrics = ::PrometheusMetric.common
+
+            for_metrics(dashboard) do |metric|
+              metric_record = common_metrics.find { |m| m.identifier == metric[:id] }
+              metric[:metric_id] = metric_record.id if metric_record
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb
new file mode 100644
index 00000000000..221610a14d1
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb
@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Stages
+        class ProjectMetricsInserter < BaseStage
+          # Inserts project-specific metrics into the dashboard
+          # config. If there are no project-specific metrics,
+          # this will have no effect.
+          def transform!
+            project.prometheus_metrics.each do |project_metric|
+              group = find_or_create_panel_group(dashboard[:panel_groups], project_metric)
+              panel = find_or_create_panel(group[:panels], project_metric)
+              find_or_create_metric(panel[:metrics], project_metric)
+            end
+          end
+
+          private
+
+          # Looks for a panel_group corresponding to the
+          # provided metric object. If unavailable, inserts one.
+          # @param panel_groups [Array<Hash>]
+          # @param metric [PrometheusMetric]
+          def find_or_create_panel_group(panel_groups, metric)
+            panel_group = find_panel_group(panel_groups, metric)
+            return panel_group if panel_group
+
+            panel_group = new_panel_group(metric)
+            panel_groups << panel_group
+
+            panel_group
+          end
+
+          # Looks for a panel corresponding to the provided
+          # metric object. If unavailable, inserts one.
+          # @param panels [Array<Hash>]
+          # @param metric [PrometheusMetric]
+          def find_or_create_panel(panels, metric)
+            panel = find_panel(panels, metric)
+            return panel if panel
+
+            panel = new_panel(metric)
+            panels << panel
+
+            panel
+          end
+
+          # Looks for a metric corresponding to the provided
+          # metric object. If unavailable, inserts one.
+          # @param metrics [Array<Hash>]
+          # @param metric [PrometheusMetric]
+          def find_or_create_metric(metrics, metric)
+            target_metric = find_metric(metrics, metric)
+            return target_metric if target_metric
+
+            target_metric = new_metric(metric)
+            metrics << target_metric
+
+            target_metric
+          end
+
+          def find_panel_group(panel_groups, metric)
+            return unless panel_groups
+
+            panel_groups.find { |group| group[:group] == metric.group_title }
+          end
+
+          def find_panel(panels, metric)
+            return unless panels
+
+            panel_identifiers = [DEFAULT_PANEL_TYPE, metric.title, metric.y_label]
+            panels.find { |panel| panel.values_at(:type, :title, :y_label) == panel_identifiers }
+          end
+
+          def find_metric(metrics, metric)
+            return unless metrics
+
+            metrics.find { |m| m[:id] == metric.identifier }
+          end
+
+          def new_panel_group(metric)
+            {
+              group: metric.group_title,
+              priority: metric.priority,
+              panels: []
+            }
+          end
+
+          def new_panel(metric)
+            {
+              type: DEFAULT_PANEL_TYPE,
+              title: metric.title,
+              y_label: metric.y_label,
+              metrics: []
+            }
+          end
+
+          def new_metric(metric)
+            metric.queries.first.merge(metric_id: metric.id)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/stages/sorter.rb b/lib/gitlab/metrics/dashboard/stages/sorter.rb
new file mode 100644
index 00000000000..ba5aa78059c
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/stages/sorter.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Stages
+        class Sorter < BaseStage
+          def transform!
+            missing_panel_groups! unless dashboard[:panel_groups].is_a? Array
+
+            sort_groups!
+            sort_panels!
+          end
+
+          private
+
+          # Sorts the groups in the dashboard by the :priority key
+          def sort_groups!
+            dashboard[:panel_groups] = dashboard[:panel_groups].sort_by { |group| -group[:priority].to_i }
+          end
+
+          # Sorts the panels in the dashboard by the :weight key
+          def sort_panels!
+            dashboard[:panel_groups].each do |group|
+              missing_panels! unless group[:panels].is_a? Array
+
+              group[:panels] = group[:panels].sort_by { |panel| -panel[:weight].to_i }
+            end
+          end
+        end
+      end
+    end
+  end
+end