From bf918b68f643266e91a9308cbc64a8304c647f17 Mon Sep 17 00:00:00 2001 From: Sarah Yasonik Date: Wed, 7 Aug 2019 16:17:35 +0000 Subject: Support dashboard params for metrics dashboard https://gitlab.com/gitlab-org/gitlab-ce/issues/62971 Adds support to EnvironmentsController#metrics_dashboard for the following params: group, title, y_label These params are used to uniquely identify a panel on the metrics dashboard. Metrics are stored in several places, so this adds utilities to find a specific panel from the database or filesystem depending on the metric specified. Also moves some shared utilities into separate classes, notably default values and errors. --- .../projects/environments_controller.rb | 7 +- app/models/prometheus_metric.rb | 4 + app/models/prometheus_metric_enums.rb | 20 +++- .../metrics/dashboard/base_embed_service.rb | 36 ++++++ app/services/metrics/dashboard/base_service.rb | 9 +- .../dashboard/custom_metric_embed_service.rb | 123 +++++++++++++++++++++ .../metrics/dashboard/default_embed_service.rb | 16 +-- .../metrics/dashboard/dynamic_embed_service.rb | 78 +++++++++++++ 8 files changed, 272 insertions(+), 21 deletions(-) create mode 100644 app/services/metrics/dashboard/base_embed_service.rb create mode 100644 app/services/metrics/dashboard/custom_metric_embed_service.rb create mode 100644 app/services/metrics/dashboard/dynamic_embed_service.rb (limited to 'app') diff --git a/app/controllers/projects/environments_controller.rb b/app/controllers/projects/environments_controller.rb index ccd54b369fa..07eb689d031 100644 --- a/app/controllers/projects/environments_controller.rb +++ b/app/controllers/projects/environments_controller.rb @@ -165,7 +165,8 @@ class Projects::EnvironmentsController < Projects::ApplicationController project, current_user, environment, - embedded: params[:embedded] + dashboard_path: params[:dashboard], + **dashboard_params.to_h.symbolize_keys ) elsif Feature.enabled?(:environment_metrics_show_multiple_dashboards, project) result = dashboard_finder.find( @@ -233,6 +234,10 @@ class Projects::EnvironmentsController < Projects::ApplicationController params.require([:start, :end]) end + def dashboard_params + params.permit(:embedded, :group, :title, :y_label) + end + def dashboard_finder Gitlab::Metrics::Dashboard::Finder end diff --git a/app/models/prometheus_metric.rb b/app/models/prometheus_metric.rb index b8e7673dcf5..c7786500c5c 100644 --- a/app/models/prometheus_metric.rb +++ b/app/models/prometheus_metric.rb @@ -32,6 +32,10 @@ class PrometheusMetric < ApplicationRecord Gitlab::Prometheus::Metric.new(id: id, title: title, required_metrics: required_metrics, weight: 0, y_label: y_label, queries: queries) end + def to_metric_hash + queries.first.merge(metric_id: id) + end + def queries [ { diff --git a/app/models/prometheus_metric_enums.rb b/app/models/prometheus_metric_enums.rb index 6cb22cc69cd..d58f825f222 100644 --- a/app/models/prometheus_metric_enums.rb +++ b/app/models/prometheus_metric_enums.rb @@ -9,13 +9,17 @@ module PrometheusMetricEnums aws_elb: -3, nginx: -4, kubernetes: -5, - nginx_ingress: -6, + nginx_ingress: -6 + }.merge(custom_groups).freeze + end - # custom/user groups + # custom/user groups + def self.custom_groups + { business: 0, response: 1, system: 2 - } + }.freeze end def self.group_details @@ -50,16 +54,20 @@ module PrometheusMetricEnums group_title: _('System metrics (Kubernetes)'), required_metrics: %w(container_memory_usage_bytes container_cpu_usage_seconds_total), priority: 5 - }.freeze, + }.freeze + }.merge(custom_group_details).freeze + end - # custom/user groups + # custom/user groups + def self.custom_group_details + { business: { group_title: _('Business metrics (Custom)'), priority: 0 }.freeze, response: { group_title: _('Response metrics (Custom)'), - priority: -5 + priority: -5 }.freeze, system: { group_title: _('System metrics (Custom)'), diff --git a/app/services/metrics/dashboard/base_embed_service.rb b/app/services/metrics/dashboard/base_embed_service.rb new file mode 100644 index 00000000000..8bb5f4892cb --- /dev/null +++ b/app/services/metrics/dashboard/base_embed_service.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +# Base class for embed services. Contains a few basic helper +# methods that the embed services share. +module Metrics + module Dashboard + class BaseEmbedService < ::Metrics::Dashboard::BaseService + def cache_key + "dynamic_metrics_dashboard_#{identifiers}" + end + + protected + + def dashboard_path + params[:dashboard_path].presence || + ::Metrics::Dashboard::SystemDashboardService::SYSTEM_DASHBOARD_PATH + end + + def group + params[:group] + end + + def title + params[:title] + end + + def y_label + params[:y_label] + end + + def identifiers + [dashboard_path, group, title, y_label].join('|') + end + end + end +end diff --git a/app/services/metrics/dashboard/base_service.rb b/app/services/metrics/dashboard/base_service.rb index b331bf51874..8a42675c66d 100644 --- a/app/services/metrics/dashboard/base_service.rb +++ b/app/services/metrics/dashboard/base_service.rb @@ -5,17 +5,14 @@ module Metrics module Dashboard class BaseService < ::BaseService - PROCESSING_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardProcessingError - NOT_FOUND_ERROR = Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError + include Gitlab::Metrics::Dashboard::Errors def get_dashboard return error('Insufficient permissions.', :unauthorized) unless allowed? success(dashboard: process_dashboard) - rescue NOT_FOUND_ERROR - error("#{dashboard_path} could not be found.", :not_found) - rescue PROCESSING_ERROR => e - error(e.message, :unprocessable_entity) + rescue StandardError => e + handle_errors(e) end # Summary of all known dashboards for the service. diff --git a/app/services/metrics/dashboard/custom_metric_embed_service.rb b/app/services/metrics/dashboard/custom_metric_embed_service.rb new file mode 100644 index 00000000000..50f070989fc --- /dev/null +++ b/app/services/metrics/dashboard/custom_metric_embed_service.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +# Responsible for returning a dashboard containing specified +# custom metrics. Creates panels based on the matching metrics +# stored in the database. +# +# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. +module Metrics + module Dashboard + class CustomMetricEmbedService < ::Metrics::Dashboard::BaseEmbedService + extend ::Gitlab::Utils::Override + include Gitlab::Utils::StrongMemoize + include Gitlab::Metrics::Dashboard::Defaults + + class << self + # Determines whether the provided params are sufficient + # to uniquely identify a panel composed of user-defined + # custom metrics from the DB. + def valid_params?(params) + [ + params[:embedded], + valid_dashboard?(params[:dashboard_path]), + valid_group_title?(params[:group]), + params[:title].present?, + params.has_key?(:y_label) + ].all? + end + + private + + # A group title is valid if it is one of the limited + # options the user can select in the UI. + def valid_group_title?(group) + PrometheusMetricEnums + .custom_group_details + .map { |_, details| details[:group_title] } + .include?(group) + end + + # All custom metrics are displayed on the system dashboard. + # Nil is acceptable as we'll default to the system dashboard. + def valid_dashboard?(dashboard) + dashboard.nil? || SystemDashboardService.system_dashboard?(dashboard) + end + end + + # Returns a new dashboard with only the matching + # metrics from the system dashboard, stripped of + # group info. + # + # Note: This overrides the method #raw_dashboard, + # which means the result will not be cached. This + # is because we are inserting DB info into the + # dashboard before post-processing. This ensures + # we aren't acting on deleted or out-of-date metrics. + # + # @return [Hash] + override :raw_dashboard + def raw_dashboard + panels_not_found!(identifiers) if panels.empty? + + { 'panel_groups' => [{ 'panels' => panels }] } + end + + private + + # Generated dashboard panels for each metric which + # matches the provided input. + # @return [Array] + def panels + strong_memoize(:panels) do + metrics.map { |metric| panel_for_metric(metric) } + end + end + + # Metrics which match the provided inputs. + # There may be multiple metrics, but they should be + # displayed in a single panel/chart. + # @return [ActiveRecord::AssociationRelation] + # rubocop: disable CodeReuse/ActiveRecord + def metrics + project.prometheus_metrics.where( + group: group_key, + title: title, + y_label: y_label + ) + end + # rubocop: enable CodeReuse/ActiveRecord + + # Returns a symbol representing the group that + # the dashboard's group title belongs to. + # It will be one of the keys found under + # PrometheusMetricEnums.custom_groups. + # + # @return [String] + def group_key + strong_memoize(:group_key) do + PrometheusMetricEnums + .group_details + .find { |_, details| details[:group_title] == group } + .first + .to_s + end + end + + # Returns a representation of a PromtheusMetric + # as a dashboard panel. As the panel is generated + # on the fly, we're using default values for info + # not represented in the DB. + # + # @return [Hash] + def panel_for_metric(metric) + { + type: DEFAULT_PANEL_TYPE, + weight: DEFAULT_PANEL_WEIGHT, + title: metric.title, + y_label: metric.y_label, + metrics: [metric.to_metric_hash] + } + end + end + end +end diff --git a/app/services/metrics/dashboard/default_embed_service.rb b/app/services/metrics/dashboard/default_embed_service.rb index 8b01b44fc98..e1bd98bd5c2 100644 --- a/app/services/metrics/dashboard/default_embed_service.rb +++ b/app/services/metrics/dashboard/default_embed_service.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true # Responsible for returning a filtered system dashboard -# containing only the default embedded metrics. In future, -# this class may be updated to support filtering to -# alternate metrics/panels. +# containing only the default embedded metrics. This class +# operates by selecting metrics directly from the system +# dashboard. # # Why isn't this filtering in a processing stage? By filtering # here, we ensure the dynamically-determined dashboard is cached. @@ -11,7 +11,7 @@ # Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. module Metrics module Dashboard - class DefaultEmbedService < ::Metrics::Dashboard::BaseService + class DefaultEmbedService < ::Metrics::Dashboard::BaseEmbedService # For the default filtering for embedded metrics, # uses the 'id' key in dashboard-yml definition for # identification. @@ -33,10 +33,6 @@ module Metrics { 'panel_groups' => [{ 'panels' => panels }] } end - def cache_key - "dynamic_metrics_dashboard_#{metric_identifiers.join('_')}" - end - private # Returns an array of the panels groups on the @@ -58,6 +54,10 @@ module Metrics def metric_identifiers DEFAULT_EMBEDDED_METRICS_IDENTIFIERS end + + def identifiers + metric_identifiers.join('|') + end end end end diff --git a/app/services/metrics/dashboard/dynamic_embed_service.rb b/app/services/metrics/dashboard/dynamic_embed_service.rb new file mode 100644 index 00000000000..db5b7c9e32a --- /dev/null +++ b/app/services/metrics/dashboard/dynamic_embed_service.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +# Responsible for returning a filtered project dashboard +# containing only the request-provided metrics. The result +# is then cached for future requests. Metrics are identified +# based on a combination of identifiers for now, but the ideal +# would be similar to the approach in DefaultEmbedService, but +# a single unique identifier is not currently available across +# all metric types (custom, project-defined, cluster, or system). +# +# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. +module Metrics + module Dashboard + class DynamicEmbedService < ::Metrics::Dashboard::BaseEmbedService + include Gitlab::Utils::StrongMemoize + + class << self + # Determines whether the provided params are sufficient + # to uniquely identify a panel from a yml-defined dashboard. + # + # See https://docs.gitlab.com/ee/user/project/integrations/prometheus.html#defining-custom-dashboards-per-project + # for additional info on defining custom dashboards. + def valid_params?(params) + [ + params[:embedded], + params[:group].present?, + params[:title].present?, + params[:y_label] + ].all? + end + end + + # Returns a new dashboard with only the matching + # metrics from the system dashboard, stripped of groups. + # @return [Hash] + def get_raw_dashboard + not_found! if panels.empty? + + { 'panel_groups' => [{ 'panels' => panels }] } + end + + private + + def panels + strong_memoize(:panels) do + not_found! unless base_dashboard + not_found! unless groups = base_dashboard['panel_groups'] + not_found! unless matching_group = find_group(groups) + not_found! unless all_panels = matching_group['panels'] + + find_panels(all_panels) + end + end + + def base_dashboard + strong_memoize(:base_dashboard) do + Gitlab::Metrics::Dashboard::Finder.find_raw(project, dashboard_path: dashboard_path) + end + end + + def find_group(groups) + groups.find do |candidate_group| + candidate_group['group'] == group + end + end + + def find_panels(all_panels) + all_panels.select do |panel| + panel['title'] == title && panel['y_label'] == y_label + end + end + + def not_found! + panels_not_found!(identifiers) + end + end + end +end -- cgit v1.2.1