diff options
author | Grzegorz Bizon <grzegorz@gitlab.com> | 2019-04-28 11:53:12 +0000 |
---|---|---|
committer | Grzegorz Bizon <grzegorz@gitlab.com> | 2019-04-28 11:53:12 +0000 |
commit | 536d96a3e02daceca8398745cd27ab1e0f56711d (patch) | |
tree | 70c395ae775a7f56352aa88aaec9bd5810a555f3 | |
parent | ddee4426c4bd3df8bde7936a6ffb0a1973a4d5a1 (diff) | |
parent | 0e093940e1135897b996a5f16239eca62cc6089e (diff) | |
download | gitlab-ce-536d96a3e02daceca8398745cd27ab1e0f56711d.tar.gz |
Merge branch '60383-setup-dashboard-endpoint' into 'master'
Create dashboards endpoint & setup dashboard post-processing
Closes #60383
See merge request gitlab-org/gitlab-ce!27405
16 files changed, 598 insertions, 1 deletions
diff --git a/app/controllers/projects/environments_controller.rb b/app/controllers/projects/environments_controller.rb index e35f34be23c..1f619c8fd2f 100644 --- a/app/controllers/projects/environments_controller.rb +++ b/app/controllers/projects/environments_controller.rb @@ -10,8 +10,9 @@ class Projects::EnvironmentsController < Projects::ApplicationController before_action :environment, only: [:show, :edit, :update, :stop, :terminal, :terminal_websocket_authorize, :metrics] before_action :verify_api_request!, only: :terminal_websocket_authorize before_action :expire_etag_cache, only: [:index] - before_action only: [:metrics, :additional_metrics] do + before_action only: [:metrics, :additional_metrics, :metrics_dashboard] do push_frontend_feature_flag(:metrics_time_window) + push_frontend_feature_flag(:environment_metrics_use_prometheus_endpoint) end def index @@ -156,6 +157,20 @@ class Projects::EnvironmentsController < Projects::ApplicationController end end + def metrics_dashboard + return render_403 unless Feature.enabled?(:environment_metrics_use_prometheus_endpoint, @project) + + result = Gitlab::Metrics::Dashboard::Service.new(@project, @current_user, environment: environment).get_dashboard + + respond_to do |format| + if result[:status] == :success + format.json { render status: :ok, json: result } + else + format.json { render status: result[:http_status], json: result } + end + end + end + def search respond_to do |format| format.json do diff --git a/config/routes/project.rb b/config/routes/project.rb index 93d168fc595..f7841bbe595 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -218,6 +218,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do get :terminal get :metrics get :additional_metrics + get :metrics_dashboard get '/terminal.ws/authorize', to: 'environments#terminal_websocket_authorize', constraints: { format: nil } get '/prometheus/api/v1/*proxy_path', to: 'environments/prometheus_api#proxy' diff --git a/lib/gitlab/metrics/dashboard/processor.rb b/lib/gitlab/metrics/dashboard/processor.rb new file mode 100644 index 00000000000..cc34ac53051 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/processor.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + # Responsible for processesing a dashboard hash, inserting + # relevant DB records & sorting for proper rendering in + # the UI. These includes shared metric info, custom metrics + # info, and alerts (only in EE). + class Processor + SEQUENCE = [ + Stages::CommonMetricsInserter, + Stages::ProjectMetricsInserter, + Stages::Sorter + ].freeze + + def initialize(project, environment, dashboard) + @project = project + @environment = environment + @dashboard = dashboard + end + + # Returns a new dashboard hash with the results of + # running transforms on the dashboard. + def process + @dashboard.deep_symbolize_keys.tap do |dashboard| + sequence.each do |stage| + stage.new(@project, @environment, dashboard).transform! + end + end + end + + private + + def sequence + SEQUENCE + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/service.rb b/lib/gitlab/metrics/dashboard/service.rb new file mode 100644 index 00000000000..79d563cce4f --- /dev/null +++ b/lib/gitlab/metrics/dashboard/service.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# Fetches the metrics dashboard layout and supplemented the output with DB info. +module Gitlab + module Metrics + module Dashboard + class Service < ::BaseService + SYSTEM_DASHBOARD_NAME = 'common_metrics' + SYSTEM_DASHBOARD_PATH = Rails.root.join('config', 'prometheus', "#{SYSTEM_DASHBOARD_NAME}.yml") + + # Returns a DB-supplemented json representation of a dashboard config file. + def get_dashboard + dashboard_string = Rails.cache.fetch(cache_key) { system_dashboard } + + dashboard = process_dashboard(dashboard_string) + + success(dashboard: dashboard) + rescue Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardLayoutError => e + error(e.message, :unprocessable_entity) + end + + private + + # Returns the base metrics shipped with every GitLab service. + def system_dashboard + YAML.safe_load(File.read(SYSTEM_DASHBOARD_PATH)) + end + + def cache_key + "metrics_dashboard_#{SYSTEM_DASHBOARD_NAME}" + end + + # Returns a new dashboard Hash, supplemented with DB info + def process_dashboard(dashboard) + Gitlab::Metrics::Dashboard::Processor.new(project, params[:environment], dashboard).process + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb new file mode 100644 index 00000000000..dd4aae6c115 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/base_stage.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class BaseStage + DashboardLayoutError = Class.new(StandardError) + + DEFAULT_PANEL_TYPE = 'area-chart' + + attr_reader :project, :environment, :dashboard + + def initialize(project, environment, dashboard) + @project = project + @environment = environment + @dashboard = dashboard + end + + # Entry-point to the stage + def transform! + raise NotImplementedError + end + + protected + + def missing_panel_groups! + raise DashboardLayoutError.new('Top-level key :panel_groups must be an array') + end + + def missing_panels! + raise DashboardLayoutError.new('Each "panel_group" must define an array :panels') + end + + def missing_metrics! + raise DashboardLayoutError.new('Each "panel" must define an array :metrics') + end + + def for_metrics(dashboard) + missing_panel_groups! unless dashboard[:panel_groups].is_a?(Array) + + dashboard[:panel_groups].each do |panel_group| + missing_panels! unless panel_group[:panels].is_a?(Array) + + panel_group[:panels].each do |panel| + missing_metrics! unless panel[:metrics].is_a?(Array) + + panel[:metrics].each do |metric| + yield metric + end + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb new file mode 100644 index 00000000000..3406021bbea --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class CommonMetricsInserter < BaseStage + # For each metric in the dashboard config, attempts to + # find a corresponding database record. If found, + # includes the record's id in the dashboard config. + def transform! + common_metrics = ::PrometheusMetric.common + + for_metrics(dashboard) do |metric| + metric_record = common_metrics.find { |m| m.identifier == metric[:id] } + metric[:metric_id] = metric_record.id if metric_record + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb new file mode 100644 index 00000000000..221610a14d1 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class ProjectMetricsInserter < BaseStage + # Inserts project-specific metrics into the dashboard + # config. If there are no project-specific metrics, + # this will have no effect. + def transform! + project.prometheus_metrics.each do |project_metric| + group = find_or_create_panel_group(dashboard[:panel_groups], project_metric) + panel = find_or_create_panel(group[:panels], project_metric) + find_or_create_metric(panel[:metrics], project_metric) + end + end + + private + + # Looks for a panel_group corresponding to the + # provided metric object. If unavailable, inserts one. + # @param panel_groups [Array<Hash>] + # @param metric [PrometheusMetric] + def find_or_create_panel_group(panel_groups, metric) + panel_group = find_panel_group(panel_groups, metric) + return panel_group if panel_group + + panel_group = new_panel_group(metric) + panel_groups << panel_group + + panel_group + end + + # Looks for a panel corresponding to the provided + # metric object. If unavailable, inserts one. + # @param panels [Array<Hash>] + # @param metric [PrometheusMetric] + def find_or_create_panel(panels, metric) + panel = find_panel(panels, metric) + return panel if panel + + panel = new_panel(metric) + panels << panel + + panel + end + + # Looks for a metric corresponding to the provided + # metric object. If unavailable, inserts one. + # @param metrics [Array<Hash>] + # @param metric [PrometheusMetric] + def find_or_create_metric(metrics, metric) + target_metric = find_metric(metrics, metric) + return target_metric if target_metric + + target_metric = new_metric(metric) + metrics << target_metric + + target_metric + end + + def find_panel_group(panel_groups, metric) + return unless panel_groups + + panel_groups.find { |group| group[:group] == metric.group_title } + end + + def find_panel(panels, metric) + return unless panels + + panel_identifiers = [DEFAULT_PANEL_TYPE, metric.title, metric.y_label] + panels.find { |panel| panel.values_at(:type, :title, :y_label) == panel_identifiers } + end + + def find_metric(metrics, metric) + return unless metrics + + metrics.find { |m| m[:id] == metric.identifier } + end + + def new_panel_group(metric) + { + group: metric.group_title, + priority: metric.priority, + panels: [] + } + end + + def new_panel(metric) + { + type: DEFAULT_PANEL_TYPE, + title: metric.title, + y_label: metric.y_label, + metrics: [] + } + end + + def new_metric(metric) + metric.queries.first.merge(metric_id: metric.id) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/sorter.rb b/lib/gitlab/metrics/dashboard/stages/sorter.rb new file mode 100644 index 00000000000..ba5aa78059c --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/sorter.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class Sorter < BaseStage + def transform! + missing_panel_groups! unless dashboard[:panel_groups].is_a? Array + + sort_groups! + sort_panels! + end + + private + + # Sorts the groups in the dashboard by the :priority key + def sort_groups! + dashboard[:panel_groups] = dashboard[:panel_groups].sort_by { |group| -group[:priority].to_i } + end + + # Sorts the panels in the dashboard by the :weight key + def sort_panels! + dashboard[:panel_groups].each do |group| + missing_panels! unless group[:panels].is_a? Array + + group[:panels] = group[:panels].sort_by { |panel| -panel[:weight].to_i } + end + end + end + end + end + end +end diff --git a/spec/controllers/projects/environments_controller_spec.rb b/spec/controllers/projects/environments_controller_spec.rb index 75158f2e8e0..c1c4be45168 100644 --- a/spec/controllers/projects/environments_controller_spec.rb +++ b/spec/controllers/projects/environments_controller_spec.rb @@ -461,6 +461,43 @@ describe Projects::EnvironmentsController do end end + describe 'metrics_dashboard' do + context 'when prometheus endpoint is disabled' do + before do + stub_feature_flags(environment_metrics_use_prometheus_endpoint: false) + end + + it 'responds with status code 403' do + get :metrics_dashboard, params: environment_params(format: :json) + + expect(response).to have_gitlab_http_status(:forbidden) + end + end + + context 'when prometheus endpoint is enabled' do + it 'returns a json representation of the environment dashboard' do + get :metrics_dashboard, params: environment_params(format: :json) + + expect(response).to have_gitlab_http_status(:ok) + expect(json_response.keys).to contain_exactly('dashboard', 'status') + expect(json_response['dashboard']).to be_an_instance_of(Hash) + end + + context 'when the dashboard could not be provided' do + before do + allow(YAML).to receive(:safe_load).and_return({}) + end + + it 'returns an error response' do + get :metrics_dashboard, params: environment_params(format: :json) + + expect(response).to have_gitlab_http_status(:unprocessable_entity) + expect(json_response.keys).to contain_exactly('message', 'status', 'http_status') + end + end + end + end + describe 'GET #search' do before do create(:environment, name: 'staging', project: project) diff --git a/spec/fixtures/lib/gitlab/metrics/dashboard/sample_dashboard.yml b/spec/fixtures/lib/gitlab/metrics/dashboard/sample_dashboard.yml new file mode 100644 index 00000000000..c2d3d3d8aca --- /dev/null +++ b/spec/fixtures/lib/gitlab/metrics/dashboard/sample_dashboard.yml @@ -0,0 +1,36 @@ +dashboard: 'Test Dashboard' +priority: 1 +panel_groups: +- group: Group A + priority: 10 + panels: + - title: "Super Chart A1" + type: "area-chart" + y_label: "y_label" + weight: 1 + metrics: + - id: metric_a1 + query_range: 'query' + unit: unit + label: Legend Label + - title: "Super Chart A2" + type: "area-chart" + y_label: "y_label" + weight: 2 + metrics: + - id: metric_a2 + query_range: 'query' + label: Legend Label + unit: unit +- group: Group B + priority: 1 + panels: + - title: "Super Chart B" + type: "area-chart" + y_label: "y_label" + weight: 1 + metrics: + - id: metric_b + query_range: 'query' + unit: unit + label: Legend Label diff --git a/spec/fixtures/lib/gitlab/metrics/dashboard/schemas/dashboard.json b/spec/fixtures/lib/gitlab/metrics/dashboard/schemas/dashboard.json new file mode 100644 index 00000000000..1ee1205e29a --- /dev/null +++ b/spec/fixtures/lib/gitlab/metrics/dashboard/schemas/dashboard.json @@ -0,0 +1,13 @@ +{ + "type": "object", + "required": ["dashboard", "priority", "panel_groups"], + "properties": { + "dashboard": { "type": "string" }, + "priority": { "type": "number" }, + "panel_groups": { + "type": "array", + "items": { "$ref": "spec/fixtures/lib/gitlab/metrics/dashboard/schemas/panel_groups.json" } + } + }, + "additionalProperties": false +} diff --git a/spec/fixtures/lib/gitlab/metrics/dashboard/schemas/metrics.json b/spec/fixtures/lib/gitlab/metrics/dashboard/schemas/metrics.json new file mode 100644 index 00000000000..2d0af57ec2c --- /dev/null +++ b/spec/fixtures/lib/gitlab/metrics/dashboard/schemas/metrics.json @@ -0,0 +1,20 @@ +{ + "type": "object", + "required": [ + "unit", + "label" + ], + "oneOf": [ + { "required": ["query"] }, + { "required": ["query_range"] } + ], + "properties": { + "id": { "type": "string" }, + "query_range": { "type": "string" }, + "query": { "type": "string" }, + "unit": { "type": "string" }, + "label": { "type": "string" }, + "track": { "type": "string" } + }, + "additionalProperties": false +} diff --git a/spec/fixtures/lib/gitlab/metrics/dashboard/schemas/panel_groups.json b/spec/fixtures/lib/gitlab/metrics/dashboard/schemas/panel_groups.json new file mode 100644 index 00000000000..d7a390adcdc --- /dev/null +++ b/spec/fixtures/lib/gitlab/metrics/dashboard/schemas/panel_groups.json @@ -0,0 +1,17 @@ +{ + "type": "object", + "required": [ + "group", + "priority", + "panels" + ], + "properties": { + "group": { "type": "string" }, + "priority": { "type": "number" }, + "panels": { + "type": "array", + "items": { "$ref": "panels.json" } + } + }, + "additionalProperties": false +} diff --git a/spec/fixtures/lib/gitlab/metrics/dashboard/schemas/panels.json b/spec/fixtures/lib/gitlab/metrics/dashboard/schemas/panels.json new file mode 100644 index 00000000000..1548daacd64 --- /dev/null +++ b/spec/fixtures/lib/gitlab/metrics/dashboard/schemas/panels.json @@ -0,0 +1,20 @@ +{ + "type": "object", + "required": [ + "title", + "y_label", + "weight", + "metrics" + ], + "properties": { + "title": { "type": "string" }, + "type": { "type": "string" }, + "y_label": { "type": "string" }, + "weight": { "type": "number" }, + "metrics": { + "type": "array", + "items": { "$ref": "metrics.json" } + } + }, + "additionalProperties": false +} diff --git a/spec/lib/gitlab/metrics/dashboard/processor_spec.rb b/spec/lib/gitlab/metrics/dashboard/processor_spec.rb new file mode 100644 index 00000000000..ee7c93fce8d --- /dev/null +++ b/spec/lib/gitlab/metrics/dashboard/processor_spec.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Metrics::Dashboard::Processor do + let(:project) { build(:project) } + let(:environment) { build(:environment) } + let(:dashboard_yml) { YAML.load_file('spec/fixtures/lib/gitlab/metrics/dashboard/sample_dashboard.yml') } + + describe 'process' do + let(:process_params) { [project, environment, dashboard_yml] } + let(:dashboard) { described_class.new(*process_params).process } + + context 'when dashboard config corresponds to common metrics' do + let!(:common_metric) { create(:prometheus_metric, :common, identifier: 'metric_a1') } + + it 'inserts metric ids into the config' do + target_metric = all_metrics.find { |metric| metric[:id] == 'metric_a1' } + + expect(target_metric).to include(:metric_id) + expect(target_metric[:metric_id]).to eq(common_metric.id) + end + end + + context 'when the project has associated metrics' do + let!(:project_response_metric) { create(:prometheus_metric, project: project, group: :response) } + let!(:project_system_metric) { create(:prometheus_metric, project: project, group: :system) } + let!(:project_business_metric) { create(:prometheus_metric, project: project, group: :business) } + + it 'includes project-specific metrics' do + expect(all_metrics).to include get_metric_details(project_system_metric) + expect(all_metrics).to include get_metric_details(project_response_metric) + expect(all_metrics).to include get_metric_details(project_business_metric) + end + + it 'orders groups by priority and panels by weight' do + expected_metrics_order = [ + 'metric_a2', # group priority 10, panel weight 2 + 'metric_a1', # group priority 10, panel weight 1 + 'metric_b', # group priority 1, panel weight 1 + project_business_metric.id, # group priority 0, panel weight nil (0) + project_response_metric.id, # group priority -5, panel weight nil (0) + project_system_metric.id, # group priority -10, panel weight nil (0) + ] + actual_metrics_order = all_metrics.map { |m| m[:id] || m[:metric_id] } + + expect(actual_metrics_order).to eq expected_metrics_order + end + end + + shared_examples_for 'errors with message' do |expected_message| + it 'raises a DashboardLayoutError' do + error_class = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardLayoutError + + expect { dashboard }.to raise_error(error_class, expected_message) + end + end + + context 'when the dashboard is missing panel_groups' do + let(:dashboard_yml) { {} } + + it_behaves_like 'errors with message', 'Top-level key :panel_groups must be an array' + end + + context 'when the dashboard contains a panel_group which is missing panels' do + let(:dashboard_yml) { { panel_groups: [{}] } } + + it_behaves_like 'errors with message', 'Each "panel_group" must define an array :panels' + end + + context 'when the dashboard contains a panel which is missing metrics' do + let(:dashboard_yml) { { panel_groups: [{ panels: [{}] }] } } + + it_behaves_like 'errors with message', 'Each "panel" must define an array :metrics' + end + end + + private + + def all_metrics + dashboard[:panel_groups].map do |group| + group[:panels].map { |panel| panel[:metrics] } + end.flatten + end + + def get_metric_details(metric) + { + query_range: metric.query, + unit: metric.unit, + label: metric.legend, + metric_id: metric.id + } + end +end diff --git a/spec/lib/gitlab/metrics/dashboard/service_spec.rb b/spec/lib/gitlab/metrics/dashboard/service_spec.rb new file mode 100644 index 00000000000..e66c356bf49 --- /dev/null +++ b/spec/lib/gitlab/metrics/dashboard/service_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Metrics::Dashboard::Service, :use_clean_rails_memory_store_caching do + let(:project) { build(:project) } + let(:environment) { build(:environment) } + + describe 'get_dashboard' do + let(:dashboard_schema) { JSON.parse(fixture_file('lib/gitlab/metrics/dashboard/schemas/dashboard.json')) } + + it 'returns a json representation of the environment dashboard' do + result = described_class.new(project, environment).get_dashboard + + expect(result.keys).to contain_exactly(:dashboard, :status) + expect(result[:status]).to eq(:success) + + expect(JSON::Validator.fully_validate(dashboard_schema, result[:dashboard])).to be_empty + end + + it 'caches the dashboard for subsequent calls' do + expect(YAML).to receive(:safe_load).once.and_call_original + + described_class.new(project, environment).get_dashboard + described_class.new(project, environment).get_dashboard + end + + context 'when the dashboard is configured incorrectly' do + before do + allow(YAML).to receive(:safe_load).and_return({}) + end + + it 'returns an appropriate message and status code' do + result = described_class.new(project, environment).get_dashboard + + expect(result.keys).to contain_exactly(:message, :http_status, :status) + expect(result[:status]).to eq(:error) + expect(result[:http_status]).to eq(:unprocessable_entity) + end + end + end +end |