diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-20 12:26:25 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-20 12:26:25 +0000 |
commit | a09983ae35713f5a2bbb100981116d31ce99826e (patch) | |
tree | 2ee2af7bd104d57086db360a7e6d8c9d5d43667a /lib/gitlab/metrics | |
parent | 18c5ab32b738c0b6ecb4d0df3994000482f34bd8 (diff) | |
download | gitlab-ce-a09983ae35713f5a2bbb100981116d31ce99826e.tar.gz |
Add latest changes from gitlab-org/gitlab@13-2-stable-ee
Diffstat (limited to 'lib/gitlab/metrics')
-rw-r--r-- | lib/gitlab/metrics/background_transaction.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/errors.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/finder.rb | 45 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/service_selector.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/stages/base_stage.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb | 83 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb (renamed from lib/gitlab/metrics/dashboard/stages/endpoint_inserter.rb) | 10 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/stages/sorter.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/stages/url_validator.rb | 43 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb | 34 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/url.rb | 16 | ||||
-rw-r--r-- | lib/gitlab/metrics/methods.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/metrics/sidekiq_middleware.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/metrics/subscribers/active_record.rb | 22 | ||||
-rw-r--r-- | lib/gitlab/metrics/transaction.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/metrics/web_transaction.rb | 11 |
16 files changed, 263 insertions, 39 deletions
diff --git a/lib/gitlab/metrics/background_transaction.rb b/lib/gitlab/metrics/background_transaction.rb index fe1722b1095..7b05ae29b02 100644 --- a/lib/gitlab/metrics/background_transaction.rb +++ b/lib/gitlab/metrics/background_transaction.rb @@ -9,7 +9,7 @@ module Gitlab end def labels - { controller: @worker_class.name, action: 'perform' } + { controller: @worker_class.name, action: 'perform', feature_category: @worker_class.try(:get_feature_category).to_s } end end end diff --git a/lib/gitlab/metrics/dashboard/errors.rb b/lib/gitlab/metrics/dashboard/errors.rb index 264ea0488e7..07ddd315bcc 100644 --- a/lib/gitlab/metrics/dashboard/errors.rb +++ b/lib/gitlab/metrics/dashboard/errors.rb @@ -20,20 +20,20 @@ module Gitlab when DashboardProcessingError error(error.message, :unprocessable_entity) when NOT_FOUND_ERROR - error("#{dashboard_path} could not be found.", :not_found) + error(_("%{dashboard_path} could not be found.") % { dashboard_path: dashboard_path }, :not_found) when PanelNotFoundError error(error.message, :not_found) when ::Grafana::Client::Error error(error.message, :service_unavailable) when MissingIntegrationError - error('Proxy support for this API is not available currently', :bad_request) + error(_('Proxy support for this API is not available currently'), :bad_request) else raise error end end def panels_not_found!(opts) - raise PanelNotFoundError.new("No panels matching properties #{opts}") + raise PanelNotFoundError.new(_("No panels matching properties %{opts}") % { opts: opts }) end end end diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index d80985e0a0e..5e2d78e10a4 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -7,6 +7,19 @@ module Gitlab module Metrics module Dashboard class Finder + # Dashboards that should not be part of the list of all dashboards + # displayed on the metrics dashboard page. + PREDEFINED_DASHBOARD_EXCLUSION_LIST = [ + # This dashboard is only useful in the self monitoring project. + ::Metrics::Dashboard::SelfMonitoringDashboardService, + + # This dashboard is displayed on the K8s cluster settings health page. + ::Metrics::Dashboard::ClusterDashboardService, + + # This dashboard is not yet ready for the world. + ::Metrics::Dashboard::PodDashboardService + ].freeze + class << self # Returns a formatted dashboard packed with DB info. # @param project [Project] @@ -67,12 +80,32 @@ module Gitlab def find_all_paths_from_source(project) Gitlab::Metrics::Dashboard::Cache.delete_all! - default_dashboard_path(project) - .+ project_service.all_dashboard_paths(project) + user_facing_dashboard_services(project).flat_map do |service| + service.all_dashboard_paths(project) + end end private + def user_facing_dashboard_services(project) + predefined_dashboard_services_for(project) + [project_service] + end + + def predefined_dashboard_services_for(project) + # Only list the self monitoring dashboard on the self monitoring project, + # since it is the only dashboard (at time of writing) that shows data + # about GitLab itself. + if project.self_monitoring? + return [self_monitoring_service] + end + + predefined_dashboard_services + end + + def predefined_dashboard_services + ::Metrics::Dashboard::PredefinedDashboardService.descendants - PREDEFINED_DASHBOARD_EXCLUSION_LIST + end + def system_service ::Metrics::Dashboard::SystemDashboardService end @@ -85,14 +118,6 @@ module Gitlab ::Metrics::Dashboard::SelfMonitoringDashboardService end - def default_dashboard_path(project) - if project.self_monitoring? - self_monitoring_service.all_dashboard_paths(project) - else - system_service.all_dashboard_paths(project) - end - end - def service_for(options) Gitlab::Metrics::Dashboard::ServiceSelector.call(options) end diff --git a/lib/gitlab/metrics/dashboard/service_selector.rb b/lib/gitlab/metrics/dashboard/service_selector.rb index 49682da320c..641c0c76f8f 100644 --- a/lib/gitlab/metrics/dashboard/service_selector.rb +++ b/lib/gitlab/metrics/dashboard/service_selector.rb @@ -13,6 +13,8 @@ module Gitlab include Gitlab::Utils::StrongMemoize SERVICES = [ + ::Metrics::Dashboard::ClusterMetricsEmbedService, + ::Metrics::Dashboard::ClusterDashboardService, ::Metrics::Dashboard::GitlabAlertEmbedService, ::Metrics::Dashboard::CustomMetricEmbedService, ::Metrics::Dashboard::GrafanaMetricEmbedService, @@ -51,5 +53,3 @@ module Gitlab end end end - -Gitlab::Metrics::Dashboard::ServiceSelector.prepend_if_ee('EE::Gitlab::Metrics::Dashboard::ServiceSelector') diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb index 622d5aa8cdb..ee2d36621b4 100644 --- a/lib/gitlab/metrics/dashboard/stages/base_stage.rb +++ b/lib/gitlab/metrics/dashboard/stages/base_stage.rb @@ -48,6 +48,14 @@ module Gitlab end end + def for_variables + return unless dashboard.dig(:templating, :variables).is_a?(Hash) + + dashboard.dig(:templating, :variables).each do |variable_name, variable| + yield variable_name, variable + end + end + def for_panel_groups dashboard[:panel_groups].each do |panel_group| yield panel_group diff --git a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb new file mode 100644 index 00000000000..a12082b704c --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class ClusterEndpointInserter < BaseStage + def transform! + verify_params + + for_metrics do |metric| + metric[:prometheus_endpoint_path] = endpoint_for_metric(metric) + end + end + + private + + def admin_url(metric) + Gitlab::Routing.url_helpers.prometheus_api_admin_cluster_path( + params[:cluster], + proxy_path: query_type(metric), + query: query_for_metric(metric) + ) + end + + def endpoint_for_metric(metric) + case params[:cluster_type] + when :admin + admin_url(metric) + when :group + error!(_('Group is required when cluster_type is :group')) unless params[:group] + group_url(metric) + when :project + error!(_('Project is required when cluster_type is :project')) unless project + project_url(metric) + else + error!(_('Unrecognized cluster type')) + end + end + + def error!(message) + raise Errors::DashboardProcessingError.new(message) + end + + def group_url(metric) + Gitlab::Routing.url_helpers.prometheus_api_group_cluster_path( + params[:group], + params[:cluster], + proxy_path: query_type(metric), + query: query_for_metric(metric) + ) + end + + def project_url(metric) + Gitlab::Routing.url_helpers.prometheus_api_project_cluster_path( + project, + params[:cluster], + proxy_path: query_type(metric), + query: query_for_metric(metric) + ) + end + + def query_type(metric) + metric[:query] ? :query : :query_range + end + + def query_for_metric(metric) + query = metric[query_type(metric)] + + raise Errors::MissingQueryError.new('Each "metric" must define one of :query or :query_range') unless query + + query + end + + def verify_params + raise Errors::DashboardProcessingError.new(_('Cluster is required for Stages::ClusterEndpointInserter')) unless params[:cluster] + raise Errors::DashboardProcessingError.new(_('Cluster type must be specificed for Stages::ClusterEndpointInserter')) unless params[:cluster_type] + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb index e085f551952..c48a7ff25a5 100644 --- a/lib/gitlab/metrics/dashboard/stages/endpoint_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb @@ -4,9 +4,9 @@ module Gitlab module Metrics module Dashboard module Stages - class EndpointInserter < BaseStage + class MetricEndpointInserter < BaseStage def transform! - raise Errors::DashboardProcessingError.new('Environment is required for Stages::EndpointInserter') unless params[:environment] + raise Errors::DashboardProcessingError.new(_('Environment is required for Stages::MetricEndpointInserter')) unless params[:environment] for_metrics do |metric| metric[:prometheus_endpoint_path] = endpoint_for_metric(metric) @@ -33,7 +33,11 @@ module Gitlab end def query_type(metric) - metric[:query] ? :query : :query_range + if metric[:query] + ::Prometheus::ProxyService::PROMETHEUS_QUERY_API.to_sym + else + ::Prometheus::ProxyService::PROMETHEUS_QUERY_RANGE_API.to_sym + end end def query_for_metric(metric) diff --git a/lib/gitlab/metrics/dashboard/stages/sorter.rb b/lib/gitlab/metrics/dashboard/stages/sorter.rb index ba5aa78059c..882211e1441 100644 --- a/lib/gitlab/metrics/dashboard/stages/sorter.rb +++ b/lib/gitlab/metrics/dashboard/stages/sorter.rb @@ -16,7 +16,7 @@ module Gitlab # Sorts the groups in the dashboard by the :priority key def sort_groups! - dashboard[:panel_groups] = dashboard[:panel_groups].sort_by { |group| -group[:priority].to_i } + dashboard[:panel_groups] = Gitlab::Utils.stable_sort_by(dashboard[:panel_groups]) { |group| -group[:priority].to_i } end # Sorts the panels in the dashboard by the :weight key @@ -24,7 +24,7 @@ module Gitlab dashboard[:panel_groups].each do |group| missing_panels! unless group[:panels].is_a? Array - group[:panels] = group[:panels].sort_by { |panel| -panel[:weight].to_i } + group[:panels] = Gitlab::Utils.stable_sort_by(group[:panels]) { |panel| -panel[:weight].to_i } end end end diff --git a/lib/gitlab/metrics/dashboard/stages/url_validator.rb b/lib/gitlab/metrics/dashboard/stages/url_validator.rb index ff36f7b605e..9e2bb0d1a70 100644 --- a/lib/gitlab/metrics/dashboard/stages/url_validator.rb +++ b/lib/gitlab/metrics/dashboard/stages/url_validator.rb @@ -6,8 +6,47 @@ module Gitlab module Stages class UrlValidator < BaseStage def transform! - dashboard[:links]&.each do |link| - Gitlab::UrlBlocker.validate!(link[:url]) + validate_dashboard_links(dashboard) + + validate_chart_links(dashboard) + end + + private + + def blocker_args + { + schemes: %w(http https), + ports: [], + allow_localhost: allow_setting_local_requests?, + allow_local_network: allow_setting_local_requests?, + ascii_only: false, + enforce_user: false, + enforce_sanitization: false, + dns_rebind_protection: true + } + end + + def allow_setting_local_requests? + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? + end + + def validate_dashboard_links(dashboard) + validate_links(dashboard[:links]) + end + + def validate_chart_links(dashboard) + dashboard[:panel_groups].each do |panel_group| + panel_group[:panels].each do |panel| + validate_links(panel[:links]) + end + end + end + + def validate_links(links) + links&.each do |link| + next unless link.is_a? Hash + + Gitlab::UrlBlocker.validate!(link[:url], blocker_args) rescue Gitlab::UrlBlocker::BlockedUrlError link[:url] = '' end diff --git a/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb new file mode 100644 index 00000000000..20e7fe477e5 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class VariableEndpointInserter < BaseStage + VARIABLE_TYPE_METRIC_LABEL_VALUES = 'metric_label_values' + + def transform! + raise Errors::DashboardProcessingError.new(_('Environment is required for Stages::VariableEndpointInserter')) unless params[:environment] + + for_variables do |variable_name, variable| + if variable.is_a?(Hash) && variable[:type] == VARIABLE_TYPE_METRIC_LABEL_VALUES + variable[:options][:prometheus_endpoint_path] = endpoint_for_variable(variable.dig(:options, :series_selector)) + end + end + end + + private + + def endpoint_for_variable(series_selector) + Gitlab::Routing.url_helpers.prometheus_api_project_environment_path( + project, + params[:environment], + proxy_path: ::Prometheus::ProxyService::PROMETHEUS_SERIES_API, + match: Array(series_selector) + ) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb index 31670a3f533..10a2f3c2397 100644 --- a/lib/gitlab/metrics/dashboard/url.rb +++ b/lib/gitlab/metrics/dashboard/url.rb @@ -60,6 +60,22 @@ module Gitlab Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(*args) end + # Matches dashboard urls for a metric chart embed + # for cluster metrics + # + # EX - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB) + def clusters_regex + strong_memoize(:clusters_regex) do + regex_for_project_metrics( + %r{ + /clusters + /(?<cluster_id>\d+) + /? + }x + ) + end + end + private def regex_for_project_metrics(path_suffix_pattern) diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb index 5955987541c..83a7b925392 100644 --- a/lib/gitlab/metrics/methods.rb +++ b/lib/gitlab/metrics/methods.rb @@ -35,7 +35,7 @@ module Gitlab end def init_metric(type, name, opts = {}, &block) - options = MetricOptions.new(opts) + options = ::Gitlab::Metrics::Methods::MetricOptions.new(opts) options.evaluate(&block) if disabled_by_feature(options) diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index de8e1ca3256..1c99e1e730c 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -26,9 +26,7 @@ module Gitlab private def add_info_to_payload(payload, trans) - payload[:db_count] = trans.get(:db_count, :counter).to_i - payload[:db_write_count] = trans.get(:db_write_count, :counter).to_i - payload[:db_cached_count] = trans.get(:db_cached_count, :counter).to_i + payload.merge!(::Gitlab::Metrics::Subscribers::ActiveRecord.db_counter_payload) end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 1628eeb5a95..d2736882432 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -23,6 +23,14 @@ module Gitlab increment_db_counters(payload) end + def self.db_counter_payload + return {} unless Gitlab::SafeRequestStore.active? + + DB_COUNTERS.map do |counter| + [counter, Gitlab::SafeRequestStore[counter].to_i] + end.to_h + end + private define_histogram :gitlab_sql_duration_seconds do @@ -36,13 +44,21 @@ module Gitlab end def increment_db_counters(payload) - current_transaction.increment(:db_count, 1) + increment(:db_count) if payload.fetch(:cached, payload[:name] == 'CACHE') - current_transaction.increment(:db_cached_count, 1) + increment(:db_cached_count) end - current_transaction.increment(:db_write_count, 1) unless select_sql_command?(payload) + increment(:db_write_count) unless select_sql_command?(payload) + end + + def increment(counter) + current_transaction.increment(counter, 1) + + if Gitlab::SafeRequestStore.active? + Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1 + end end def current_transaction diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 822f5243e9d..da06be9c79c 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -7,7 +7,7 @@ module Gitlab include Gitlab::Metrics::Methods # base labels shared among all transactions - BASE_LABELS = { controller: nil, action: nil }.freeze + BASE_LABELS = { controller: nil, action: nil, feature_category: nil }.freeze # labels that potentially contain sensitive information and will be filtered FILTERED_LABELS = [:branch, :path].freeze @@ -92,12 +92,6 @@ module Gitlab self.class.transaction_metric(name, :gauge).set(labels, value) if use_prometheus end - def get(name, type, tags = {}) - metric = self.class.transaction_metric(name, type) - - metric.get(filter_tags(tags).merge(labels)) - end - def labels BASE_LABELS end diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb index fa17548723e..2064f9290d3 100644 --- a/lib/gitlab/metrics/web_transaction.rb +++ b/lib/gitlab/metrics/web_transaction.rb @@ -32,6 +32,10 @@ module Gitlab action = "#{controller.action_name}" + # Try to get the feature category, but don't fail when the controller is + # not an ApplicationController. + feature_category = controller.class.try(:feature_category_for_action, action).to_s + # Devise exposes a method called "request_format" that does the below. # However, this method is not available to all controllers (e.g. certain # Doorkeeper controllers). As such we use the underlying code directly. @@ -45,7 +49,7 @@ module Gitlab action = "#{action}.#{suffix}" end - { controller: controller.class.name, action: action } + { controller: controller.class.name, action: action, feature_category: feature_category } end def labels_from_endpoint @@ -61,7 +65,10 @@ module Gitlab if route path = endpoint_paths_cache[route.request_method][route.path] - { controller: 'Grape', action: "#{route.request_method} #{path}" } + + # Feature categories will be added for grape endpoints in + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/462 + { controller: 'Grape', action: "#{route.request_method} #{path}", feature_category: '' } end end |