From c64fc07793fc20c5e2ff1b5f557d5c3118eca30e Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Thu, 25 May 2017 14:01:10 +0200 Subject: Finalize refactoring additional metrics query --- app/models/environment.rb | 4 +- app/models/project_services/prometheus_service.rb | 2 +- lib/gitlab/prometheus/metric_group.rb | 8 +-- .../prometheus/queries/additional_metrics_query.rb | 79 +++++++++++----------- .../prometheus/queries/matched_metrics_query.rb | 6 +- 5 files changed, 49 insertions(+), 50 deletions(-) diff --git a/app/models/environment.rb b/app/models/environment.rb index b4a4f74a8d5..7ab4a23ab16 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -158,7 +158,9 @@ class Environment < ActiveRecord::Base end def additional_metrics - project.monitoring_service.reactive_query(Gitlab::Prometheus::Queries::AdditionalMetricsQuery, self.id) if has_additional_metrics? + if has_additional_metrics? + project.monitoring_service.reactive_query(Gitlab::Prometheus::Queries::AdditionalMetricsQuery.name, self.id, &:itself) + end end # An environment name is not necessarily suitable for use in URLs, DNS diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb index 674d485a03c..d3f43d66937 100644 --- a/app/models/project_services/prometheus_service.rb +++ b/app/models/project_services/prometheus_service.rb @@ -73,7 +73,7 @@ class PrometheusService < MonitoringService end def reactive_query(query_class, *args, &block) - calculate_reactive_cache(query_class, *args, &block) + with_reactive_cache(query_class, *args, &block) end # Cache metrics for specific environment diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb index 093390b4fa7..9f95525bc0c 100644 --- a/lib/gitlab/prometheus/metric_group.rb +++ b/lib/gitlab/prometheus/metric_group.rb @@ -13,6 +13,10 @@ module Gitlab::Prometheus load_groups_from_yaml end + def self.load_groups_from_yaml + additional_metrics_raw.map(&method(:group_from_entry)) + end + def self.group_from_entry(entry) missing_fields = [:group, :priority, :metrics].select { |key| !entry.has_key?(key) } raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty? @@ -22,10 +26,6 @@ module Gitlab::Prometheus group end - def self.load_groups_from_yaml - additional_metrics_raw.map(&method(:group_from_entry)) - end - def self.additional_metrics_raw @additional_metrics_raw ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml'))&.map(&:deep_symbolize_keys).freeze end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_query.rb index 001701383c3..de0dab0b19c 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_query.rb @@ -1,47 +1,46 @@ module Gitlab::Prometheus::Queries class AdditionalMetricsQuery < BaseQuery - def self.metrics - @metrics ||= YAML.load_file(Rails.root.join('config/custom_metrics.yml')).freeze - end - def query(environment_id) - environment = Environment.find_by(id: environment_id) - - context = { - environment_slug: environment.slug, - environment_filter: %{container_name!="POD",environment="#{environment.slug}"} - } - - timeframe_start = 8.hours.ago.to_f - timeframe_end = Time.now.to_f + query_processor = method(:process_query).curry[query_context(environment_id)] matched_metrics.map do |group| - group[:metrics].map! do |metric| - metric[:queries].map! do |query| - query = query.symbolize_keys - query[:result] = - if query.has_key?(:query_range) - client_query_range(query[:query_range] % context, start: timeframe_start, stop: timeframe_end) - else - client_query(query[:query] % context, time: timeframe_end) - end - query - end - metric + metrics = group.metrics.map do |metric| + { + title: metric.title, + weight: metric.weight, + queries: metric.queries.map(&query_processor) + } end - group + + { + group: group.name, + priority: group.priority, + metrics: metrics + } end end - def process_query(group, query) - result = if query.has_key?(:query_range) - client_query_range(query[:query_range] % context, start: timeframe_start, stop: timeframe_end) - else - client_query(query[:query] % context, time: timeframe_end) - end - contains_metrics = result.all? do |item| - item&.[](:values)&.any? || item&.[](:value)&.any? - end + private + + def query_context(environment_id) + environment = Environment.find_by(id: environment_id) + { + environment_slug: environment.slug, + environment_filter: %{container_name!="POD",environment="#{environment.slug}"}, + timeframe_start: 8.hours.ago.to_f, + timeframe_end: Time.now.to_f + } + end + + def process_query(context, query) + query_with_result = query.dup + query_with_result[:result] = + if query.has_key?(:query_range) + client_query_range(query[:query_range] % context, start: context[:timeframe_start], stop: context[:timeframe_end]) + else + client_query(query[:query] % context, time: context[:timeframe_end]) + end + query_with_result end def process_result(query_result) @@ -54,17 +53,17 @@ module Gitlab::Prometheus::Queries def matched_metrics label_values = client_label_values || [] + Gitlab::Prometheus::MetricGroup.all - result = Gitlab::Prometheus::MetricsSources.additional_metrics.map do |group| - group[:metrics].map!(&:symbolize_keys) - group[:metrics].select! do |metric| - matcher = Regexp.compile(metric[:detect]) + result = Gitlab::Prometheus::MetricGroup.all.map do |group| + group.metrics.select! do |metric| + matcher = Regexp.compile(metric.detect) label_values.any? &matcher.method(:match) end group end - result.select {|group| !group[:metrics].empty?} + result.select { |group| group.metrics.any? } end end end diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb index 61926320e40..5a8b0f6c701 100644 --- a/lib/gitlab/prometheus/queries/matched_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb @@ -2,10 +2,6 @@ module Gitlab::Prometheus::Queries class MatchedMetricsQuery < BaseQuery MAX_QUERY_ITEMS = 40.freeze - def self.metrics - @metrics ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml')).map(&:deep_symbolize_keys) - end - def query groups_data.map do |group, data| { @@ -17,6 +13,8 @@ module Gitlab::Prometheus::Queries end end + private + def groups_data metrics_series = metrics_with_series(Gitlab::Prometheus::MetricGroup.all) lookup = active_series_lookup(metrics_series) -- cgit v1.2.1