summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--app/controllers/projects/environments_controller.rb6
-rw-r--r--app/controllers/projects/prometheus_controller.rb22
-rw-r--r--app/models/environment.rb8
-rw-r--r--app/models/project_services/prometheus_service.rb20
-rw-r--r--config/additional_metrics.yml26
-rw-r--r--config/routes/project.rb5
-rw-r--r--lib/gitlab/prometheus/metric.rb28
-rw-r--r--lib/gitlab/prometheus/metric_group.rb33
-rw-r--r--lib/gitlab/prometheus/metrics_sources.rb7
-rw-r--r--lib/gitlab/prometheus/parsing_error.rb3
-rw-r--r--lib/gitlab/prometheus/queries/additional_metrics_query.rb70
-rw-r--r--lib/gitlab/prometheus/queries/base_query.rb2
-rw-r--r--lib/gitlab/prometheus/queries/matched_metrics_query.rb72
-rw-r--r--lib/gitlab/prometheus_client.rb8
14 files changed, 304 insertions, 6 deletions
diff --git a/app/controllers/projects/environments_controller.rb b/app/controllers/projects/environments_controller.rb
index efe83776834..6d230e84ef7 100644
--- a/app/controllers/projects/environments_controller.rb
+++ b/app/controllers/projects/environments_controller.rb
@@ -129,6 +129,12 @@ class Projects::EnvironmentsController < Projects::ApplicationController
end
end
+ def additional_metrics
+ additional_metrics = environment.additional_metrics || {}
+
+ render json: additional_metrics, status: additional_metrics.any? ? :ok : :no_content
+ end
+
private
def verify_api_request!
diff --git a/app/controllers/projects/prometheus_controller.rb b/app/controllers/projects/prometheus_controller.rb
new file mode 100644
index 00000000000..74e247535d5
--- /dev/null
+++ b/app/controllers/projects/prometheus_controller.rb
@@ -0,0 +1,22 @@
+class Projects::PrometheusController < Projects::ApplicationController
+ before_action :authorize_read_project!
+
+ def active_metrics
+ return render_404 unless has_prometheus_metrics?
+ matched_metrics = prometheus_service.reactive_query(Gitlab::Prometheus::Queries::MatchedMetricsQuery.name, &:itself)
+
+ if matched_metrics
+ render json: matched_metrics, status: :ok
+ else
+ head :no_content
+ end
+ end
+
+ def prometheus_service
+ project.monitoring_service
+ end
+
+ def has_prometheus_metrics?
+ prometheus_service&.respond_to?(:reactive_query)
+ end
+end
diff --git a/app/models/environment.rb b/app/models/environment.rb
index 61572d8d69a..b4a4f74a8d5 100644
--- a/app/models/environment.rb
+++ b/app/models/environment.rb
@@ -149,10 +149,18 @@ class Environment < ActiveRecord::Base
project.monitoring_service.present? && available? && last_deployment.present?
end
+ def has_additional_metrics?
+ has_metrics? && project.monitoring_service&.respond_to?(:reactive_query)
+ end
+
def metrics
project.monitoring_service.environment_metrics(self) if has_metrics?
end
+ def additional_metrics
+ project.monitoring_service.reactive_query(Gitlab::Prometheus::Queries::AdditionalMetricsQuery, self.id) if has_additional_metrics?
+ end
+
# An environment name is not necessarily suitable for use in URLs, DNS
# or other third-party contexts, so provide a slugified version. A slug has
# the following properties:
diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb
index ec72cb6856d..674d485a03c 100644
--- a/app/models/project_services/prometheus_service.rb
+++ b/app/models/project_services/prometheus_service.rb
@@ -64,23 +64,26 @@ class PrometheusService < MonitoringService
end
def environment_metrics(environment)
- with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &:itself)
+ with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &method(:rename_data_to_metrics))
end
def deployment_metrics(deployment)
- metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.id, &:itself)
+ metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.id, &method(:rename_data_to_metrics))
metrics&.merge(deployment_time: created_at.to_i) || {}
end
+ def reactive_query(query_class, *args, &block)
+ calculate_reactive_cache(query_class, *args, &block)
+ end
+
# Cache metrics for specific environment
def calculate_reactive_cache(query_class_name, *args)
return unless active? && project && !project.pending_delete?
- metrics = Kernel.const_get(query_class_name).new(client).query(*args)
-
+ data = Kernel.const_get(query_class_name).new(client).query(*args)
{
success: true,
- metrics: metrics,
+ data: data,
last_update: Time.now.utc
}
rescue Gitlab::PrometheusError => err
@@ -90,4 +93,11 @@ class PrometheusService < MonitoringService
def client
@prometheus ||= Gitlab::PrometheusClient.new(api_url: api_url)
end
+
+ private
+
+ def rename_data_to_metrics(metrics)
+ metrics[:metrics] = metrics.delete :data
+ metrics
+ end
end
diff --git a/config/additional_metrics.yml b/config/additional_metrics.yml
new file mode 100644
index 00000000000..209209f4b30
--- /dev/null
+++ b/config/additional_metrics.yml
@@ -0,0 +1,26 @@
+- group: Kubernetes
+ priority: 1
+ metrics:
+ - title: "Memory usage"
+ detect: container_memory_usage_bytes
+ weight: 1
+ queries:
+ - query_range: 'avg(container_memory_usage_bytes{%{environment_filter}}) / 2^20'
+ label: Container memory
+ unit: MiB
+ - title: "Current memory usage"
+ detect: container_memory_usage_bytes
+ weight: 1
+ queries:
+ - query: 'avg(container_memory_usage_bytes{%{environment_filter}}) / 2^20'
+ unit: MiB
+ - title: "CPU usage"
+ detect: container_cpu_usage_seconds_total
+ weight: 1
+ queries:
+ - query_range: 'avg(rate(container_cpu_usage_seconds_total{%{environment_filter}}[2m])) * 100'
+ - title: "Current CPU usage"
+ detect: container_cpu_usage_seconds_total
+ weight: 1
+ queries:
+ - query: 'avg(rate(container_cpu_usage_seconds_total{%{environment_filter}}[2m])) * 100'
diff --git a/config/routes/project.rb b/config/routes/project.rb
index 9fe8372edf9..f5d00e4e93b 100644
--- a/config/routes/project.rb
+++ b/config/routes/project.rb
@@ -72,6 +72,10 @@ constraints(ProjectUrlConstrainer.new) do
resource :mattermost, only: [:new, :create]
+ namespace :prometheus do
+ get :active_metrics
+ end
+
resources :deploy_keys, constraints: { id: /\d+/ }, only: [:index, :new, :create] do
member do
put :enable
@@ -152,6 +156,7 @@ constraints(ProjectUrlConstrainer.new) do
post :stop
get :terminal
get :metrics
+ get :additional_metrics
get '/terminal.ws/authorize', to: 'environments#terminal_websocket_authorize', constraints: { format: nil }
end
diff --git a/lib/gitlab/prometheus/metric.rb b/lib/gitlab/prometheus/metric.rb
new file mode 100644
index 00000000000..2818afb34b0
--- /dev/null
+++ b/lib/gitlab/prometheus/metric.rb
@@ -0,0 +1,28 @@
+module Gitlab::Prometheus
+ class Metric
+ attr_reader :group, :title, :detect, :weight, :queries
+
+ def initialize(group, title, detect, weight, queries = [])
+ @group = group
+ @title = title
+ @detect = detect
+ @weight = weight
+ @queries = queries
+ end
+
+ def self.metric_from_entry(group, entry)
+ missing_fields = [:title, :detect, :weight, :queries].select { |key| !entry.has_key?(key) }
+ raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty?
+
+ Metric.new(group, entry[:title], entry[:detect], entry[:weight], entry[:queries])
+ end
+
+ def self.metrics_from_list(group, list)
+ list.map { |entry| metric_from_entry(group, entry) }
+ end
+
+ def self.additional_metrics_raw
+ @additional_metrics_raw ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml')).map(&:deep_symbolize_keys)
+ end
+ end
+end
diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb
new file mode 100644
index 00000000000..093390b4fa7
--- /dev/null
+++ b/lib/gitlab/prometheus/metric_group.rb
@@ -0,0 +1,33 @@
+module Gitlab::Prometheus
+ class MetricGroup
+ attr_reader :priority, :name
+ attr_accessor :metrics
+
+ def initialize(name, priority, metrics = [])
+ @name = name
+ @priority = priority
+ @metrics = metrics
+ end
+
+ def self.all
+ load_groups_from_yaml
+ end
+
+ def self.group_from_entry(entry)
+ missing_fields = [:group, :priority, :metrics].select { |key| !entry.has_key?(key) }
+ raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty?
+
+ group = MetricGroup.new(entry[:group], entry[:priority])
+ group.metrics = Metric.metrics_from_list(group, entry[:metrics])
+ group
+ end
+
+ def self.load_groups_from_yaml
+ additional_metrics_raw.map(&method(:group_from_entry))
+ end
+
+ def self.additional_metrics_raw
+ @additional_metrics_raw ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml'))&.map(&:deep_symbolize_keys).freeze
+ end
+ end
+end
diff --git a/lib/gitlab/prometheus/metrics_sources.rb b/lib/gitlab/prometheus/metrics_sources.rb
new file mode 100644
index 00000000000..500b6e971a2
--- /dev/null
+++ b/lib/gitlab/prometheus/metrics_sources.rb
@@ -0,0 +1,7 @@
+module Gitlab::Prometheus
+ module MetricsSources
+ def self.additional_metrics
+ @additional_metrics ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml')).deep_symbolize_keys.freeze
+ end
+ end
+end
diff --git a/lib/gitlab/prometheus/parsing_error.rb b/lib/gitlab/prometheus/parsing_error.rb
new file mode 100644
index 00000000000..067ea7f878a
--- /dev/null
+++ b/lib/gitlab/prometheus/parsing_error.rb
@@ -0,0 +1,3 @@
+module Gitlab::Prometheus
+ ParsingError = Class.new(StandardError)
+end
diff --git a/lib/gitlab/prometheus/queries/additional_metrics_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_query.rb
new file mode 100644
index 00000000000..001701383c3
--- /dev/null
+++ b/lib/gitlab/prometheus/queries/additional_metrics_query.rb
@@ -0,0 +1,70 @@
+module Gitlab::Prometheus::Queries
+ class AdditionalMetricsQuery < BaseQuery
+ def self.metrics
+ @metrics ||= YAML.load_file(Rails.root.join('config/custom_metrics.yml')).freeze
+ end
+
+ def query(environment_id)
+ environment = Environment.find_by(id: environment_id)
+
+ context = {
+ environment_slug: environment.slug,
+ environment_filter: %{container_name!="POD",environment="#{environment.slug}"}
+ }
+
+ timeframe_start = 8.hours.ago.to_f
+ timeframe_end = Time.now.to_f
+
+ matched_metrics.map do |group|
+ group[:metrics].map! do |metric|
+ metric[:queries].map! do |query|
+ query = query.symbolize_keys
+ query[:result] =
+ if query.has_key?(:query_range)
+ client_query_range(query[:query_range] % context, start: timeframe_start, stop: timeframe_end)
+ else
+ client_query(query[:query] % context, time: timeframe_end)
+ end
+ query
+ end
+ metric
+ end
+ group
+ end
+ end
+
+ def process_query(group, query)
+ result = if query.has_key?(:query_range)
+ client_query_range(query[:query_range] % context, start: timeframe_start, stop: timeframe_end)
+ else
+ client_query(query[:query] % context, time: timeframe_end)
+ end
+ contains_metrics = result.all? do |item|
+ item&.[](:values)&.any? || item&.[](:value)&.any?
+ end
+ end
+
+ def process_result(query_result)
+ contains_metrics = query_result.all? do |item|
+ item&.[](:values)&.any? || item&.[](:value)&.any?
+ end
+
+ contains_metrics
+ end
+
+ def matched_metrics
+ label_values = client_label_values || []
+
+ result = Gitlab::Prometheus::MetricsSources.additional_metrics.map do |group|
+ group[:metrics].map!(&:symbolize_keys)
+ group[:metrics].select! do |metric|
+ matcher = Regexp.compile(metric[:detect])
+ label_values.any? &matcher.method(:match)
+ end
+ group
+ end
+
+ result.select {|group| !group[:metrics].empty?}
+ end
+ end
+end
diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb
index 2a2eb4ae57f..c60828165bd 100644
--- a/lib/gitlab/prometheus/queries/base_query.rb
+++ b/lib/gitlab/prometheus/queries/base_query.rb
@@ -3,7 +3,7 @@ module Gitlab
module Queries
class BaseQuery
attr_accessor :client
- delegate :query_range, :query, to: :client, prefix: true
+ delegate :query_range, :query, :label_values, :series, to: :client, prefix: true
def raw_memory_usage_query(environment_slug)
%{avg(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}) / 2^20}
diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb
new file mode 100644
index 00000000000..61926320e40
--- /dev/null
+++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb
@@ -0,0 +1,72 @@
+module Gitlab::Prometheus::Queries
+ class MatchedMetricsQuery < BaseQuery
+ MAX_QUERY_ITEMS = 40.freeze
+
+ def self.metrics
+ @metrics ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml')).map(&:deep_symbolize_keys)
+ end
+
+ def query
+ groups_data.map do |group, data|
+ {
+ group: group.name,
+ priority: group.priority,
+ active_metrics: data[:active_metrics],
+ metrics_missing_requirements: data[:metrics_missing_requirements]
+ }
+ end
+ end
+
+ def groups_data
+ metrics_series = metrics_with_series(Gitlab::Prometheus::MetricGroup.all)
+ lookup = active_series_lookup(metrics_series)
+
+ groups = {}
+
+ metrics_series.each do |metrics, series|
+ groups[metrics.group] ||= { active_metrics: 0, metrics_missing_requirements: 0 }
+ group = groups[metrics.group]
+
+ if series.all?(&lookup.method(:has_key?))
+ group[:active_metrics] += 1
+ else
+ group[:metrics_missing_requirements] += 1
+ end
+ group
+ end
+
+ groups
+ end
+
+ def active_series_lookup(metrics)
+ timeframe_start = 8.hours.ago
+ timeframe_end = Time.now
+
+ series = metrics.flat_map { |metrics, series| series }.uniq
+
+ lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series|
+ client_series(*batched_series, start: timeframe_start, stop: timeframe_end)
+ .select(&method(:has_matching_label))
+ .map { |series_info| [series_info['__name__'], true] }
+ end
+ lookup.to_h
+ end
+
+ def has_matching_label(series_info)
+ series_info.has_key?('environment')
+ end
+
+ def metrics_with_series(metric_groups)
+ label_values = client_label_values || []
+
+ metrics = metric_groups.flat_map do |group|
+ group.metrics.map do |metric|
+ matcher = Regexp.compile(metric.detect)
+ [metric, label_values.select(&matcher.method(:match))]
+ end
+ end
+
+ metrics.select { |metric, labels| labels&.any? }
+ end
+ end
+end
diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb
index 5b51a1779dd..f4ef4ff8ba4 100644
--- a/lib/gitlab/prometheus_client.rb
+++ b/lib/gitlab/prometheus_client.rb
@@ -29,6 +29,14 @@ module Gitlab
end
end
+ def label_values(name='__name__')
+ json_api_get("label/#{name}/values")
+ end
+
+ def series(*matches, start: 8.hours.ago, stop: Time.now)
+ json_api_get('series', 'match': matches, start: start.to_f, end: stop.to_f)
+ end
+
private
def json_api_get(type, args = {})