summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--app/models/project.rb2
-rw-r--r--app/models/prometheus_metric.rb96
-rw-r--r--changelogs/unreleased/alerts-for-built-in-metrics.yml5
-rw-r--r--config/prometheus/common_metrics.yml (renamed from config/prometheus/additional_metrics.yml)55
-rw-r--r--db/fixtures/development/99_common_metrics.rb3
-rw-r--r--db/fixtures/production/999_common_metrics.rb3
-rw-r--r--db/importers/common_metrics_importer.rb101
-rw-r--r--db/migrate/20180101160629_create_prometheus_metrics.rb16
-rw-r--r--db/migrate/20180831164905_add_common_to_prometheus_metrics.rb15
-rw-r--r--db/migrate/20180831164906_change_project_id_for_prometheus_metrics.rb11
-rw-r--r--db/migrate/20180831164907_add_index_on_default_prometheus_metrics.rb15
-rw-r--r--db/migrate/20180831164908_add_identifier_to_prometheus_metric.rb11
-rw-r--r--db/migrate/20180831164909_import_common_metrics.rb15
-rw-r--r--db/schema.rb21
-rw-r--r--doc/user/project/integrations/prometheus_library/metrics.md43
-rw-r--r--lib/gitlab/prometheus/additional_metrics_parser.rb2
-rw-r--r--lib/gitlab/prometheus/metric_group.rb5
-rw-r--r--spec/db/importers/common_metrics_importer_spec.rb119
-rw-r--r--spec/factories/prometheus_metrics.rb16
-rw-r--r--spec/lib/gitlab/prometheus/metric_group_spec.rb22
-rw-r--r--spec/migrations/import_common_metrics_spec.rb14
-rw-r--r--spec/models/prometheus_metric_spec.rb97
22 files changed, 665 insertions, 22 deletions
diff --git a/app/models/project.rb b/app/models/project.rb
index 67593c9b2fe..435ecc59bc4 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -232,6 +232,8 @@ class Project < ActiveRecord::Base
has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster'
has_many :cluster_ingresses, through: :clusters, source: :application_ingress, class_name: 'Clusters::Applications::Ingress'
+ has_many :prometheus_metrics
+
# Container repositories need to remove data from the container registry,
# which is not managed by the DB. Hence we're still using dependent: :destroy
# here.
diff --git a/app/models/prometheus_metric.rb b/app/models/prometheus_metric.rb
new file mode 100644
index 00000000000..87ea0762856
--- /dev/null
+++ b/app/models/prometheus_metric.rb
@@ -0,0 +1,96 @@
+class PrometheusMetric < ActiveRecord::Base
+ belongs_to :project, validate: true, inverse_of: :prometheus_metrics
+
+ enum group: {
+ # built-in groups
+ nginx_ingress: -1,
+ ha_proxy: -2,
+ aws_elb: -3,
+ nginx: -4,
+ kubernetes: -5,
+
+ # custom/user groups
+ business: 0,
+ response: 1,
+ system: 2
+ }
+
+ validates :title, presence: true
+ validates :query, presence: true
+ validates :group, presence: true
+ validates :y_label, presence: true
+ validates :unit, presence: true
+
+ validate :require_project
+
+ scope :common, -> { where(common: true) }
+
+ GROUP_TITLES = {
+ # built-in groups
+ nginx_ingress: _('Response metrics (NGINX Ingress)'),
+ ha_proxy: _('Response metrics (HA Proxy)'),
+ aws_elb: _('Response metrics (AWS ELB)'),
+ nginx: _('Response metrics (NGINX)'),
+ kubernetes: _('System metrics (Kubernetes)'),
+
+ # custom/user groups
+ business: _('Business metrics (Custom)'),
+ response: _('Response metrics (Custom)'),
+ system: _('System metrics (Custom)')
+ }.freeze
+
+ REQUIRED_METRICS = {
+ nginx_ingress: %w(nginx_upstream_responses_total nginx_upstream_response_msecs_avg),
+ ha_proxy: %w(haproxy_frontend_http_requests_total haproxy_frontend_http_responses_total),
+ aws_elb: %w(aws_elb_request_count_sum aws_elb_latency_average aws_elb_httpcode_backend_5_xx_sum),
+ nginx: %w(nginx_server_requests nginx_server_requestMsec),
+ kubernetes: %w(container_memory_usage_bytes container_cpu_usage_seconds_total)
+ }.freeze
+
+ def group_title
+ GROUP_TITLES[group.to_sym]
+ end
+
+ def required_metrics
+ (REQUIRED_METRICS[group.to_sym] || []).map(&:to_s)
+ end
+
+ def to_query_metric
+ Gitlab::Prometheus::Metric.new(id: id, title: title, required_metrics: required_metrics, weight: 0, y_label: y_label, queries: queries)
+ end
+
+ def queries
+ [
+ {
+ query_range: query,
+ unit: unit,
+ label: legend,
+ series: query_series
+ }
+ ]
+ end
+
+ def query_series
+ case legend
+ when 'Status Code'
+ {
+ label: 'status_code',
+ when: [
+ { value: '2xx', color: 'green' },
+ { value: '4xx', color: 'orange' },
+ { value: '5xx', color: 'red' }
+ ]
+ }
+ end
+ end
+
+ private
+
+ def require_project
+ if project
+ errors.add(:project, "cannot be set if this is common metric") if common?
+ else
+ errors.add(:project, "has to be set when this is project-specific metric") unless common?
+ end
+ end
+end
diff --git a/changelogs/unreleased/alerts-for-built-in-metrics.yml b/changelogs/unreleased/alerts-for-built-in-metrics.yml
new file mode 100644
index 00000000000..f88077eaca0
--- /dev/null
+++ b/changelogs/unreleased/alerts-for-built-in-metrics.yml
@@ -0,0 +1,5 @@
+---
+title: Import all common metrics into database
+merge_request:
+author:
+type: changed
diff --git a/config/prometheus/additional_metrics.yml b/config/prometheus/common_metrics.yml
index c994bad7865..61ade829cfa 100644
--- a/config/prometheus/additional_metrics.yml
+++ b/config/prometheus/common_metrics.yml
@@ -7,7 +7,8 @@
- nginx_upstream_responses_total
weight: 1
queries:
- - query_range: 'sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) by (status_code)'
+ - id: response_metrics_nginx_ingress_throughput_status_code
+ query_range: 'sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) by (status_code)'
unit: req / sec
label: Status Code
series:
@@ -25,7 +26,8 @@
- nginx_upstream_response_msecs_avg
weight: 1
queries:
- - query_range: 'avg(nginx_upstream_response_msecs_avg{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"})'
+ - id: response_metrics_nginx_ingress_latency_pod_average
+ query_range: 'avg(nginx_upstream_response_msecs_avg{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"})'
label: Pod average
unit: ms
- title: "HTTP Error Rate"
@@ -34,7 +36,8 @@
- nginx_upstream_responses_total
weight: 1
queries:
- - query_range: 'sum(rate(nginx_upstream_responses_total{status_code="5xx", upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) / sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) * 100'
+ - id: response_metrics_nginx_ingress_http_error_rate
+ query_range: 'sum(rate(nginx_upstream_responses_total{status_code="5xx", upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) / sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) * 100'
label: 5xx Errors
unit: "%"
- group: Response metrics (HA Proxy)
@@ -46,10 +49,12 @@
- haproxy_frontend_http_requests_total
weight: 1
queries:
- - query_range: 'sum(rate(haproxy_frontend_http_requests_total{%{environment_filter}}[2m])) by (code)'
+ - id: response_metrics_ha_proxy_throughput_status_code
+ query_range: 'sum(rate(haproxy_frontend_http_requests_total{%{environment_filter}}[2m])) by (code)'
unit: req / sec
+ label: Status Code
series:
- - label: code
+ - label: status_code
when:
- value: 2xx
color: green
@@ -63,7 +68,8 @@
- haproxy_frontend_http_responses_total
weight: 1
queries:
- - query_range: 'sum(rate(haproxy_frontend_http_responses_total{code="5xx",%{environment_filter}}[2m])) / sum(rate(haproxy_frontend_http_responses_total{%{environment_filter}}[2m]))'
+ - id: response_metrics_ha_proxy_http_error_rate
+ query_range: 'sum(rate(haproxy_frontend_http_responses_total{code="5xx",%{environment_filter}}[2m])) / sum(rate(haproxy_frontend_http_responses_total{%{environment_filter}}[2m]))'
label: HTTP Errors
unit: "%"
- group: Response metrics (AWS ELB)
@@ -75,7 +81,8 @@
- aws_elb_request_count_sum
weight: 1
queries:
- - query_range: 'sum(aws_elb_request_count_sum{%{environment_filter}}) / 60'
+ - id: response_metrics_aws_elb_throughput_requests
+ query_range: 'sum(aws_elb_request_count_sum{%{environment_filter}}) / 60'
label: Total
unit: req / sec
- title: "Latency"
@@ -84,7 +91,8 @@
- aws_elb_latency_average
weight: 1
queries:
- - query_range: 'avg(aws_elb_latency_average{%{environment_filter}}) * 1000'
+ - id: response_metrics_aws_elb_latency_average
+ query_range: 'avg(aws_elb_latency_average{%{environment_filter}}) * 1000'
label: Average
unit: ms
- title: "HTTP Error Rate"
@@ -94,7 +102,8 @@
- aws_elb_httpcode_backend_5_xx_sum
weight: 1
queries:
- - query_range: 'sum(aws_elb_httpcode_backend_5_xx_sum{%{environment_filter}}) / sum(aws_elb_request_count_sum{%{environment_filter}})'
+ - id: response_metrics_aws_elb_http_error_rate
+ query_range: 'sum(aws_elb_httpcode_backend_5_xx_sum{%{environment_filter}}) / sum(aws_elb_request_count_sum{%{environment_filter}})'
label: HTTP Errors
unit: "%"
- group: Response metrics (NGINX)
@@ -106,7 +115,8 @@
- nginx_server_requests
weight: 1
queries:
- - query_range: 'sum(rate(nginx_server_requests{server_zone!="*", server_zone!="_", %{environment_filter}}[2m])) by (code)'
+ - id: response_metrics_nginx_throughput_status_code
+ query_range: 'sum(rate(nginx_server_requests{server_zone!="*", server_zone!="_", %{environment_filter}}[2m])) by (code)'
unit: req / sec
label: Status Code
series:
@@ -124,7 +134,8 @@
- nginx_server_requestMsec
weight: 1
queries:
- - query_range: 'avg(nginx_server_requestMsec{%{environment_filter}})'
+ - id: response_metrics_nginx_latency
+ query_range: 'avg(nginx_server_requestMsec{%{environment_filter}})'
label: Upstream
unit: ms
- title: "HTTP Error Rate"
@@ -133,7 +144,8 @@
- nginx_server_requests
weight: 1
queries:
- - query_range: 'sum(rate(nginx_server_requests{code="5xx", %{environment_filter}}[2m]))'
+ - id: response_metrics_nginx_http_error_rate
+ query_range: 'sum(rate(nginx_server_requests{code="5xx", %{environment_filter}}[2m]))'
label: HTTP Errors
unit: "errors / sec"
- group: System metrics (Kubernetes)
@@ -145,7 +157,8 @@
- container_memory_usage_bytes
weight: 4
queries:
- - query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) /1024/1024/1024'
+ - id: system_metrics_kubernetes_container_memory_total
+ query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) /1024/1024/1024'
label: Total
unit: GB
- title: "Core Usage (Total)"
@@ -154,7 +167,8 @@
- container_cpu_usage_seconds_total
weight: 3
queries:
- - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job)'
+ - id: system_metrics_kubernetes_container_cores_total
+ query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job)'
label: Total
unit: "cores"
- title: "Memory Usage (Pod average)"
@@ -163,7 +177,8 @@
- container_memory_usage_bytes
weight: 2
queries:
- - query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024'
+ - id: system_metrics_kubernetes_container_memory_average
+ query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024'
label: Pod average
unit: MB
- title: "Core Usage (Pod average)"
@@ -172,6 +187,12 @@
- container_cpu_usage_seconds_total
weight: 1
queries:
- - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (pod_name))'
+ - id: system_metrics_kubernetes_container_core_usage
+ query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (pod_name))'
label: Pod average
- unit: "cores" \ No newline at end of file
+ unit: "cores"
+ - id: system_metrics_kubernetes_container_core_usage_canary
+ query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-canary-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-canary-(.*)",namespace="%{kube_namespace}"}[15m])) by (pod_name))'
+ label: Pod average
+ unit: "cores"
+ track: canary
diff --git a/db/fixtures/development/99_common_metrics.rb b/db/fixtures/development/99_common_metrics.rb
new file mode 100644
index 00000000000..89424905bd4
--- /dev/null
+++ b/db/fixtures/development/99_common_metrics.rb
@@ -0,0 +1,3 @@
+require_relative '../importers/common_metrics_importer.rb'
+
+::Importers::CommonMetricsImporter.new.execute
diff --git a/db/fixtures/production/999_common_metrics.rb b/db/fixtures/production/999_common_metrics.rb
new file mode 100644
index 00000000000..89424905bd4
--- /dev/null
+++ b/db/fixtures/production/999_common_metrics.rb
@@ -0,0 +1,3 @@
+require_relative '../importers/common_metrics_importer.rb'
+
+::Importers::CommonMetricsImporter.new.execute
diff --git a/db/importers/common_metrics_importer.rb b/db/importers/common_metrics_importer.rb
new file mode 100644
index 00000000000..9e6d2dd0518
--- /dev/null
+++ b/db/importers/common_metrics_importer.rb
@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Importers
+ class PrometheusMetric < ActiveRecord::Base
+ enum group: {
+ # built-in groups
+ nginx_ingress: -1,
+ ha_proxy: -2,
+ aws_elb: -3,
+ nginx: -4,
+ kubernetes: -5,
+
+ # custom groups
+ business: 0,
+ response: 1,
+ system: 2,
+ }
+
+ scope :common, -> { where(common: true) }
+
+ GROUP_TITLES = {
+ business: _('Business metrics (Custom)'),
+ response: _('Response metrics (Custom)'),
+ system: _('System metrics (Custom)'),
+ nginx_ingress: _('Response metrics (NGINX Ingress)'),
+ ha_proxy: _('Response metrics (HA Proxy)'),
+ aws_elb: _('Response metrics (AWS ELB)'),
+ nginx: _('Response metrics (NGINX)'),
+ kubernetes: _('System metrics (Kubernetes)')
+ }.freeze
+ end
+
+ class CommonMetricsImporter
+ MissingQueryId = Class.new(StandardError)
+
+ attr_reader :content
+
+ def initialize(file = 'config/prometheus/common_metrics.yml')
+ @content = YAML.load_file(file)
+ end
+
+ def execute
+ process_content do |id, attributes|
+ find_or_build_metric!(id)
+ .update!(**attributes)
+ end
+ end
+
+ private
+
+ def process_content(&blk)
+ content.map do |group|
+ process_group(group, &blk)
+ end
+ end
+
+ def process_group(group, &blk)
+ attributes = {
+ group: find_group_title_key(group['group'])
+ }
+
+ group['metrics'].map do |metric|
+ process_metric(metric, attributes, &blk)
+ end
+ end
+
+ def process_metric(metric, attributes, &blk)
+ attributes = attributes.merge(
+ title: metric['title'],
+ y_label: metric['y_label'])
+
+ metric['queries'].map do |query|
+ process_metric_query(query, attributes, &blk)
+ end
+ end
+
+ def process_metric_query(query, attributes, &blk)
+ attributes = attributes.merge(
+ legend: query['label'],
+ query: query['query_range'],
+ unit: query['unit'])
+
+ blk.call(query['id'], attributes)
+ end
+
+ def find_or_build_metric!(id)
+ raise MissingQueryId unless id
+
+ PrometheusMetric.common.find_by(identifier: id) ||
+ PrometheusMetric.new(common: true, identifier: id)
+ end
+
+ def find_group_title_key(title)
+ PrometheusMetric.groups[find_group_title(title)]
+ end
+
+ def find_group_title(title)
+ PrometheusMetric::GROUP_TITLES.invert[title]
+ end
+ end
+end
diff --git a/db/migrate/20180101160629_create_prometheus_metrics.rb b/db/migrate/20180101160629_create_prometheus_metrics.rb
new file mode 100644
index 00000000000..da6371084cb
--- /dev/null
+++ b/db/migrate/20180101160629_create_prometheus_metrics.rb
@@ -0,0 +1,16 @@
+class CreatePrometheusMetrics < ActiveRecord::Migration
+ DOWNTIME = false
+
+ def change
+ create_table :prometheus_metrics do |t|
+ t.references :project, index: true, foreign_key: { on_delete: :cascade }, null: false
+ t.string :title, null: false
+ t.string :query, null: false
+ t.string :y_label
+ t.string :unit
+ t.string :legend
+ t.integer :group, null: false, index: true
+ t.timestamps_with_timezone null: false
+ end
+ end
+end
diff --git a/db/migrate/20180831164905_add_common_to_prometheus_metrics.rb b/db/migrate/20180831164905_add_common_to_prometheus_metrics.rb
new file mode 100644
index 00000000000..43e36d2047c
--- /dev/null
+++ b/db/migrate/20180831164905_add_common_to_prometheus_metrics.rb
@@ -0,0 +1,15 @@
+class AddCommonToPrometheusMetrics < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ disable_ddl_transaction!
+
+ def up
+ add_column_with_default(:prometheus_metrics, :common, :boolean, default: false)
+ end
+
+ def down
+ remove_column(:prometheus_metrics, :common)
+ end
+end
diff --git a/db/migrate/20180831164906_change_project_id_for_prometheus_metrics.rb b/db/migrate/20180831164906_change_project_id_for_prometheus_metrics.rb
new file mode 100644
index 00000000000..c9aae868e52
--- /dev/null
+++ b/db/migrate/20180831164906_change_project_id_for_prometheus_metrics.rb
@@ -0,0 +1,11 @@
+class ChangeProjectIdForPrometheusMetrics < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ disable_ddl_transaction!
+
+ def change
+ change_column_null :prometheus_metrics, :project_id, true
+ end
+end
diff --git a/db/migrate/20180831164907_add_index_on_default_prometheus_metrics.rb b/db/migrate/20180831164907_add_index_on_default_prometheus_metrics.rb
new file mode 100644
index 00000000000..2e2014ccdce
--- /dev/null
+++ b/db/migrate/20180831164907_add_index_on_default_prometheus_metrics.rb
@@ -0,0 +1,15 @@
+class AddIndexOnDefaultPrometheusMetrics < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ disable_ddl_transaction!
+
+ def up
+ add_concurrent_index :prometheus_metrics, :common
+ end
+
+ def down
+ remove_concurrent_index :prometheus_metrics, :project_id
+ end
+end
diff --git a/db/migrate/20180831164908_add_identifier_to_prometheus_metric.rb b/db/migrate/20180831164908_add_identifier_to_prometheus_metric.rb
new file mode 100644
index 00000000000..d5f33a293cc
--- /dev/null
+++ b/db/migrate/20180831164908_add_identifier_to_prometheus_metric.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class AddIdentifierToPrometheusMetric < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ def change
+ add_column :prometheus_metrics, :identifier, :string, unique: true
+ end
+end
diff --git a/db/migrate/20180831164909_import_common_metrics.rb b/db/migrate/20180831164909_import_common_metrics.rb
new file mode 100644
index 00000000000..adb35a9b211
--- /dev/null
+++ b/db/migrate/20180831164909_import_common_metrics.rb
@@ -0,0 +1,15 @@
+class ImportCommonMetrics < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ require_relative '../importers/common_metrics_importer.rb'
+
+ DOWNTIME = false
+
+ def up
+ Importers::CommonMetricsImporter.new.execute
+ end
+
+ def down
+ # no-op
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 02e545bec7d..c3d4eccd826 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20180826111825) do
+ActiveRecord::Schema.define(version: 20180831164909) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@@ -1696,6 +1696,24 @@ ActiveRecord::Schema.define(version: 20180826111825) do
add_index "projects", ["star_count"], name: "index_projects_on_star_count", using: :btree
add_index "projects", ["visibility_level"], name: "index_projects_on_visibility_level", using: :btree
+ create_table "prometheus_metrics", force: :cascade do |t|
+ t.integer "project_id"
+ t.string "title", null: false
+ t.string "query", null: false
+ t.string "y_label"
+ t.string "unit"
+ t.string "legend"
+ t.integer "group", null: false
+ t.datetime_with_timezone "created_at", null: false
+ t.datetime_with_timezone "updated_at", null: false
+ t.boolean "common", default: false, null: false
+ t.string "identifier"
+ end
+
+ add_index "prometheus_metrics", ["common"], name: "index_prometheus_metrics_on_common", using: :btree
+ add_index "prometheus_metrics", ["group"], name: "index_prometheus_metrics_on_group", using: :btree
+ add_index "prometheus_metrics", ["project_id"], name: "index_prometheus_metrics_on_project_id", using: :btree
+
create_table "protected_branch_merge_access_levels", force: :cascade do |t|
t.integer "protected_branch_id", null: false
t.integer "access_level", default: 40, null: false
@@ -2375,6 +2393,7 @@ ActiveRecord::Schema.define(version: 20180826111825) do
add_foreign_key "project_import_data", "projects", name: "fk_ffb9ee3a10", on_delete: :cascade
add_foreign_key "project_mirror_data", "projects", on_delete: :cascade
add_foreign_key "project_statistics", "projects", on_delete: :cascade
+ add_foreign_key "prometheus_metrics", "projects", on_delete: :cascade
add_foreign_key "protected_branch_merge_access_levels", "protected_branches", name: "fk_8a3072ccb3", on_delete: :cascade
add_foreign_key "protected_branch_push_access_levels", "protected_branches", name: "fk_9ffc86a3d9", on_delete: :cascade
add_foreign_key "protected_branches", "projects", name: "fk_7a9c6d93e7", on_delete: :cascade
diff --git a/doc/user/project/integrations/prometheus_library/metrics.md b/doc/user/project/integrations/prometheus_library/metrics.md
index 96a22316265..1e0f9d4e249 100644
--- a/doc/user/project/integrations/prometheus_library/metrics.md
+++ b/doc/user/project/integrations/prometheus_library/metrics.md
@@ -20,6 +20,45 @@ GitLab uses the defined queries and fills in the environment specific variables.
## Adding to the library
-We strive to support the 2-4 most important metrics for each common system service that supports Prometheus. If you are looking for support for a particular exporter which has not yet been added to the library, additions can be made [to the `additional_metrics.yml`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/prometheus/additional_metrics.yml) file.
+We strive to support the 2-4 most important metrics for each common system service that supports Prometheus. If you are looking for support for a particular exporter which has not yet been added to the library, additions can be made [to the `common_metrics.yml`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/prometheus/common_metrics.yml) file.
-> Note: The library is only for monitoring public, common, system services which all customers can benefit from. Support for monitoring [customer proprietary metrics](https://gitlab.com/gitlab-org/gitlab-ee/issues/2273) will be added in a subsequent release.
+### Query identifier
+
+The requirement for adding metrics is to have each query to have unique identifier.
+Identifier is used to update the metric later when changed.
+
+```yaml
+- group: Response metrics (NGINX Ingress)
+ metrics:
+ - title: "Throughput"
+ y_label: "Requests / Sec"
+ queries:
+ - id: response_metrics_nginx_ingress_throughput_status_code
+ query_range: 'sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) by (status_code)'
+ unit: req / sec
+ label: Status Code
+```
+
+### Update existing metrics
+
+After you add or change existing _common_ metric you have to create a new database migration that will query and update all existing metrics.
+
+**Note: If a query metric (which is identified by `id:`) is removed it will not be removed from database by default.**
+**You might want to add additional database migration that makes a decision what to do with removed one.**
+**For example: you might be interested in migrating all dependent data to a different metric.**
+
+```ruby
+class ImportCommonMetrics < ActiveRecord::Migration
+ require_relative '../importers/common_metrics_importer.rb'
+
+ DOWNTIME = false
+
+ def up
+ Importers::CommonMetricsImporter.new.execute
+ end
+
+ def down
+ # no-op
+ end
+end
+```
diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb
index bb1172f82a1..a240d090074 100644
--- a/lib/gitlab/prometheus/additional_metrics_parser.rb
+++ b/lib/gitlab/prometheus/additional_metrics_parser.rb
@@ -5,7 +5,7 @@ module Gitlab
MUTEX = Mutex.new
extend self
- def load_groups_from_yaml(file_name = 'additional_metrics.yml')
+ def load_groups_from_yaml(file_name)
yaml_metrics_raw(file_name).map(&method(:group_from_entry))
end
diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb
index e91c6fb2e27..d696a8fc00c 100644
--- a/lib/gitlab/prometheus/metric_group.rb
+++ b/lib/gitlab/prometheus/metric_group.rb
@@ -4,10 +4,13 @@ module Gitlab
include ActiveModel::Model
attr_accessor :name, :priority, :metrics
+
validates :name, :priority, :metrics, presence: true
def self.common_metrics
- AdditionalMetricsParser.load_groups_from_yaml
+ ::PrometheusMetric.common.group_by(&:group_title).map do |name, metrics|
+ MetricGroup.new(name: name, priority: 0, metrics: metrics.map(&:to_query_metric))
+ end
end
# EE only
diff --git a/spec/db/importers/common_metrics_importer_spec.rb b/spec/db/importers/common_metrics_importer_spec.rb
new file mode 100644
index 00000000000..81eba6377c3
--- /dev/null
+++ b/spec/db/importers/common_metrics_importer_spec.rb
@@ -0,0 +1,119 @@
+require 'rails_helper'
+require Rails.root.join("db", "importers", "common_metrics_importer.rb")
+
+describe Importers::PrometheusMetric do
+ it 'group enum equals ::PrometheusMetric' do
+ expect(described_class.groups).to eq(::PrometheusMetric.groups)
+ end
+
+ it 'GROUP_TITLES equals ::PrometheusMetric' do
+ expect(described_class::GROUP_TITLES).to eq(::PrometheusMetric::GROUP_TITLES)
+ end
+end
+
+describe Importers::CommonMetricsImporter do
+ subject { described_class.new }
+
+ context "does import common_metrics.yml" do
+ let(:groups) { subject.content }
+ let(:metrics) { groups.map { |group| group['metrics'] }.flatten }
+ let(:queries) { metrics.map { |group| group['queries'] }.flatten }
+ let(:query_ids) { queries.map { |query| query['id'] } }
+
+ before do
+ subject.execute
+ end
+
+ it "has the same amount of groups" do
+ expect(PrometheusMetric.common.group(:group).count.count).to eq(groups.count)
+ end
+
+ it "has the same amount of metrics" do
+ expect(PrometheusMetric.common.group(:group, :title).count.count).to eq(metrics.count)
+ end
+
+ it "has the same amount of queries" do
+ expect(PrometheusMetric.common.count).to eq(queries.count)
+ end
+
+ it "does not have duplicate IDs" do
+ expect(query_ids).to eq(query_ids.uniq)
+ end
+
+ it "imports all IDs" do
+ expect(PrometheusMetric.common.pluck(:identifier)).to eq(query_ids)
+ end
+ end
+
+ context 'does import properly all fields' do
+ let(:query_identifier) { 'response-metric' }
+ let(:group) do
+ {
+ group: 'Response metrics (NGINX Ingress)',
+ metrics: [{
+ title: "Throughput",
+ y_label: "Requests / Sec",
+ queries: [{
+ id: query_identifier,
+ query_range: 'my-query',
+ unit: 'my-unit',
+ label: 'status code'
+ }]
+ }]
+ }
+ end
+
+ before do
+ expect(subject).to receive(:content) { [group.deep_stringify_keys] }
+ end
+
+ shared_examples 'stores metric' do
+ let(:metric) { PrometheusMetric.find_by(identifier: query_identifier) }
+
+ it 'with all data' do
+ expect(metric.group).to eq('nginx_ingress')
+ expect(metric.title).to eq('Throughput')
+ expect(metric.y_label).to eq('Requests / Sec')
+ expect(metric.unit).to eq('my-unit')
+ expect(metric.legend).to eq('status code')
+ expect(metric.query).to eq('my-query')
+ end
+ end
+
+ context 'if ID is missing' do
+ let(:query_identifier) { }
+
+ it 'raises exception' do
+ expect { subject.execute }.to raise_error(described_class::MissingQueryId)
+ end
+ end
+
+ context 'for existing common metric with different ID' do
+ let!(:existing_metric) { create(:prometheus_metric, :common, identifier: 'my-existing-metric') }
+
+ before do
+ subject.execute
+ end
+
+ it_behaves_like 'stores metric' do
+ it 'and existing metric is not changed' do
+ expect(metric).not_to eq(existing_metric)
+ end
+ end
+ end
+
+ context 'when metric with ID exists ' do
+ let!(:existing_metric) { create(:prometheus_metric, :common, identifier: 'response-metric') }
+
+ before do
+ subject.execute
+ end
+
+ it_behaves_like 'stores metric' do
+ it 'and existing metric is changed' do
+ expect(metric).to eq(existing_metric)
+ end
+ end
+ end
+ end
+end \ No newline at end of file
diff --git a/spec/factories/prometheus_metrics.rb b/spec/factories/prometheus_metrics.rb
new file mode 100644
index 00000000000..0ab237d6b3e
--- /dev/null
+++ b/spec/factories/prometheus_metrics.rb
@@ -0,0 +1,16 @@
+FactoryBot.define do
+ factory :prometheus_metric, class: PrometheusMetric do
+ title 'title'
+ query 'avg(metric)'
+ y_label 'y_label'
+ unit 'm/s'
+ group :business
+ project
+ legend 'legend'
+
+ trait :common do
+ common true
+ project nil
+ end
+ end
+end
diff --git a/spec/lib/gitlab/prometheus/metric_group_spec.rb b/spec/lib/gitlab/prometheus/metric_group_spec.rb
new file mode 100644
index 00000000000..fa36b83f575
--- /dev/null
+++ b/spec/lib/gitlab/prometheus/metric_group_spec.rb
@@ -0,0 +1,22 @@
+require 'rails_helper'
+
+describe Gitlab::Prometheus::MetricGroup do
+ describe '.common_metrics' do
+ set(:project_metric) { create(:prometheus_metric) }
+ set(:common_metric_group_a) { create(:prometheus_metric, :common, group: :aws_elb) }
+ set(:common_metric_group_b_q1) { create(:prometheus_metric, :common, group: :kubernetes) }
+ set(:common_metric_group_b_q2) { create(:prometheus_metric, :common, group: :kubernetes) }
+
+ subject { described_class.common_metrics }
+
+ it 'returns exactly two groups' do
+ expect(subject.map(&:name)).to contain_exactly('Response metrics (AWS ELB)', 'System metrics (Kubernetes)')
+ end
+
+ it 'returns exactly three metric queries' do
+ expect(subject.map(&:metrics).flatten.map(&:queries)).to contain_exactly(
+ common_metric_group_a.queries, common_metric_group_b_q1.queries,
+ common_metric_group_b_q2.queries)
+ end
+ end
+end
diff --git a/spec/migrations/import_common_metrics_spec.rb b/spec/migrations/import_common_metrics_spec.rb
new file mode 100644
index 00000000000..83ecb39a6ee
--- /dev/null
+++ b/spec/migrations/import_common_metrics_spec.rb
@@ -0,0 +1,14 @@
+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20180831164909_import_common_metrics.rb')
+
+describe ImportCommonMetrics, :migration do
+ describe '#up' do
+ it "imports all prometheus metrics" do
+ expect(PrometheusMetric.common).to be_empty
+
+ migrate!
+
+ expect(PrometheusMetric.common).not_to be_empty
+ end
+ end
+end
diff --git a/spec/models/prometheus_metric_spec.rb b/spec/models/prometheus_metric_spec.rb
new file mode 100644
index 00000000000..7469832898a
--- /dev/null
+++ b/spec/models/prometheus_metric_spec.rb
@@ -0,0 +1,97 @@
+require 'spec_helper'
+
+describe PrometheusMetric do
+ subject { build(:prometheus_metric) }
+ let(:other_project) { build(:project) }
+
+ it { is_expected.to belong_to(:project) }
+ it { is_expected.to validate_presence_of(:title) }
+ it { is_expected.to validate_presence_of(:query) }
+ it { is_expected.to validate_presence_of(:group) }
+
+ describe 'common metrics' do
+ using RSpec::Parameterized::TableSyntax
+
+ where(:common, :project, :result) do
+ false | other_project | true
+ false | nil | false
+ true | other_project | false
+ true | nil | true
+ end
+
+ with_them do
+ before do
+ subject.common = common
+ subject.project = project
+ end
+
+ it { expect(subject.valid?).to eq(result) }
+ end
+ end
+
+ describe '#query_series' do
+ using RSpec::Parameterized::TableSyntax
+
+ where(:legend, :type) do
+ 'Some other legend' | NilClass
+ 'Status Code' | Hash
+ end
+
+ with_them do
+ before do
+ subject.legend = legend
+ end
+
+ it { expect(subject.query_series).to be_a(type) }
+ end
+ end
+
+ describe '#group_title' do
+ shared_examples 'group_title' do |group, title|
+ subject { build(:prometheus_metric, group: group).group_title }
+
+ it "returns text #{title} for group #{group}" do
+ expect(subject).to eq(title)
+ end
+ end
+
+ it_behaves_like 'group_title', :business, 'Business metrics (Custom)'
+ it_behaves_like 'group_title', :response, 'Response metrics (Custom)'
+ it_behaves_like 'group_title', :system, 'System metrics (Custom)'
+ end
+
+ describe '#to_query_metric' do
+ it 'converts to queryable metric object' do
+ expect(subject.to_query_metric).to be_instance_of(Gitlab::Prometheus::Metric)
+ end
+
+ it 'queryable metric object has title' do
+ expect(subject.to_query_metric.title).to eq(subject.title)
+ end
+
+ it 'queryable metric object has y_label' do
+ expect(subject.to_query_metric.y_label).to eq(subject.y_label)
+ end
+
+ it 'queryable metric has no required_metric' do
+ expect(subject.to_query_metric.required_metrics).to eq([])
+ end
+
+ it 'queryable metric has weight 0' do
+ expect(subject.to_query_metric.weight).to eq(0)
+ end
+
+ it 'queryable metrics has query description' do
+ queries = [
+ {
+ query_range: subject.query,
+ unit: subject.unit,
+ label: subject.legend,
+ series: nil
+ }
+ ]
+
+ expect(subject.to_query_metric.queries).to eq(queries)
+ end
+ end
+end