summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--app/models/prometheus_metric.rb65
-rw-r--r--app/models/prometheus_metric_enums.rb70
-rw-r--r--db/fixtures/development/99_common_metrics.rb4
-rw-r--r--db/fixtures/production/999_common_metrics.rb4
-rw-r--r--db/importers/common_metrics_importer.rb105
-rw-r--r--db/migrate/20180831164910_import_common_metrics.rb4
-rw-r--r--db/migrate/20181006004100_import_common_metrics_nginx_vts.rb4
-rw-r--r--db/migrate/20190326164045_import_common_metrics_knative.rb4
-rw-r--r--db/migrate/20190408163745_prometheus_knative05_fix.rb4
-rw-r--r--doc/development/prometheus_metrics.md4
-rw-r--r--lib/gitlab/database_importers/common_metrics.rb8
-rw-r--r--lib/gitlab/database_importers/common_metrics/importer.rb78
-rw-r--r--lib/gitlab/database_importers/common_metrics/prometheus_metric.rb12
-rw-r--r--lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb40
-rw-r--r--spec/lib/gitlab/database_importers/common_metrics/importer_spec.rb (renamed from spec/db/importers/common_metrics_importer_spec.rb)21
-rw-r--r--spec/lib/gitlab/database_importers/common_metrics/prometheus_metric_spec.rb16
16 files changed, 235 insertions, 208 deletions
diff --git a/app/models/prometheus_metric.rb b/app/models/prometheus_metric.rb
index 62090444f79..b8e7673dcf5 100644
--- a/app/models/prometheus_metric.rb
+++ b/app/models/prometheus_metric.rb
@@ -3,68 +3,7 @@
class PrometheusMetric < ApplicationRecord
belongs_to :project, validate: true, inverse_of: :prometheus_metrics
- enum group: {
- # built-in groups
- nginx_ingress_vts: -1,
- ha_proxy: -2,
- aws_elb: -3,
- nginx: -4,
- kubernetes: -5,
- nginx_ingress: -6,
-
- # custom/user groups
- business: 0,
- response: 1,
- system: 2
- }
-
- GROUP_DETAILS = {
- # built-in groups
- nginx_ingress_vts: {
- group_title: _('Response metrics (NGINX Ingress VTS)'),
- required_metrics: %w(nginx_upstream_responses_total nginx_upstream_response_msecs_avg),
- priority: 10
- }.freeze,
- nginx_ingress: {
- group_title: _('Response metrics (NGINX Ingress)'),
- required_metrics: %w(nginx_ingress_controller_requests nginx_ingress_controller_ingress_upstream_latency_seconds_sum),
- priority: 10
- }.freeze,
- ha_proxy: {
- group_title: _('Response metrics (HA Proxy)'),
- required_metrics: %w(haproxy_frontend_http_requests_total haproxy_frontend_http_responses_total),
- priority: 10
- }.freeze,
- aws_elb: {
- group_title: _('Response metrics (AWS ELB)'),
- required_metrics: %w(aws_elb_request_count_sum aws_elb_latency_average aws_elb_httpcode_backend_5_xx_sum),
- priority: 10
- }.freeze,
- nginx: {
- group_title: _('Response metrics (NGINX)'),
- required_metrics: %w(nginx_server_requests nginx_server_requestMsec),
- priority: 10
- }.freeze,
- kubernetes: {
- group_title: _('System metrics (Kubernetes)'),
- required_metrics: %w(container_memory_usage_bytes container_cpu_usage_seconds_total),
- priority: 5
- }.freeze,
-
- # custom/user groups
- business: {
- group_title: _('Business metrics (Custom)'),
- priority: 0
- }.freeze,
- response: {
- group_title: _('Response metrics (Custom)'),
- priority: -5
- }.freeze,
- system: {
- group_title: _('System metrics (Custom)'),
- priority: -10
- }.freeze
- }.freeze
+ enum group: PrometheusMetricEnums.groups
validates :title, presence: true
validates :query, presence: true
@@ -121,6 +60,6 @@ class PrometheusMetric < ApplicationRecord
private
def group_details(group)
- GROUP_DETAILS.fetch(group.to_sym)
+ PrometheusMetricEnums.group_details.fetch(group.to_sym)
end
end
diff --git a/app/models/prometheus_metric_enums.rb b/app/models/prometheus_metric_enums.rb
new file mode 100644
index 00000000000..6cb22cc69cd
--- /dev/null
+++ b/app/models/prometheus_metric_enums.rb
@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module PrometheusMetricEnums
+ def self.groups
+ {
+ # built-in groups
+ nginx_ingress_vts: -1,
+ ha_proxy: -2,
+ aws_elb: -3,
+ nginx: -4,
+ kubernetes: -5,
+ nginx_ingress: -6,
+
+ # custom/user groups
+ business: 0,
+ response: 1,
+ system: 2
+ }
+ end
+
+ def self.group_details
+ {
+ # built-in groups
+ nginx_ingress_vts: {
+ group_title: _('Response metrics (NGINX Ingress VTS)'),
+ required_metrics: %w(nginx_upstream_responses_total nginx_upstream_response_msecs_avg),
+ priority: 10
+ }.freeze,
+ nginx_ingress: {
+ group_title: _('Response metrics (NGINX Ingress)'),
+ required_metrics: %w(nginx_ingress_controller_requests nginx_ingress_controller_ingress_upstream_latency_seconds_sum),
+ priority: 10
+ }.freeze,
+ ha_proxy: {
+ group_title: _('Response metrics (HA Proxy)'),
+ required_metrics: %w(haproxy_frontend_http_requests_total haproxy_frontend_http_responses_total),
+ priority: 10
+ }.freeze,
+ aws_elb: {
+ group_title: _('Response metrics (AWS ELB)'),
+ required_metrics: %w(aws_elb_request_count_sum aws_elb_latency_average aws_elb_httpcode_backend_5_xx_sum),
+ priority: 10
+ }.freeze,
+ nginx: {
+ group_title: _('Response metrics (NGINX)'),
+ required_metrics: %w(nginx_server_requests nginx_server_requestMsec),
+ priority: 10
+ }.freeze,
+ kubernetes: {
+ group_title: _('System metrics (Kubernetes)'),
+ required_metrics: %w(container_memory_usage_bytes container_cpu_usage_seconds_total),
+ priority: 5
+ }.freeze,
+
+ # custom/user groups
+ business: {
+ group_title: _('Business metrics (Custom)'),
+ priority: 0
+ }.freeze,
+ response: {
+ group_title: _('Response metrics (Custom)'),
+ priority: -5
+ }.freeze,
+ system: {
+ group_title: _('System metrics (Custom)'),
+ priority: -10
+ }.freeze
+ }.freeze
+ end
+end
diff --git a/db/fixtures/development/99_common_metrics.rb b/db/fixtures/development/99_common_metrics.rb
index 1f39c0ce5a0..d52f78ea536 100644
--- a/db/fixtures/development/99_common_metrics.rb
+++ b/db/fixtures/development/99_common_metrics.rb
@@ -1,5 +1,3 @@
# frozen_string_literal: true
-require Rails.root.join('db/importers/common_metrics_importer.rb')
-
-::Importers::CommonMetricsImporter.new.execute
+::Gitlab::DatabaseImporters::CommonMetrics::Importer.new.execute
diff --git a/db/fixtures/production/999_common_metrics.rb b/db/fixtures/production/999_common_metrics.rb
index 1f39c0ce5a0..d52f78ea536 100644
--- a/db/fixtures/production/999_common_metrics.rb
+++ b/db/fixtures/production/999_common_metrics.rb
@@ -1,5 +1,3 @@
# frozen_string_literal: true
-require Rails.root.join('db/importers/common_metrics_importer.rb')
-
-::Importers::CommonMetricsImporter.new.execute
+::Gitlab::DatabaseImporters::CommonMetrics::Importer.new.execute
diff --git a/db/importers/common_metrics_importer.rb b/db/importers/common_metrics_importer.rb
deleted file mode 100644
index 195bde8f34a..00000000000
--- a/db/importers/common_metrics_importer.rb
+++ /dev/null
@@ -1,105 +0,0 @@
-# frozen_string_literal: true
-
-module Importers
- class PrometheusMetric < ActiveRecord::Base
- enum group: {
- # built-in groups
- nginx_ingress_vts: -1,
- ha_proxy: -2,
- aws_elb: -3,
- nginx: -4,
- kubernetes: -5,
- nginx_ingress: -6,
-
- # custom groups
- business: 0,
- response: 1,
- system: 2
- }
-
- scope :common, -> { where(common: true) }
-
- GROUP_TITLES = {
- business: _('Business metrics (Custom)'),
- response: _('Response metrics (Custom)'),
- system: _('System metrics (Custom)'),
- nginx_ingress_vts: _('Response metrics (NGINX Ingress VTS)'),
- nginx_ingress: _('Response metrics (NGINX Ingress)'),
- ha_proxy: _('Response metrics (HA Proxy)'),
- aws_elb: _('Response metrics (AWS ELB)'),
- nginx: _('Response metrics (NGINX)'),
- kubernetes: _('System metrics (Kubernetes)')
- }.freeze
- end
-
- class CommonMetricsImporter
- MissingQueryId = Class.new(StandardError)
-
- attr_reader :content
-
- def initialize(filename = 'common_metrics.yml')
- @content = YAML.load_file(Rails.root.join('config', 'prometheus', filename))
- end
-
- def execute
- PrometheusMetric.reset_column_information
-
- process_content do |id, attributes|
- find_or_build_metric!(id)
- .update!(**attributes)
- end
- end
-
- private
-
- def process_content(&blk)
- content['panel_groups'].map do |group|
- process_group(group, &blk)
- end
- end
-
- def process_group(group, &blk)
- attributes = {
- group: find_group_title_key(group['group'])
- }
-
- group['panels'].map do |panel|
- process_panel(panel, attributes, &blk)
- end
- end
-
- def process_panel(panel, attributes, &blk)
- attributes = attributes.merge(
- title: panel['title'],
- y_label: panel['y_label'])
-
- panel['metrics'].map do |metric_details|
- process_metric_details(metric_details, attributes, &blk)
- end
- end
-
- def process_metric_details(metric_details, attributes, &blk)
- attributes = attributes.merge(
- legend: metric_details['label'],
- query: metric_details['query_range'],
- unit: metric_details['unit'])
-
- yield(metric_details['id'], attributes)
- end
-
- def find_or_build_metric!(id)
- raise MissingQueryId unless id
-
- PrometheusMetric.common.find_by(identifier: id) ||
- PrometheusMetric.new(common: true, identifier: id)
- end
-
- def find_group_title_key(title)
- PrometheusMetric.groups[find_group_title(title)]
- end
-
- def find_group_title(title)
- PrometheusMetric::GROUP_TITLES.invert[title]
- end
- end
-end
diff --git a/db/migrate/20180831164910_import_common_metrics.rb b/db/migrate/20180831164910_import_common_metrics.rb
index f67d5f40aad..4e61a25c1ad 100644
--- a/db/migrate/20180831164910_import_common_metrics.rb
+++ b/db/migrate/20180831164910_import_common_metrics.rb
@@ -3,12 +3,10 @@
class ImportCommonMetrics < ActiveRecord::Migration[4.2]
include Gitlab::Database::MigrationHelpers
- require Rails.root.join('db/importers/common_metrics_importer.rb')
-
DOWNTIME = false
def up
- Importers::CommonMetricsImporter.new.execute
+ ::Gitlab::DatabaseImporters::CommonMetrics::Importer.new.execute
end
def down
diff --git a/db/migrate/20181006004100_import_common_metrics_nginx_vts.rb b/db/migrate/20181006004100_import_common_metrics_nginx_vts.rb
index 5cd312837df..2b238774dca 100644
--- a/db/migrate/20181006004100_import_common_metrics_nginx_vts.rb
+++ b/db/migrate/20181006004100_import_common_metrics_nginx_vts.rb
@@ -1,12 +1,10 @@
class ImportCommonMetricsNginxVts < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
- require Rails.root.join('db/importers/common_metrics_importer.rb')
-
DOWNTIME = false
def up
- Importers::CommonMetricsImporter.new.execute
+ ::Gitlab::DatabaseImporters::CommonMetrics::Importer.new.execute
end
def down
diff --git a/db/migrate/20190326164045_import_common_metrics_knative.rb b/db/migrate/20190326164045_import_common_metrics_knative.rb
index 340ec1e1f75..6b331755774 100644
--- a/db/migrate/20190326164045_import_common_metrics_knative.rb
+++ b/db/migrate/20190326164045_import_common_metrics_knative.rb
@@ -3,12 +3,10 @@
class ImportCommonMetricsKnative < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
- require Rails.root.join('db/importers/common_metrics_importer.rb')
-
DOWNTIME = false
def up
- Importers::CommonMetricsImporter.new.execute
+ ::Gitlab::DatabaseImporters::CommonMetrics::Importer.new.execute
end
def down
diff --git a/db/migrate/20190408163745_prometheus_knative05_fix.rb b/db/migrate/20190408163745_prometheus_knative05_fix.rb
index c11f6f0e29b..3d0aa782669 100644
--- a/db/migrate/20190408163745_prometheus_knative05_fix.rb
+++ b/db/migrate/20190408163745_prometheus_knative05_fix.rb
@@ -6,12 +6,10 @@
class PrometheusKnative05Fix < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
- require Rails.root.join('db/importers/common_metrics_importer.rb')
-
DOWNTIME = false
def up
- Importers::CommonMetricsImporter.new.execute
+ ::Gitlab::DatabaseImporters::CommonMetrics::Importer.new.execute
end
def down
diff --git a/doc/development/prometheus_metrics.md b/doc/development/prometheus_metrics.md
index 0511e735843..576601372a3 100644
--- a/doc/development/prometheus_metrics.md
+++ b/doc/development/prometheus_metrics.md
@@ -33,12 +33,10 @@ For example: you might be interested in migrating all dependent data to a differ
class ImportCommonMetrics < ActiveRecord::Migration[4.2]
include Gitlab::Database::MigrationHelpers
- require Rails.root.join('db/importers/common_metrics_importer.rb')
-
DOWNTIME = false
def up
- Importers::CommonMetricsImporter.new.execute
+ ::Gitlab::DatabaseImporters::CommonMetrics::Importer.new.execute
end
def down
diff --git a/lib/gitlab/database_importers/common_metrics.rb b/lib/gitlab/database_importers/common_metrics.rb
new file mode 100644
index 00000000000..f964ae8a275
--- /dev/null
+++ b/lib/gitlab/database_importers/common_metrics.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module DatabaseImporters
+ module CommonMetrics
+ end
+ end
+end
diff --git a/lib/gitlab/database_importers/common_metrics/importer.rb b/lib/gitlab/database_importers/common_metrics/importer.rb
new file mode 100644
index 00000000000..6c61e05674e
--- /dev/null
+++ b/lib/gitlab/database_importers/common_metrics/importer.rb
@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module DatabaseImporters
+ module CommonMetrics
+ class Importer
+ MissingQueryId = Class.new(StandardError)
+
+ attr_reader :content
+
+ def initialize(filename = 'common_metrics.yml')
+ @content = YAML.load_file(Rails.root.join('config', 'prometheus', filename))
+ end
+
+ def execute
+ CommonMetrics::PrometheusMetric.reset_column_information
+
+ process_content do |id, attributes|
+ find_or_build_metric!(id)
+ .update!(**attributes)
+ end
+ end
+
+ private
+
+ def process_content(&blk)
+ content['panel_groups'].map do |group|
+ process_group(group, &blk)
+ end
+ end
+
+ def process_group(group, &blk)
+ attributes = {
+ group: find_group_title_key(group['group'])
+ }
+
+ group['panels'].map do |panel|
+ process_panel(panel, attributes, &blk)
+ end
+ end
+
+ def process_panel(panel, attributes, &blk)
+ attributes = attributes.merge(
+ title: panel['title'],
+ y_label: panel['y_label'])
+
+ panel['metrics'].map do |metric_details|
+ process_metric_details(metric_details, attributes, &blk)
+ end
+ end
+
+ def process_metric_details(metric_details, attributes, &blk)
+ attributes = attributes.merge(
+ legend: metric_details['label'],
+ query: metric_details['query_range'],
+ unit: metric_details['unit'])
+
+ yield(metric_details['id'], attributes)
+ end
+
+ def find_or_build_metric!(id)
+ raise MissingQueryId unless id
+
+ CommonMetrics::PrometheusMetric.common.find_by(identifier: id) ||
+ CommonMetrics::PrometheusMetric.new(common: true, identifier: id)
+ end
+
+ def find_group_title_key(title)
+ CommonMetrics::PrometheusMetricEnums.groups[find_group_title(title)]
+ end
+
+ def find_group_title(title)
+ CommonMetrics::PrometheusMetricEnums.group_titles.invert[title]
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database_importers/common_metrics/prometheus_metric.rb b/lib/gitlab/database_importers/common_metrics/prometheus_metric.rb
new file mode 100644
index 00000000000..b4a392cbea9
--- /dev/null
+++ b/lib/gitlab/database_importers/common_metrics/prometheus_metric.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module DatabaseImporters
+ module CommonMetrics
+ class PrometheusMetric < ApplicationRecord
+ enum group: PrometheusMetricEnums.groups
+ scope :common, -> { where(common: true) }
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb b/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb
new file mode 100644
index 00000000000..c9e957ec7c0
--- /dev/null
+++ b/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module DatabaseImporters
+ module CommonMetrics
+ module PrometheusMetricEnums
+ def self.groups
+ {
+ # built-in groups
+ nginx_ingress_vts: -1,
+ ha_proxy: -2,
+ aws_elb: -3,
+ nginx: -4,
+ kubernetes: -5,
+ nginx_ingress: -6,
+
+ # custom groups
+ business: 0,
+ response: 1,
+ system: 2
+ }
+ end
+
+ def self.group_titles
+ {
+ business: _('Business metrics (Custom)'),
+ response: _('Response metrics (Custom)'),
+ system: _('System metrics (Custom)'),
+ nginx_ingress_vts: _('Response metrics (NGINX Ingress VTS)'),
+ nginx_ingress: _('Response metrics (NGINX Ingress)'),
+ ha_proxy: _('Response metrics (HA Proxy)'),
+ aws_elb: _('Response metrics (AWS ELB)'),
+ nginx: _('Response metrics (NGINX)'),
+ kubernetes: _('System metrics (Kubernetes)')
+ }
+ end
+ end
+ end
+ end
+end
diff --git a/spec/db/importers/common_metrics_importer_spec.rb b/spec/lib/gitlab/database_importers/common_metrics/importer_spec.rb
index a717c8cd04d..57c8bafd488 100644
--- a/spec/db/importers/common_metrics_importer_spec.rb
+++ b/spec/lib/gitlab/database_importers/common_metrics/importer_spec.rb
@@ -1,25 +1,8 @@
# frozen_string_literal: true
require 'rails_helper'
-require Rails.root.join("db", "importers", "common_metrics_importer.rb")
-describe Importers::PrometheusMetric do
- let(:existing_group_titles) do
- ::PrometheusMetric::GROUP_DETAILS.each_with_object({}) do |(key, value), memo|
- memo[key] = value[:group_title]
- end
- end
-
- it 'group enum equals ::PrometheusMetric' do
- expect(described_class.groups).to eq(::PrometheusMetric.groups)
- end
-
- it 'GROUP_TITLES equals ::PrometheusMetric' do
- expect(described_class::GROUP_TITLES).to eq(existing_group_titles)
- end
-end
-
-describe Importers::CommonMetricsImporter do
+describe Gitlab::DatabaseImporters::CommonMetrics::Importer do
subject { described_class.new }
context "does import common_metrics.yml" do
@@ -104,7 +87,7 @@ describe Importers::CommonMetricsImporter do
let(:query_identifier) { }
it 'raises exception' do
- expect { subject.execute }.to raise_error(described_class::MissingQueryId)
+ expect { subject.execute }.to raise_error(Gitlab::DatabaseImporters::CommonMetrics::Importer::MissingQueryId)
end
end
diff --git a/spec/lib/gitlab/database_importers/common_metrics/prometheus_metric_spec.rb b/spec/lib/gitlab/database_importers/common_metrics/prometheus_metric_spec.rb
new file mode 100644
index 00000000000..94f544e59b3
--- /dev/null
+++ b/spec/lib/gitlab/database_importers/common_metrics/prometheus_metric_spec.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe Gitlab::DatabaseImporters::CommonMetrics::PrometheusMetric do
+ it 'group enum equals ::PrometheusMetric' do
+ expect(described_class.groups).to eq(::PrometheusMetric.groups)
+ end
+
+ it '.group_titles equals ::PrometheusMetric' do
+ existing_group_titles = ::PrometheusMetricEnums.group_details.each_with_object({}) do |(key, value), memo|
+ memo[key] = value[:group_title]
+ end
+ expect(Gitlab::DatabaseImporters::CommonMetrics::PrometheusMetricEnums.group_titles).to eq(existing_group_titles)
+ end
+end