diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-06-18 11:18:50 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-06-18 11:18:50 +0000 |
commit | 8c7f4e9d5f36cff46365a7f8c4b9c21578c1e781 (patch) | |
tree | a77e7fe7a93de11213032ed4ab1f33a3db51b738 /lib/gitlab/metrics | |
parent | 00b35af3db1abfe813a778f643dad221aad51fca (diff) | |
download | gitlab-ce-8c7f4e9d5f36cff46365a7f8c4b9c21578c1e781.tar.gz |
Add latest changes from gitlab-org/gitlab@13-1-stable-ee
Diffstat (limited to 'lib/gitlab/metrics')
-rw-r--r-- | lib/gitlab/metrics/dashboard/stages/url_validator.rb | 19 | ||||
-rw-r--r-- | lib/gitlab/metrics/elasticsearch_rack_middleware.rb | 41 | ||||
-rw-r--r-- | lib/gitlab/metrics/methods.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/metrics/redis_rack_middleware.rb | 39 | ||||
-rw-r--r-- | lib/gitlab/metrics/samplers/base_sampler.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/metrics/samplers/puma_sampler.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/metrics/samplers/ruby_sampler.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/metrics/sidekiq_middleware.rb | 15 | ||||
-rw-r--r-- | lib/gitlab/metrics/subscribers/active_record.rb | 18 | ||||
-rw-r--r-- | lib/gitlab/metrics/transaction.rb | 10 |
10 files changed, 143 insertions, 10 deletions
diff --git a/lib/gitlab/metrics/dashboard/stages/url_validator.rb b/lib/gitlab/metrics/dashboard/stages/url_validator.rb new file mode 100644 index 00000000000..ff36f7b605e --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/url_validator.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class UrlValidator < BaseStage + def transform! + dashboard[:links]&.each do |link| + Gitlab::UrlBlocker.validate!(link[:url]) + rescue Gitlab::UrlBlocker::BlockedUrlError + link[:url] = '' + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/elasticsearch_rack_middleware.rb b/lib/gitlab/metrics/elasticsearch_rack_middleware.rb new file mode 100644 index 00000000000..6830eed68d5 --- /dev/null +++ b/lib/gitlab/metrics/elasticsearch_rack_middleware.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + # Rack middleware for tracking Elasticsearch metrics from Grape and Web requests. + class ElasticsearchRackMiddleware + HISTOGRAM_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60].freeze + + def initialize(app) + @app = app + + @requests_total_counter = Gitlab::Metrics.counter(:http_elasticsearch_requests_total, + 'Amount of calls to Elasticsearch servers during web requests', + Gitlab::Metrics::Transaction::BASE_LABELS) + @requests_duration_histogram = Gitlab::Metrics.histogram(:http_elasticsearch_requests_duration_seconds, + 'Query time for Elasticsearch servers during web requests', + Gitlab::Metrics::Transaction::BASE_LABELS, + HISTOGRAM_BUCKETS) + end + + def call(env) + transaction = Gitlab::Metrics.current_transaction + + @app.call(env) + ensure + record_metrics(transaction) + end + + private + + def record_metrics(transaction) + labels = transaction.labels + query_time = ::Gitlab::Instrumentation::ElasticsearchTransport.query_time + request_count = ::Gitlab::Instrumentation::ElasticsearchTransport.get_request_count + + @requests_total_counter.increment(labels, request_count) + @requests_duration_histogram.observe(labels, query_time) + end + end + end +end diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb index cee601ff14c..5955987541c 100644 --- a/lib/gitlab/metrics/methods.rb +++ b/lib/gitlab/metrics/methods.rb @@ -52,7 +52,7 @@ module Gitlab end def disabled_by_feature(options) - options.with_feature && !::Feature.get(options.with_feature).enabled? + options.with_feature && !::Feature.enabled?(options.with_feature) end def build_metric!(type, name, options) diff --git a/lib/gitlab/metrics/redis_rack_middleware.rb b/lib/gitlab/metrics/redis_rack_middleware.rb new file mode 100644 index 00000000000..f0f99c5f45d --- /dev/null +++ b/lib/gitlab/metrics/redis_rack_middleware.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + # Rack middleware for tracking Redis metrics from Grape and Web requests. + class RedisRackMiddleware + def initialize(app) + @app = app + + @requests_total_counter = Gitlab::Metrics.counter(:http_redis_requests_total, + 'Amount of calls to Redis servers during web requests', + Gitlab::Metrics::Transaction::BASE_LABELS) + @requests_duration_histogram = Gitlab::Metrics.histogram(:http_redis_requests_duration_seconds, + 'Query time for Redis servers during web requests', + Gitlab::Metrics::Transaction::BASE_LABELS, + Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS) + end + + def call(env) + transaction = Gitlab::Metrics.current_transaction + + @app.call(env) + ensure + record_metrics(transaction) + end + + private + + def record_metrics(transaction) + labels = transaction.labels + query_time = Gitlab::Instrumentation::Redis.query_time + request_count = Gitlab::Instrumentation::Redis.get_request_count + + @requests_total_counter.increment(labels, request_count) + @requests_duration_histogram.observe(labels, query_time) + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index 90051f85f31..ff3e7be567f 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -6,8 +6,10 @@ module Gitlab module Metrics module Samplers class BaseSampler < Daemon + attr_reader :interval + # interval - The sampling interval in seconds. - def initialize(interval) + def initialize(interval = self.class::SAMPLING_INTERVAL_SECONDS) interval_half = interval.to_f / 2 @interval = interval diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb index 98dd517ee3b..b5343d5e66a 100644 --- a/lib/gitlab/metrics/samplers/puma_sampler.rb +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -4,6 +4,8 @@ module Gitlab module Metrics module Samplers class PumaSampler < BaseSampler + SAMPLING_INTERVAL_SECONDS = 5 + def metrics @metrics ||= init_metrics end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index df59c06911b..dac9fbd1247 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -6,9 +6,10 @@ module Gitlab module Metrics module Samplers class RubySampler < BaseSampler + SAMPLING_INTERVAL_SECONDS = 60 GC_REPORT_BUCKETS = [0.005, 0.01, 0.02, 0.04, 0.07, 0.1, 0.5].freeze - def initialize(interval) + def initialize(*) GC::Profiler.clear metrics[:process_start_time_seconds].set(labels, Time.now.to_i) diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index 8dfb61046c4..de8e1ca3256 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -6,19 +6,30 @@ module Gitlab # # This middleware is intended to be used as a server-side middleware. class SidekiqMiddleware - def call(worker, message, queue) + def call(worker, payload, queue) trans = BackgroundTransaction.new(worker.class) begin # Old gitlad-shell messages don't provide enqueued_at/created_at attributes - trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0)) + enqueued_at = payload['enqueued_at'] || payload['created_at'] || 0 + trans.set(:sidekiq_queue_duration, Time.current.to_f - enqueued_at) trans.run { yield } rescue Exception => error # rubocop: disable Lint/RescueException trans.add_event(:sidekiq_exception) raise error + ensure + add_info_to_payload(payload, trans) end end + + private + + def add_info_to_payload(payload, trans) + payload[:db_count] = trans.get(:db_count, :counter).to_i + payload[:db_write_count] = trans.get(:db_write_count, :counter).to_i + payload[:db_cached_count] = trans.get(:db_cached_count, :counter).to_i + end end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index a02dd850582..1628eeb5a95 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -9,6 +9,7 @@ module Gitlab attach_to :active_record IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze + DB_COUNTERS = %i{db_count db_write_count db_cached_count}.freeze def sql(event) return unless current_transaction @@ -19,8 +20,7 @@ module Gitlab self.class.gitlab_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0) - current_transaction.increment(:sql_duration, event.duration, false) - current_transaction.increment(:sql_count, 1, false) + increment_db_counters(payload) end private @@ -31,6 +31,20 @@ module Gitlab buckets [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0] end + def select_sql_command?(payload) + payload[:sql].match(/\A((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i) + end + + def increment_db_counters(payload) + current_transaction.increment(:db_count, 1) + + if payload.fetch(:cached, payload[:name] == 'CACHE') + current_transaction.increment(:db_cached_count, 1) + end + + current_transaction.increment(:db_write_count, 1) unless select_sql_command?(payload) + end + def current_transaction Transaction.current end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index b126efd2dd5..822f5243e9d 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -16,7 +16,7 @@ module Gitlab # The series to store events (e.g. Git pushes) in. EVENT_SERIES = 'events' - attr_reader :tags, :method + attr_reader :method def self.current Thread.current[THREAD_KEY] @@ -28,8 +28,6 @@ module Gitlab @started_at = nil @finished_at = nil - @tags = {} - @memory_before = 0 @memory_after = 0 end @@ -94,6 +92,12 @@ module Gitlab self.class.transaction_metric(name, :gauge).set(labels, value) if use_prometheus end + def get(name, type, tags = {}) + metric = self.class.transaction_metric(name, type) + + metric.get(filter_tags(tags).merge(labels)) + end + def labels BASE_LABELS end |