diff options
Diffstat (limited to 'lib/gitlab/metrics')
| -rw-r--r-- | lib/gitlab/metrics/dashboard/base_service.rb | 72 | ||||
| -rw-r--r-- | lib/gitlab/metrics/dashboard/dynamic_dashboard_service.rb | 65 | ||||
| -rw-r--r-- | lib/gitlab/metrics/dashboard/finder.rb | 10 | ||||
| -rw-r--r-- | lib/gitlab/metrics/dashboard/project_dashboard_service.rb | 50 | ||||
| -rw-r--r-- | lib/gitlab/metrics/dashboard/system_dashboard_service.rb | 49 | ||||
| -rw-r--r-- | lib/gitlab/metrics/samplers/influx_sampler.rb | 22 | ||||
| -rw-r--r-- | lib/gitlab/metrics/samplers/ruby_sampler.rb | 23 |
7 files changed, 22 insertions, 269 deletions
diff --git a/lib/gitlab/metrics/dashboard/base_service.rb b/lib/gitlab/metrics/dashboard/base_service.rb deleted file mode 100644 index 0628e82e592..00000000000 --- a/lib/gitlab/metrics/dashboard/base_service.rb +++ /dev/null @@ -1,72 +0,0 @@ -# frozen_string_literal: true - -# Searches a projects repository for a metrics dashboard and formats the output. -# Expects any custom dashboards will be located in `.gitlab/dashboards` -module Gitlab - module Metrics - module Dashboard - class BaseService < ::BaseService - PROCESSING_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardProcessingError - NOT_FOUND_ERROR = Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError - - def get_dashboard - return error('Insufficient permissions.', :unauthorized) unless allowed? - - success(dashboard: process_dashboard) - rescue NOT_FOUND_ERROR - error("#{dashboard_path} could not be found.", :not_found) - rescue PROCESSING_ERROR => e - error(e.message, :unprocessable_entity) - end - - # Summary of all known dashboards for the service. - # @return [Array<Hash>] ex) [{ path: String, default: Boolean }] - def self.all_dashboard_paths(_project) - raise NotImplementedError - end - - # Returns an un-processed dashboard from the cache. - def raw_dashboard - Gitlab::Metrics::Dashboard::Cache.fetch(cache_key) { get_raw_dashboard } - end - - private - - # Determines whether users should be able to view - # dashboards at all. - def allowed? - Ability.allowed?(current_user, :read_environment, project) - end - - # Returns a new dashboard Hash, supplemented with DB info - def process_dashboard - Gitlab::Metrics::Dashboard::Processor - .new(project, params[:environment], raw_dashboard) - .process(insert_project_metrics: insert_project_metrics?) - end - - # @return [String] Relative filepath of the dashboard yml - def dashboard_path - params[:dashboard_path] - end - - # @return [Hash] an unmodified dashboard - def get_raw_dashboard - raise NotImplementedError - end - - # @return [String] - def cache_key - raise NotImplementedError - end - - # Determines whether custom metrics should be included - # in the processed output. - # @return [Boolean] - def insert_project_metrics? - false - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/dynamic_dashboard_service.rb b/lib/gitlab/metrics/dashboard/dynamic_dashboard_service.rb deleted file mode 100644 index 81ed8922e17..00000000000 --- a/lib/gitlab/metrics/dashboard/dynamic_dashboard_service.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -# Responsible for returning a filtered system dashboard -# containing only the default embedded metrics. In future, -# this class may be updated to support filtering to -# alternate metrics/panels. -# -# Why isn't this filtering in a processing stage? By filtering -# here, we ensure the dynamically-determined dashboard is cached. -# -# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. -module Gitlab - module Metrics - module Dashboard - class DynamicDashboardService < Gitlab::Metrics::Dashboard::BaseService - # For the default filtering for embedded metrics, - # uses the 'id' key in dashboard-yml definition for - # identification. - DEFAULT_EMBEDDED_METRICS_IDENTIFIERS = %w( - system_metrics_kubernetes_container_memory_total - system_metrics_kubernetes_container_cores_total - ).freeze - - # Returns a new dashboard with only the matching - # metrics from the system dashboard, stripped of groups. - # @return [Hash] - def raw_dashboard - panels = panel_groups.each_with_object([]) do |group, panels| - matched_panels = group['panels'].select { |panel| matching_panel?(panel) } - - panels.concat(matched_panels) - end - - { 'panel_groups' => [{ 'panels' => panels }] } - end - - def cache_key - "dynamic_metrics_dashboard_#{metric_identifiers.join('_')}" - end - - private - - # Returns an array of the panels groups on the - # system dashboard - def panel_groups - Gitlab::Metrics::Dashboard::SystemDashboardService - .new(project, nil) - .raw_dashboard['panel_groups'] - end - - # Identifies a panel as "matching" if any metric ids in - # the panel is in the list of identifiers to collect. - def matching_panel?(panel) - panel['metrics'].any? do |metric| - metric_identifiers.include?(metric['id']) - end - end - - def metric_identifiers - DEFAULT_EMBEDDED_METRICS_IDENTIFIERS - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index d7491d1553d..1373830844b 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -47,22 +47,22 @@ module Gitlab private def service_for_path(dashboard_path, embedded:) - return dynamic_service if embedded + return embed_service if embedded return system_service if system_dashboard?(dashboard_path) project_service end def system_service - Gitlab::Metrics::Dashboard::SystemDashboardService + ::Metrics::Dashboard::SystemDashboardService end def project_service - Gitlab::Metrics::Dashboard::ProjectDashboardService + ::Metrics::Dashboard::ProjectDashboardService end - def dynamic_service - Gitlab::Metrics::Dashboard::DynamicDashboardService + def embed_service + ::Metrics::Dashboard::DefaultEmbedService end def system_dashboard?(filepath) diff --git a/lib/gitlab/metrics/dashboard/project_dashboard_service.rb b/lib/gitlab/metrics/dashboard/project_dashboard_service.rb deleted file mode 100644 index 5a1c4ecf886..00000000000 --- a/lib/gitlab/metrics/dashboard/project_dashboard_service.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -# Searches a projects repository for a metrics dashboard and formats the output. -# Expects any custom dashboards will be located in `.gitlab/dashboards` -# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. -module Gitlab - module Metrics - module Dashboard - class ProjectDashboardService < Gitlab::Metrics::Dashboard::BaseService - DASHBOARD_ROOT = ".gitlab/dashboards" - - class << self - def all_dashboard_paths(project) - file_finder(project) - .list_files_for(DASHBOARD_ROOT) - .map do |filepath| - { - path: filepath, - display_name: name_for_path(filepath), - default: false - } - end - end - - def file_finder(project) - Gitlab::Template::Finders::RepoTemplateFinder.new(project, DASHBOARD_ROOT, '.yml') - end - - # Grabs the filepath after the base directory. - def name_for_path(filepath) - filepath.delete_prefix("#{DASHBOARD_ROOT}/") - end - end - - private - - # Searches the project repo for a custom-defined dashboard. - def get_raw_dashboard - yml = self.class.file_finder(project).read(dashboard_path) - - YAML.safe_load(yml) - end - - def cache_key - "project_#{project.id}_metrics_dashboard_#{dashboard_path}" - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/system_dashboard_service.rb b/lib/gitlab/metrics/dashboard/system_dashboard_service.rb deleted file mode 100644 index 82421572f4a..00000000000 --- a/lib/gitlab/metrics/dashboard/system_dashboard_service.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -# Fetches the system metrics dashboard and formats the output. -# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. -module Gitlab - module Metrics - module Dashboard - class SystemDashboardService < Gitlab::Metrics::Dashboard::BaseService - SYSTEM_DASHBOARD_PATH = 'config/prometheus/common_metrics.yml' - SYSTEM_DASHBOARD_NAME = 'Default' - - class << self - def all_dashboard_paths(_project) - [{ - path: SYSTEM_DASHBOARD_PATH, - display_name: SYSTEM_DASHBOARD_NAME, - default: true - }] - end - - def system_dashboard?(filepath) - filepath == SYSTEM_DASHBOARD_PATH - end - end - - private - - def dashboard_path - SYSTEM_DASHBOARD_PATH - end - - # Returns the base metrics shipped with every GitLab service. - def get_raw_dashboard - yml = File.read(Rails.root.join(dashboard_path)) - - YAML.safe_load(yml) - end - - def cache_key - "metrics_dashboard_#{dashboard_path}" - end - - def insert_project_metrics? - true - end - end - end - end -end diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb index 5138b37f83e..1eae0a7bf45 100644 --- a/lib/gitlab/metrics/samplers/influx_sampler.rb +++ b/lib/gitlab/metrics/samplers/influx_sampler.rb @@ -15,19 +15,14 @@ module Gitlab @last_step = nil @metrics = [] - - @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) - @last_major_gc = Delta.new(GC.stat[:major_gc_count]) end def sample sample_memory_usage sample_file_descriptors - sample_gc flush ensure - GC::Profiler.clear @metrics.clear end @@ -43,23 +38,6 @@ module Gitlab add_metric('file_descriptors', value: System.file_descriptor_count) end - def sample_gc - time = GC::Profiler.total_time * 1000.0 - stats = GC.stat.merge(total_time: time) - - # We want the difference of GC runs compared to the last sample, not the - # total amount since the process started. - stats[:minor_gc_count] = - @last_minor_gc.compared_with(stats[:minor_gc_count]) - - stats[:major_gc_count] = - @last_major_gc.compared_with(stats[:major_gc_count]) - - stats[:count] = stats[:minor_gc_count] + stats[:major_gc_count] - - add_metric('gc_statistics', stats) - end - def add_metric(series, values, tags = {}) prefix = sidekiq? ? 'sidekiq_' : 'rails_' diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 1e200db0baf..3bfa3da35e0 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -6,7 +6,11 @@ module Gitlab module Metrics module Samplers class RubySampler < BaseSampler + GC_REPORT_BUCKETS = [0.001, 0.002, 0.005, 0.01, 0.05, 0.1, 0.5].freeze + def initialize(interval) + GC::Profiler.clear + metrics[:process_start_time_seconds].set(labels, Time.now.to_i) super @@ -37,7 +41,7 @@ module Gitlab process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels), process_start_time_seconds: ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'), sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels), - total_time: ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + gc_duration_seconds: ::Gitlab::Metrics.histogram(with_prefix(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS) } GC.stat.keys.each do |key| @@ -57,20 +61,27 @@ module Gitlab sample_gc metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time) - ensure - GC::Profiler.clear end private def sample_gc - # Collect generic GC stats. + # Observe all GC samples + sample_gc_reports.each do |report| + metrics[:gc_duration_seconds].observe(labels, report[:GC_TIME]) + end + + # Collect generic GC stats GC.stat.each do |key, value| metrics[key].set(labels, value) end + end - # Collect the GC time since last sample in float seconds. - metrics[:total_time].increment(labels, GC::Profiler.total_time) + def sample_gc_reports + GC::Profiler.enable + GC::Profiler.raw_data + ensure + GC::Profiler.clear end def set_memory_usage_metrics |