diff options
Diffstat (limited to 'lib/gitlab/metrics')
| -rw-r--r-- | lib/gitlab/metrics/dashboard/base_service.rb | 73 | ||||
| -rw-r--r-- | lib/gitlab/metrics/dashboard/finder.rb | 51 | ||||
| -rw-r--r-- | lib/gitlab/metrics/dashboard/processor.rb | 46 | ||||
| -rw-r--r-- | lib/gitlab/metrics/dashboard/project_dashboard_service.rb | 47 | ||||
| -rw-r--r-- | lib/gitlab/metrics/dashboard/stages/base_stage.rb | 58 | ||||
| -rw-r--r-- | lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb | 23 | ||||
| -rw-r--r-- | lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb | 106 | ||||
| -rw-r--r-- | lib/gitlab/metrics/dashboard/stages/sorter.rb | 34 | ||||
| -rw-r--r-- | lib/gitlab/metrics/dashboard/system_dashboard_service.rb | 47 | ||||
| -rw-r--r-- | lib/gitlab/metrics/influx_db.rb | 6 | ||||
| -rw-r--r-- | lib/gitlab/metrics/metric.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/metrics/samplers/puma_sampler.rb | 93 | ||||
| -rw-r--r-- | lib/gitlab/metrics/samplers/ruby_sampler.rb | 33 | ||||
| -rw-r--r-- | lib/gitlab/metrics/samplers/unicorn_sampler.rb | 36 | ||||
| -rw-r--r-- | lib/gitlab/metrics/system.rb | 28 | ||||
| -rw-r--r-- | lib/gitlab/metrics/transaction.rb | 15 |
16 files changed, 668 insertions, 30 deletions
diff --git a/lib/gitlab/metrics/dashboard/base_service.rb b/lib/gitlab/metrics/dashboard/base_service.rb new file mode 100644 index 00000000000..94aabd0466c --- /dev/null +++ b/lib/gitlab/metrics/dashboard/base_service.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +# Searches a projects repository for a metrics dashboard and formats the output. +# Expects any custom dashboards will be located in `.gitlab/dashboards` +module Gitlab + module Metrics + module Dashboard + class BaseService < ::BaseService + DASHBOARD_LAYOUT_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardLayoutError + + def get_dashboard + return error("#{dashboard_path} could not be found.", :not_found) unless path_available? + + success(dashboard: process_dashboard) + rescue DASHBOARD_LAYOUT_ERROR => e + error(e.message, :unprocessable_entity) + end + + # Summary of all known dashboards for the service. + # @return [Array<Hash>] ex) [{ path: String, default: Boolean }] + def all_dashboard_paths(_project) + raise NotImplementedError + end + + private + + # Returns a new dashboard Hash, supplemented with DB info + def process_dashboard + Gitlab::Metrics::Dashboard::Processor + .new(project, params[:environment], raw_dashboard) + .process(insert_project_metrics: insert_project_metrics?) + end + + # @return [String] Relative filepath of the dashboard yml + def dashboard_path + params[:dashboard_path] + end + + # Returns an un-processed dashboard from the cache. + def raw_dashboard + Rails.cache.fetch(cache_key) { get_raw_dashboard } + end + + # @return [Hash] an unmodified dashboard + def get_raw_dashboard + raise NotImplementedError + end + + # @return [String] + def cache_key + raise NotImplementedError + end + + # Determines whether custom metrics should be included + # in the processed output. + def insert_project_metrics? + false + end + + # Checks if dashboard path exists or should be rejected + # as a result of file-changes to the project repository. + # @return [Boolean] + def path_available? + available_paths = Gitlab::Metrics::Dashboard::Finder.find_all_paths(project) + + available_paths.any? do |path_params| + path_params[:path] == dashboard_path + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb new file mode 100644 index 00000000000..4a41590f000 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +# Returns DB-supplmented dashboard info for determining +# the layout of UI. Intended entry-point for the Metrics::Dashboard +# module. +module Gitlab + module Metrics + module Dashboard + class Finder + class << self + # Returns a formatted dashboard packed with DB info. + # @return [Hash] + def find(project, user, environment, dashboard_path = nil) + service = system_dashboard?(dashboard_path) ? system_service : project_service + + service + .new(project, user, environment: environment, dashboard_path: dashboard_path) + .get_dashboard + end + + # Summary of all known dashboards. + # @return [Array<Hash>] ex) [{ path: String, default: Boolean }] + def find_all_paths(project) + project.repository.metrics_dashboard_paths + end + + # Summary of all known dashboards. Used to populate repo cache. + # Prefer #find_all_paths. + def find_all_paths_from_source(project) + system_service.all_dashboard_paths(project) + .+ project_service.all_dashboard_paths(project) + end + + private + + def system_service + Gitlab::Metrics::Dashboard::SystemDashboardService + end + + def project_service + Gitlab::Metrics::Dashboard::ProjectDashboardService + end + + def system_dashboard?(filepath) + !filepath || system_service.system_dashboard?(filepath) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/processor.rb b/lib/gitlab/metrics/dashboard/processor.rb new file mode 100644 index 00000000000..dd986020693 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/processor.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + # Responsible for processesing a dashboard hash, inserting + # relevant DB records & sorting for proper rendering in + # the UI. These includes shared metric info, custom metrics + # info, and alerts (only in EE). + class Processor + SYSTEM_SEQUENCE = [ + Stages::CommonMetricsInserter, + Stages::ProjectMetricsInserter, + Stages::Sorter + ].freeze + + PROJECT_SEQUENCE = [ + Stages::CommonMetricsInserter, + Stages::Sorter + ].freeze + + def initialize(project, environment, dashboard) + @project = project + @environment = environment + @dashboard = dashboard + end + + # Returns a new dashboard hash with the results of + # running transforms on the dashboard. + def process(insert_project_metrics:) + @dashboard.deep_symbolize_keys.tap do |dashboard| + sequence(insert_project_metrics).each do |stage| + stage.new(@project, @environment, dashboard).transform! + end + end + end + + private + + def sequence(insert_project_metrics) + insert_project_metrics ? SYSTEM_SEQUENCE : PROJECT_SEQUENCE + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/project_dashboard_service.rb b/lib/gitlab/metrics/dashboard/project_dashboard_service.rb new file mode 100644 index 00000000000..fdffd067c93 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/project_dashboard_service.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +# Searches a projects repository for a metrics dashboard and formats the output. +# Expects any custom dashboards will be located in `.gitlab/dashboards` +# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. +module Gitlab + module Metrics + module Dashboard + class ProjectDashboardService < Gitlab::Metrics::Dashboard::BaseService + DASHBOARD_ROOT = ".gitlab/dashboards" + + class << self + def all_dashboard_paths(project) + file_finder(project) + .list_files_for(DASHBOARD_ROOT) + .map do |filepath| + Rails.cache.delete(cache_key(project.id, filepath)) + + { path: filepath, default: false } + end + end + + def file_finder(project) + Gitlab::Template::Finders::RepoTemplateFinder.new(project, DASHBOARD_ROOT, '.yml') + end + + def cache_key(id, dashboard_path) + "project_#{id}_metrics_dashboard_#{dashboard_path}" + end + end + + private + + # Searches the project repo for a custom-defined dashboard. + def get_raw_dashboard + yml = self.class.file_finder(project).read(dashboard_path) + + YAML.safe_load(yml) + end + + def cache_key + self.class.cache_key(project.id, dashboard_path) + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb new file mode 100644 index 00000000000..a6d1f974556 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/base_stage.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class BaseStage + DashboardLayoutError = Class.new(StandardError) + + DEFAULT_PANEL_TYPE = 'area-chart' + + attr_reader :project, :environment, :dashboard + + def initialize(project, environment, dashboard) + @project = project + @environment = environment + @dashboard = dashboard + end + + # Entry-point to the stage + def transform! + raise NotImplementedError + end + + protected + + def missing_panel_groups! + raise DashboardLayoutError.new('Top-level key :panel_groups must be an array') + end + + def missing_panels! + raise DashboardLayoutError.new('Each "panel_group" must define an array :panels') + end + + def missing_metrics! + raise DashboardLayoutError.new('Each "panel" must define an array :metrics') + end + + def for_metrics + missing_panel_groups! unless dashboard[:panel_groups].is_a?(Array) + + dashboard[:panel_groups].each do |panel_group| + missing_panels! unless panel_group[:panels].is_a?(Array) + + panel_group[:panels].each do |panel| + missing_metrics! unless panel[:metrics].is_a?(Array) + + panel[:metrics].each do |metric| + yield metric + end + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb new file mode 100644 index 00000000000..188912bedb4 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class CommonMetricsInserter < BaseStage + # For each metric in the dashboard config, attempts to + # find a corresponding database record. If found, + # includes the record's id in the dashboard config. + def transform! + common_metrics = ::PrometheusMetric.common + + for_metrics do |metric| + metric_record = common_metrics.find { |m| m.identifier == metric[:id] } + metric[:metric_id] = metric_record.id if metric_record + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb new file mode 100644 index 00000000000..221610a14d1 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class ProjectMetricsInserter < BaseStage + # Inserts project-specific metrics into the dashboard + # config. If there are no project-specific metrics, + # this will have no effect. + def transform! + project.prometheus_metrics.each do |project_metric| + group = find_or_create_panel_group(dashboard[:panel_groups], project_metric) + panel = find_or_create_panel(group[:panels], project_metric) + find_or_create_metric(panel[:metrics], project_metric) + end + end + + private + + # Looks for a panel_group corresponding to the + # provided metric object. If unavailable, inserts one. + # @param panel_groups [Array<Hash>] + # @param metric [PrometheusMetric] + def find_or_create_panel_group(panel_groups, metric) + panel_group = find_panel_group(panel_groups, metric) + return panel_group if panel_group + + panel_group = new_panel_group(metric) + panel_groups << panel_group + + panel_group + end + + # Looks for a panel corresponding to the provided + # metric object. If unavailable, inserts one. + # @param panels [Array<Hash>] + # @param metric [PrometheusMetric] + def find_or_create_panel(panels, metric) + panel = find_panel(panels, metric) + return panel if panel + + panel = new_panel(metric) + panels << panel + + panel + end + + # Looks for a metric corresponding to the provided + # metric object. If unavailable, inserts one. + # @param metrics [Array<Hash>] + # @param metric [PrometheusMetric] + def find_or_create_metric(metrics, metric) + target_metric = find_metric(metrics, metric) + return target_metric if target_metric + + target_metric = new_metric(metric) + metrics << target_metric + + target_metric + end + + def find_panel_group(panel_groups, metric) + return unless panel_groups + + panel_groups.find { |group| group[:group] == metric.group_title } + end + + def find_panel(panels, metric) + return unless panels + + panel_identifiers = [DEFAULT_PANEL_TYPE, metric.title, metric.y_label] + panels.find { |panel| panel.values_at(:type, :title, :y_label) == panel_identifiers } + end + + def find_metric(metrics, metric) + return unless metrics + + metrics.find { |m| m[:id] == metric.identifier } + end + + def new_panel_group(metric) + { + group: metric.group_title, + priority: metric.priority, + panels: [] + } + end + + def new_panel(metric) + { + type: DEFAULT_PANEL_TYPE, + title: metric.title, + y_label: metric.y_label, + metrics: [] + } + end + + def new_metric(metric) + metric.queries.first.merge(metric_id: metric.id) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/sorter.rb b/lib/gitlab/metrics/dashboard/stages/sorter.rb new file mode 100644 index 00000000000..ba5aa78059c --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/sorter.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class Sorter < BaseStage + def transform! + missing_panel_groups! unless dashboard[:panel_groups].is_a? Array + + sort_groups! + sort_panels! + end + + private + + # Sorts the groups in the dashboard by the :priority key + def sort_groups! + dashboard[:panel_groups] = dashboard[:panel_groups].sort_by { |group| -group[:priority].to_i } + end + + # Sorts the panels in the dashboard by the :weight key + def sort_panels! + dashboard[:panel_groups].each do |group| + missing_panels! unless group[:panels].is_a? Array + + group[:panels] = group[:panels].sort_by { |panel| -panel[:weight].to_i } + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/system_dashboard_service.rb b/lib/gitlab/metrics/dashboard/system_dashboard_service.rb new file mode 100644 index 00000000000..67509ed4230 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/system_dashboard_service.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +# Fetches the system metrics dashboard and formats the output. +# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. +module Gitlab + module Metrics + module Dashboard + class SystemDashboardService < Gitlab::Metrics::Dashboard::BaseService + SYSTEM_DASHBOARD_PATH = 'config/prometheus/common_metrics.yml' + + class << self + def all_dashboard_paths(_project) + [{ + path: SYSTEM_DASHBOARD_PATH, + default: true + }] + end + + def system_dashboard?(filepath) + filepath == SYSTEM_DASHBOARD_PATH + end + end + + private + + def dashboard_path + SYSTEM_DASHBOARD_PATH + end + + # Returns the base metrics shipped with every GitLab service. + def get_raw_dashboard + yml = File.read(Rails.root.join(dashboard_path)) + + YAML.safe_load(yml) + end + + def cache_key + "metrics_dashboard_#{dashboard_path}" + end + + def insert_project_metrics? + true + end + end + end + end +end diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb index 0b04340fbb5..269d90fa971 100644 --- a/lib/gitlab/metrics/influx_db.rb +++ b/lib/gitlab/metrics/influx_db.rb @@ -52,10 +52,8 @@ module Gitlab pool&.with do |connection| prepared.each_slice(settings[:packet_size]) do |slice| - begin - connection.write_points(slice) - rescue StandardError - end + connection.write_points(slice) + rescue StandardError end end rescue Errno::EADDRNOTAVAIL, SocketError => ex diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 9e4d70a71ff..30f181542be 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -4,7 +4,7 @@ module Gitlab module Metrics # Class for storing details of a single metric (label, value, etc). class Metric - JITTER_RANGE = 0.000001..0.001 + JITTER_RANGE = (0.000001..0.001).freeze attr_reader :series, :values, :tags, :type diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb new file mode 100644 index 00000000000..25e40c70230 --- /dev/null +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +require 'puma/state_file' + +module Gitlab + module Metrics + module Samplers + class PumaSampler < BaseSampler + def metrics + @metrics ||= init_metrics + end + + def init_metrics + { + puma_workers: ::Gitlab::Metrics.gauge(:puma_workers, 'Total number of workers'), + puma_running_workers: ::Gitlab::Metrics.gauge(:puma_running_workers, 'Number of active workers'), + puma_stale_workers: ::Gitlab::Metrics.gauge(:puma_stale_workers, 'Number of stale workers'), + puma_phase: ::Gitlab::Metrics.gauge(:puma_phase, 'Phase number (increased during phased restarts)'), + puma_running: ::Gitlab::Metrics.gauge(:puma_running, 'Number of running threads'), + puma_queued_connections: ::Gitlab::Metrics.gauge(:puma_queued_connections, 'Number of connections in that worker\'s "todo" set waiting for a worker thread'), + puma_active_connections: ::Gitlab::Metrics.gauge(:puma_active_connections, 'Number of threads processing a request'), + puma_pool_capacity: ::Gitlab::Metrics.gauge(:puma_pool_capacity, 'Number of requests the worker is capable of taking right now'), + puma_max_threads: ::Gitlab::Metrics.gauge(:puma_max_threads, 'Maximum number of worker threads'), + puma_idle_threads: ::Gitlab::Metrics.gauge(:puma_idle_threads, 'Number of spawned threads which are not processing a request') + } + end + + def sample + json_stats = puma_stats + return unless json_stats + + stats = JSON.parse(json_stats) + + if cluster?(stats) + sample_cluster(stats) + else + sample_single_worker(stats) + end + end + + private + + def puma_stats + Puma.stats + rescue NoMethodError + Rails.logger.info "PumaSampler: stats are not available yet, waiting for Puma to boot" + nil + end + + def sample_cluster(stats) + set_master_metrics(stats) + + stats['worker_status'].each do |worker| + last_status = worker['last_status'] + labels = { worker: "worker_#{worker['index']}" } + + metrics[:puma_phase].set(labels, worker['phase']) + set_worker_metrics(last_status, labels) if last_status.present? + end + end + + def sample_single_worker(stats) + metrics[:puma_workers].set({}, 1) + metrics[:puma_running_workers].set({}, 1) + + set_worker_metrics(stats) + end + + def cluster?(stats) + stats['worker_status'].present? + end + + def set_master_metrics(stats) + labels = { worker: "master" } + + metrics[:puma_workers].set(labels, stats['workers']) + metrics[:puma_running_workers].set(labels, stats['booted_workers']) + metrics[:puma_stale_workers].set(labels, stats['old_workers']) + metrics[:puma_phase].set(labels, stats['phase']) + end + + def set_worker_metrics(stats, labels = {}) + metrics[:puma_running].set(labels, stats['running']) + metrics[:puma_queued_connections].set(labels, stats['backlog']) + metrics[:puma_active_connections].set(labels, stats['max_threads'] - stats['pool_capacity']) + metrics[:puma_pool_capacity].set(labels, stats['pool_capacity']) + metrics[:puma_max_threads].set(labels, stats['max_threads']) + metrics[:puma_idle_threads].set(labels, stats['running'] + stats['pool_capacity'] - stats['max_threads']) + end + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 18a69321905..17eacbd21d8 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -23,25 +23,32 @@ module Gitlab end def init_metrics - metrics = {} - metrics[:sampler_duration] = ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels) - metrics[:total_time] = ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + metrics = { + file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum), + memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum), + process_cpu_seconds_total: ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'), + process_max_fds: ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'), + process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels, :livesum), + process_start_time_seconds: ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'), + sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels), + total_time: ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + } + GC.stat.keys.each do |key| metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) end - metrics[:memory_usage] = ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum) - metrics[:file_descriptors] = ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum) - metrics end def sample start_time = System.monotonic_time - metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage) metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) - + metrics[:process_cpu_seconds_total].set(labels.merge(worker_label), ::Gitlab::Metrics::System.cpu_time) + metrics[:process_max_fds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.max_open_file_descriptors) + metrics[:process_start_time_seconds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.process_start_time) + set_memory_usage_metrics sample_gc metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time) @@ -61,11 +68,19 @@ module Gitlab metrics[:total_time].increment(labels, GC::Profiler.total_time) end + def set_memory_usage_metrics + memory_usage = System.memory_usage + memory_labels = labels.merge(worker_label) + + metrics[:memory_bytes].set(memory_labels, memory_usage) + metrics[:process_resident_memory_bytes].set(memory_labels, memory_usage) + end + def worker_label + return { worker: 'sidekiq' } if Sidekiq.server? return {} unless defined?(Unicorn::Worker) worker_no = ::Prometheus::Client::Support::Unicorn.worker_id - if worker_no { worker: worker_no } else diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index bec64e864b3..9af7e0afed4 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -4,16 +4,16 @@ module Gitlab module Metrics module Samplers class UnicornSampler < BaseSampler - def initialize(interval) - super(interval) + def metrics + @metrics ||= init_metrics end - def unicorn_active_connections - @unicorn_active_connections ||= ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max) - end - - def unicorn_queued_connections - @unicorn_queued_connections ||= ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max) + def init_metrics + { + unicorn_active_connections: ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max), + unicorn_queued_connections: ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max), + unicorn_workers: ::Gitlab::Metrics.gauge(:unicorn_workers, 'Unicorn workers') + } end def enabled? @@ -23,14 +23,13 @@ module Gitlab def sample Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats| - unicorn_active_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.active) - unicorn_queued_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.queued) + set_unicorn_connection_metrics('tcp', addr, stats) end - Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats| - unicorn_active_connections.set({ socket_type: 'unix', socket_address: addr }, stats.active) - unicorn_queued_connections.set({ socket_type: 'unix', socket_address: addr }, stats.queued) + set_unicorn_connection_metrics('unix', addr, stats) end + + metrics[:unicorn_workers].set({}, unicorn_workers_count) end private @@ -39,6 +38,13 @@ module Gitlab @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z}) end + def set_unicorn_connection_metrics(type, addr, stats) + labels = { socket_type: type, socket_address: addr } + + metrics[:unicorn_active_connections].set(labels, stats.active) + metrics[:unicorn_queued_connections].set(labels, stats.queued) + end + def unix_listeners @unix_listeners ||= Unicorn.listener_names - tcp_listeners end @@ -46,6 +52,10 @@ module Gitlab def unicorn_with_listeners? defined?(Unicorn) && Unicorn.listener_names.any? end + + def unicorn_workers_count + `pgrep -f '[u]nicorn_rails worker.+ #{Rails.root.to_s}'`.split.count + end end end end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 426496855e3..33c0de91c11 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -23,6 +23,22 @@ module Gitlab def self.file_descriptor_count Dir.glob('/proc/self/fd/*').length end + + def self.max_open_file_descriptors + match = File.read('/proc/self/limits').match(/Max open files\s*(\d+)/) + + return unless match && match[1] + + match[1].to_i + end + + def self.process_start_time + fields = File.read('/proc/self/stat').split + + # fields[21] is linux proc stat field "(22) starttime". + # The value is expressed in clock ticks, divide by clock ticks for seconds. + ( fields[21].to_i || 0 ) / clk_tck + end else def self.memory_usage 0.0 @@ -31,6 +47,14 @@ module Gitlab def self.file_descriptor_count 0 end + + def self.max_open_file_descriptors + 0 + end + + def self.process_start_time + 0 + end end # THREAD_CPUTIME is not supported on OS X @@ -59,6 +83,10 @@ module Gitlab def self.monotonic_time Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) end + + def self.clk_tck + @clk_tck ||= `getconf CLK_TCK`.to_i + end end end end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index e91803ecd62..d7986685c65 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -8,6 +8,8 @@ module Gitlab # base labels shared among all transactions BASE_LABELS = { controller: nil, action: nil }.freeze + # labels that potentially contain sensitive information and will be filtered + FILTERED_LABELS = [:branch, :path].freeze THREAD_KEY = :_gitlab_metrics_transaction @@ -64,7 +66,7 @@ module Gitlab end def add_metric(series, values, tags = {}) - @metrics << Metric.new("#{::Gitlab::Metrics.series_prefix}#{series}", values, tags) + @metrics << Metric.new("#{::Gitlab::Metrics.series_prefix}#{series}", values, filter_tags(tags)) end # Tracks a business level event @@ -75,8 +77,9 @@ module Gitlab # event_name - The name of the event (e.g. "git_push"). # tags - A set of tags to attach to the event. def add_event(event_name, tags = {}) - self.class.transaction_metric(event_name, :counter, prefix: 'event_', use_feature_flag: true, tags: tags).increment(tags.merge(labels)) - @metrics << Metric.new(EVENT_SERIES, { count: 1 }, tags.merge(event: event_name), :event) + filtered_tags = filter_tags(tags) + self.class.transaction_metric(event_name, :counter, prefix: 'event_', use_feature_flag: true, tags: filtered_tags).increment(filtered_tags.merge(labels)) + @metrics << Metric.new(EVENT_SERIES, { count: 1 }, filtered_tags.merge(event: event_name), :event) end # Returns a MethodCall object for the given name. @@ -164,6 +167,12 @@ module Gitlab end end end + + private + + def filter_tags(tags) + tags.without(*FILTERED_LABELS) + end end end end |