summaryrefslogtreecommitdiff
path: root/lib/gitlab
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2020-01-16 18:08:46 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2020-01-16 18:08:46 +0000
commitaa0f0e992153e84e1cdec8a1c7310d5eb93a9f8f (patch)
tree4a662bc77fb43e1d1deec78cc7a95d911c0da1c5 /lib/gitlab
parentd47f9d2304dbc3a23bba7fe7a5cd07218eeb41cd (diff)
downloadgitlab-ce-aa0f0e992153e84e1cdec8a1c7310d5eb93a9f8f.tar.gz
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib/gitlab')
-rw-r--r--lib/gitlab/danger/commit_linter.rb232
-rw-r--r--lib/gitlab/danger/emoji_checker.rb45
-rw-r--r--lib/gitlab/danger/helper.rb4
-rw-r--r--lib/gitlab/error_tracking/repo.rb15
-rw-r--r--lib/gitlab/import_export/relation_factory.rb58
-rw-r--r--lib/gitlab/redis/wrapper.rb7
-rw-r--r--lib/gitlab/sidekiq_middleware.rb3
-rw-r--r--lib/gitlab/sidekiq_middleware/client_metrics.rb29
-rw-r--r--lib/gitlab/sidekiq_middleware/metrics.rb61
-rw-r--r--lib/gitlab/sidekiq_middleware/server_metrics.rb70
10 files changed, 448 insertions, 76 deletions
diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb
new file mode 100644
index 00000000000..c0748a4b8e6
--- /dev/null
+++ b/lib/gitlab/danger/commit_linter.rb
@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+emoji_checker_path = File.expand_path('emoji_checker', __dir__)
+defined?(Rails) ? require_dependency(emoji_checker_path) : require_relative(emoji_checker_path)
+
+module Gitlab
+ module Danger
+ class CommitLinter
+ MIN_SUBJECT_WORDS_COUNT = 3
+ MAX_LINE_LENGTH = 72
+ WARN_SUBJECT_LENGTH = 50
+ URL_LIMIT_SUBJECT = "https://chris.beams.io/posts/git-commit/#limit-50"
+ MAX_CHANGED_FILES_IN_COMMIT = 3
+ MAX_CHANGED_LINES_IN_COMMIT = 30
+ SHORT_REFERENCE_REGEX = %r{([\w\-\/]+)?(#|!|&|%)\d+\b}.freeze
+ DEFAULT_SUBJECT_DESCRIPTION = 'commit subject'
+ PROBLEMS = {
+ subject_too_short: "The %s must contain at least #{MIN_SUBJECT_WORDS_COUNT} words",
+ subject_too_long: "The %s may not be longer than #{MAX_LINE_LENGTH} characters",
+ subject_above_warning: "The %s length is acceptable, but please try to [reduce it to #{WARN_SUBJECT_LENGTH} characters](#{URL_LIMIT_SUBJECT}).",
+ subject_starts_with_lowercase: "The %s must start with a capital letter",
+ subject_ends_with_a_period: "The %s must not end with a period",
+ separator_missing: "The commit subject and body must be separated by a blank line",
+ details_too_many_changes: "Commits that change #{MAX_CHANGED_LINES_IN_COMMIT} or more lines across " \
+ "at least #{MAX_CHANGED_FILES_IN_COMMIT} files must describe these changes in the commit body",
+ details_line_too_long: "The commit body should not contain more than #{MAX_LINE_LENGTH} characters per line",
+ message_contains_text_emoji: "Avoid the use of Markdown Emoji such as `:+1:`. These add limited value " \
+ "to the commit message, and are displayed as plain text outside of GitLab",
+ message_contains_unicode_emoji: "Avoid the use of Unicode Emoji. These add no value to the commit " \
+ "message, and may not be displayed properly everywhere",
+ message_contains_short_reference: "Use full URLs instead of short references (`gitlab-org/gitlab#123` or " \
+ "`!123`), as short references are displayed as plain text outside of GitLab"
+ }.freeze
+
+ attr_reader :commit, :problems
+
+ def initialize(commit)
+ @commit = commit
+ @problems = {}
+ @linted = false
+ end
+
+ def fixup?
+ commit.message.start_with?('fixup!', 'squash!')
+ end
+
+ def suggestion?
+ commit.message.start_with?('Apply suggestion to')
+ end
+
+ def merge?
+ commit.message.start_with?('Merge branch')
+ end
+
+ def revert?
+ commit.message.start_with?('Revert "')
+ end
+
+ def multi_line?
+ !details.nil? && !details.empty?
+ end
+
+ def failed?
+ problems.any?
+ end
+
+ def add_problem(problem_key, *args)
+ @problems[problem_key] = sprintf(PROBLEMS[problem_key], *args)
+ end
+
+ def lint(subject_description = "commit subject")
+ return self if @linted
+
+ @linted = true
+ lint_subject(subject_description)
+ lint_separator
+ lint_details
+ lint_message
+
+ self
+ end
+
+ def lint_subject(subject_description)
+ if subject_too_short?
+ add_problem(:subject_too_short, subject_description)
+ end
+
+ if subject_too_long?
+ add_problem(:subject_too_long, subject_description)
+ elsif subject_above_warning?
+ add_problem(:subject_above_warning, subject_description)
+ end
+
+ if subject_starts_with_lowercase?
+ add_problem(:subject_starts_with_lowercase, subject_description)
+ end
+
+ if subject_ends_with_a_period?
+ add_problem(:subject_ends_with_a_period, subject_description)
+ end
+
+ self
+ end
+
+ private
+
+ def lint_separator
+ return self unless separator && !separator.empty?
+
+ add_problem(:separator_missing)
+
+ self
+ end
+
+ def lint_details
+ if !multi_line? && many_changes?
+ add_problem(:details_too_many_changes)
+ end
+
+ details&.each_line do |line|
+ line = line.strip
+
+ next unless line_too_long?(line)
+
+ url_size = line.scan(%r((https?://\S+))).sum { |(url)| url.length } # rubocop:disable CodeReuse/ActiveRecord
+
+ # If the line includes a URL, we'll allow it to exceed MAX_LINE_LENGTH characters, but
+ # only if the line _without_ the URL does not exceed this limit.
+ next unless line_too_long?(line.length - url_size)
+
+ add_problem(:details_line_too_long)
+ break
+ end
+
+ self
+ end
+
+ def lint_message
+ if message_contains_text_emoji?
+ add_problem(:message_contains_text_emoji)
+ end
+
+ if message_contains_unicode_emoji?
+ add_problem(:message_contains_unicode_emoji)
+ end
+
+ if message_contains_short_reference?
+ add_problem(:message_contains_short_reference)
+ end
+
+ self
+ end
+
+ def files_changed
+ commit.diff_parent.stats[:total][:files]
+ end
+
+ def lines_changed
+ commit.diff_parent.stats[:total][:lines]
+ end
+
+ def many_changes?
+ files_changed > MAX_CHANGED_FILES_IN_COMMIT && lines_changed > MAX_CHANGED_LINES_IN_COMMIT
+ end
+
+ def subject
+ message_parts[0]
+ end
+
+ def separator
+ message_parts[1]
+ end
+
+ def details
+ message_parts[2]
+ end
+
+ def line_too_long?(line)
+ case line
+ when String
+ line.length > MAX_LINE_LENGTH
+ when Integer
+ line > MAX_LINE_LENGTH
+ else
+ raise ArgumentError, "The line argument (#{line}) should be a String or an Integer! #{line.class} given."
+ end
+ end
+
+ def subject_too_short?
+ subject.split(' ').length < MIN_SUBJECT_WORDS_COUNT
+ end
+
+ def subject_too_long?
+ line_too_long?(subject)
+ end
+
+ def subject_above_warning?
+ subject.length > WARN_SUBJECT_LENGTH
+ end
+
+ def subject_starts_with_lowercase?
+ first_char = subject[0]
+
+ first_char.downcase == first_char
+ end
+
+ def subject_ends_with_a_period?
+ subject.end_with?('.')
+ end
+
+ def message_contains_text_emoji?
+ emoji_checker.includes_text_emoji?(commit.message)
+ end
+
+ def message_contains_unicode_emoji?
+ emoji_checker.includes_unicode_emoji?(commit.message)
+ end
+
+ def message_contains_short_reference?
+ commit.message.match?(SHORT_REFERENCE_REGEX)
+ end
+
+ def emoji_checker
+ @emoji_checker ||= Gitlab::Danger::EmojiChecker.new
+ end
+
+ def message_parts
+ @message_parts ||= commit.message.split("\n", 3)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/danger/emoji_checker.rb b/lib/gitlab/danger/emoji_checker.rb
new file mode 100644
index 00000000000..e31a6ae5011
--- /dev/null
+++ b/lib/gitlab/danger/emoji_checker.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Gitlab
+ module Danger
+ class EmojiChecker
+ DIGESTS = File.expand_path('../../../fixtures/emojis/digests.json', __dir__)
+ ALIASES = File.expand_path('../../../fixtures/emojis/aliases.json', __dir__)
+
+ # A regex that indicates a piece of text _might_ include an Emoji. The regex
+ # alone is not enough, as we'd match `:foo:bar:baz`. Instead, we use this
+ # regex to save us from having to check for all possible emoji names when we
+ # know one definitely is not included.
+ LIKELY_EMOJI = /:[\+a-z0-9_\-]+:/.freeze
+
+ UNICODE_EMOJI_REGEX = %r{(
+ [\u{1F300}-\u{1F5FF}] |
+ [\u{1F1E6}-\u{1F1FF}] |
+ [\u{2700}-\u{27BF}] |
+ [\u{1F900}-\u{1F9FF}] |
+ [\u{1F600}-\u{1F64F}] |
+ [\u{1F680}-\u{1F6FF}] |
+ [\u{2600}-\u{26FF}]
+ )}x.freeze
+
+ def initialize
+ names = JSON.parse(File.read(DIGESTS)).keys +
+ JSON.parse(File.read(ALIASES)).keys
+
+ @emoji = names.map { |name| ":#{name}:" }
+ end
+
+ def includes_text_emoji?(text)
+ return false unless text.match?(LIKELY_EMOJI)
+
+ @emoji.any? { |emoji| text.include?(emoji) }
+ end
+
+ def includes_unicode_emoji?(text)
+ text.match?(UNICODE_EMOJI_REGEX)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb
index 90cef384a1b..5363533ace5 100644
--- a/lib/gitlab/danger/helper.rb
+++ b/lib/gitlab/danger/helper.rb
@@ -174,6 +174,10 @@ module Gitlab
labels - current_mr_labels
end
+ def sanitize_mr_title(title)
+ title.gsub(/^WIP: */, '').gsub(/`/, '\\\`')
+ end
+
def security_mr?
return false unless gitlab_helper
diff --git a/lib/gitlab/error_tracking/repo.rb b/lib/gitlab/error_tracking/repo.rb
new file mode 100644
index 00000000000..50611943bac
--- /dev/null
+++ b/lib/gitlab/error_tracking/repo.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ErrorTracking
+ class Repo
+ attr_accessor :status, :integration_id, :project_id
+
+ def initialize(status:, integration_id:, project_id:)
+ @status = status
+ @integration_id = integration_id
+ @project_id = project_id
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index 9a5e01462fb..43f3b673614 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -3,6 +3,8 @@
module Gitlab
module ImportExport
class RelationFactory
+ include Gitlab::Utils::StrongMemoize
+
prepend_if_ee('::EE::Gitlab::ImportExport::RelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule
OVERRIDES = { snippets: :project_snippets,
@@ -40,7 +42,7 @@ module Gitlab
IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
- EXISTING_OBJECT_CHECK = %i[
+ EXISTING_OBJECT_RELATIONS = %i[
milestone
milestones
label
@@ -58,9 +60,6 @@ module Gitlab
TOKEN_RESET_MODELS = %i[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze
- # This represents all relations that have unique key on `project_id`
- UNIQUE_RELATIONS = %i[project_feature ProjectCiCdSetting container_expiration_policy].freeze
-
def self.create(*args)
new(*args).create
end
@@ -115,12 +114,18 @@ module Gitlab
OVERRIDES
end
- def self.existing_object_check
- EXISTING_OBJECT_CHECK
+ def self.existing_object_relations
+ EXISTING_OBJECT_RELATIONS
end
private
+ def existing_object?
+ strong_memoize(:_existing_object) do
+ self.class.existing_object_relations.include?(@relation_name) || unique_relation?
+ end
+ end
+
def setup_models
case @relation_name
when :merge_request_diff_files then setup_diff
@@ -229,7 +234,7 @@ module Gitlab
end
def update_group_references
- return unless self.class.existing_object_check.include?(@relation_name)
+ return unless existing_object?
return unless @relation_hash['group_id']
@relation_hash['group_id'] = @project.namespace_id
@@ -322,7 +327,7 @@ module Gitlab
# Only find existing records to avoid mapping tables such as milestones
# Otherwise always create the record, skipping the extra SELECT clause.
@existing_or_new_object ||= begin
- if self.class.existing_object_check.include?(@relation_name)
+ if existing_object?
attribute_hash = attribute_hash_for(['events'])
existing_object.assign_attributes(attribute_hash) if attribute_hash.any?
@@ -356,8 +361,43 @@ module Gitlab
!Object.const_defined?(parsed_relation_hash['type'])
end
+ def unique_relation?
+ strong_memoize(:unique_relation) do
+ project_foreign_key.present? &&
+ (has_unique_index_on_project_fk? || uses_project_fk_as_primary_key?)
+ end
+ end
+
+ def has_unique_index_on_project_fk?
+ cache = cached_has_unique_index_on_project_fk
+ table_name = relation_class.table_name
+ return cache[table_name] if cache.has_key?(table_name)
+
+ index_exists =
+ ActiveRecord::Base.connection.index_exists?(
+ relation_class.table_name,
+ project_foreign_key,
+ unique: true)
+
+ cache[table_name] = index_exists
+ end
+
+ # Avoid unnecessary DB requests
+ def cached_has_unique_index_on_project_fk
+ Thread.current[:cached_has_unique_index_on_project_fk] ||= {}
+ end
+
+ def uses_project_fk_as_primary_key?
+ relation_class.primary_key == project_foreign_key
+ end
+
+ # Should be `:project_id` for most of the cases, but this is more general
+ def project_foreign_key
+ relation_class.reflect_on_association(:project)&.foreign_key
+ end
+
def find_or_create_object!
- if UNIQUE_RELATIONS.include?(@relation_name)
+ if unique_relation?
unique_relation_object = relation_class.find_or_create_by(project_id: @project.id)
unique_relation_object.assign_attributes(parsed_relation_hash)
diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb
index beceed3fa75..c8932b26925 100644
--- a/lib/gitlab/redis/wrapper.rb
+++ b/lib/gitlab/redis/wrapper.rb
@@ -22,11 +22,8 @@ module Gitlab
def pool_size
# heuristic constant 5 should be a config setting somewhere -- related to CPU count?
size = 5
- if Gitlab::Runtime.sidekiq?
- # the pool will be used in a multi-threaded context
- size += Sidekiq.options[:concurrency]
- elsif Gitlab::Runtime.puma?
- size += Puma.cli_config.options[:max_threads]
+ if Gitlab::Runtime.multi_threaded?
+ size += Gitlab::Runtime.max_threads
end
size
diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb
index 4893cbc1f45..3dda244233f 100644
--- a/lib/gitlab/sidekiq_middleware.rb
+++ b/lib/gitlab/sidekiq_middleware.rb
@@ -10,7 +10,7 @@ module Gitlab
def self.server_configurator(metrics: true, arguments_logger: true, memory_killer: true, request_store: true)
lambda do |chain|
chain.add Gitlab::SidekiqMiddleware::Monitor
- chain.add Gitlab::SidekiqMiddleware::Metrics if metrics
+ chain.add Gitlab::SidekiqMiddleware::ServerMetrics if metrics
chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if arguments_logger
chain.add Gitlab::SidekiqMiddleware::MemoryKiller if memory_killer
chain.add Gitlab::SidekiqMiddleware::RequestStoreMiddleware if request_store
@@ -27,6 +27,7 @@ module Gitlab
def self.client_configurator
lambda do |chain|
chain.add Gitlab::SidekiqStatus::ClientMiddleware
+ chain.add Gitlab::SidekiqMiddleware::ClientMetrics
chain.add Labkit::Middleware::Sidekiq::Client
end
end
diff --git a/lib/gitlab/sidekiq_middleware/client_metrics.rb b/lib/gitlab/sidekiq_middleware/client_metrics.rb
new file mode 100644
index 00000000000..cd11415b55e
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/client_metrics.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module SidekiqMiddleware
+ class ClientMetrics < SidekiqMiddleware::Metrics
+ ENQUEUED = :sidekiq_enqueued_jobs_total
+
+ def initialize
+ @metrics = init_metrics
+ end
+
+ def call(worker, _job, queue, _redis_pool)
+ labels = create_labels(worker.class, queue)
+
+ @metrics.fetch(ENQUEUED).increment(labels, 1)
+
+ yield
+ end
+
+ private
+
+ def init_metrics
+ {
+ ENQUEUED => ::Gitlab::Metrics.counter(ENQUEUED, 'Sidekiq jobs enqueued')
+ }
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/sidekiq_middleware/metrics.rb b/lib/gitlab/sidekiq_middleware/metrics.rb
index 7bfb0d54d80..9588e9ef19a 100644
--- a/lib/gitlab/sidekiq_middleware/metrics.rb
+++ b/lib/gitlab/sidekiq_middleware/metrics.rb
@@ -3,68 +3,11 @@
module Gitlab
module SidekiqMiddleware
class Metrics
- # SIDEKIQ_LATENCY_BUCKETS are latency histogram buckets better suited to Sidekiq
- # timeframes than the DEFAULT_BUCKET definition. Defined in seconds.
- SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60, 300, 600].freeze
-
TRUE_LABEL = "yes"
FALSE_LABEL = "no"
- def initialize
- @metrics = init_metrics
-
- @metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i)
- end
-
- def call(worker, job, queue)
- labels = create_labels(worker.class, queue)
- queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job)
-
- @metrics[:sidekiq_jobs_queue_duration_seconds].observe(labels, queue_duration) if queue_duration
- @metrics[:sidekiq_running_jobs].increment(labels, 1)
-
- if job['retry_count'].present?
- @metrics[:sidekiq_jobs_retried_total].increment(labels, 1)
- end
-
- job_succeeded = false
- monotonic_time_start = Gitlab::Metrics::System.monotonic_time
- job_thread_cputime_start = get_thread_cputime
- begin
- yield
- job_succeeded = true
- ensure
- monotonic_time_end = Gitlab::Metrics::System.monotonic_time
- job_thread_cputime_end = get_thread_cputime
-
- monotonic_time = monotonic_time_end - monotonic_time_start
- job_thread_cputime = job_thread_cputime_end - job_thread_cputime_start
-
- # sidekiq_running_jobs, sidekiq_jobs_failed_total should not include the job_status label
- @metrics[:sidekiq_running_jobs].increment(labels, -1)
- @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) unless job_succeeded
-
- # job_status: done, fail match the job_status attribute in structured logging
- labels[:job_status] = job_succeeded ? "done" : "fail"
- @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime)
- @metrics[:sidekiq_jobs_completion_seconds].observe(labels, monotonic_time)
- end
- end
-
private
- def init_metrics
- {
- sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
- sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
- sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS),
- sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'),
- sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'),
- sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all),
- sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all)
- }
- end
-
def create_labels(worker_class, queue)
labels = { queue: queue.to_s, latency_sensitive: FALSE_LABEL, external_dependencies: FALSE_LABEL, feature_category: "", boundary: "" }
return labels unless worker_class.include? WorkerAttributes
@@ -84,10 +27,6 @@ module Gitlab
def bool_as_label(value)
value ? TRUE_LABEL : FALSE_LABEL
end
-
- def get_thread_cputime
- defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0
- end
end
end
end
diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb
new file mode 100644
index 00000000000..fa7f56b8d9c
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb
@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module SidekiqMiddleware
+ class ServerMetrics < SidekiqMiddleware::Metrics
+ # SIDEKIQ_LATENCY_BUCKETS are latency histogram buckets better suited to Sidekiq
+ # timeframes than the DEFAULT_BUCKET definition. Defined in seconds.
+ SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60, 300, 600].freeze
+
+ def initialize
+ @metrics = init_metrics
+
+ @metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i)
+ end
+
+ def call(worker, job, queue)
+ labels = create_labels(worker.class, queue)
+ queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job)
+
+ @metrics[:sidekiq_jobs_queue_duration_seconds].observe(labels, queue_duration) if queue_duration
+ @metrics[:sidekiq_running_jobs].increment(labels, 1)
+
+ if job['retry_count'].present?
+ @metrics[:sidekiq_jobs_retried_total].increment(labels, 1)
+ end
+
+ job_succeeded = false
+ monotonic_time_start = Gitlab::Metrics::System.monotonic_time
+ job_thread_cputime_start = get_thread_cputime
+ begin
+ yield
+ job_succeeded = true
+ ensure
+ monotonic_time_end = Gitlab::Metrics::System.monotonic_time
+ job_thread_cputime_end = get_thread_cputime
+
+ monotonic_time = monotonic_time_end - monotonic_time_start
+ job_thread_cputime = job_thread_cputime_end - job_thread_cputime_start
+
+ # sidekiq_running_jobs, sidekiq_jobs_failed_total should not include the job_status label
+ @metrics[:sidekiq_running_jobs].increment(labels, -1)
+ @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) unless job_succeeded
+
+ # job_status: done, fail match the job_status attribute in structured logging
+ labels[:job_status] = job_succeeded ? "done" : "fail"
+ @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime)
+ @metrics[:sidekiq_jobs_completion_seconds].observe(labels, monotonic_time)
+ end
+ end
+
+ private
+
+ def init_metrics
+ {
+ sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
+ sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
+ sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS),
+ sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'),
+ sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'),
+ sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all),
+ sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all)
+ }
+ end
+
+ def get_thread_cputime
+ defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0
+ end
+ end
+ end
+end