diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-01-16 18:08:46 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-01-16 18:08:46 +0000 |
commit | aa0f0e992153e84e1cdec8a1c7310d5eb93a9f8f (patch) | |
tree | 4a662bc77fb43e1d1deec78cc7a95d911c0da1c5 /lib | |
parent | d47f9d2304dbc3a23bba7fe7a5cd07218eeb41cd (diff) | |
download | gitlab-ce-aa0f0e992153e84e1cdec8a1c7310d5eb93a9f8f.tar.gz |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
27 files changed, 537 insertions, 93 deletions
diff --git a/lib/api/applications.rb b/lib/api/applications.rb index 92717e04543..4e9843e17e8 100644 --- a/lib/api/applications.rb +++ b/lib/api/applications.rb @@ -38,7 +38,7 @@ module API application = ApplicationsFinder.new(params).execute application.destroy - status 204 + no_content! end end end diff --git a/lib/api/badges.rb b/lib/api/badges.rb index e987c24c707..d2152fad07b 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -135,7 +135,6 @@ module API end destroy_conditionally!(badge) - body false end end end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index ce3ee0d7e61..999bf1627c1 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -57,7 +57,7 @@ module API requires :branch, type: String, desc: 'The name of the branch' end head do - user_project.repository.branch_exists?(params[:branch]) ? status(204) : status(404) + user_project.repository.branch_exists?(params[:branch]) ? no_content! : not_found! end get do branch = find_branch!(params[:branch]) diff --git a/lib/api/custom_attributes_endpoints.rb b/lib/api/custom_attributes_endpoints.rb index 2149e04451e..ef1264126f4 100644 --- a/lib/api/custom_attributes_endpoints.rb +++ b/lib/api/custom_attributes_endpoints.rb @@ -77,7 +77,7 @@ module API resource.custom_attributes.find_by!(key: params[:key]).destroy - status 204 + no_content! end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/api/features.rb b/lib/api/features.rb index 4dc1834c644..69b751e9bdb 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -74,7 +74,7 @@ module API delete ':name' do Feature.get(params[:name]).remove - status 204 + no_content! end end end diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb index eae29f5b5dd..9e9f5101285 100644 --- a/lib/api/group_milestones.rb +++ b/lib/api/group_milestones.rb @@ -67,7 +67,7 @@ module API milestone = user_group.milestones.find(params[:milestone_id]) Milestones::DestroyService.new(user_group, current_user).execute(milestone) - status(204) + no_content! end desc 'Get all issues for a single group milestone' do diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 1fe2988ec1c..b2f5def4048 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -31,6 +31,7 @@ module API check_unmodified_since!(last_updated) status 204 + body false if block_given? yield resource diff --git a/lib/api/pages.rb b/lib/api/pages.rb index 39c8f1e6bdf..ee7fe669519 100644 --- a/lib/api/pages.rb +++ b/lib/api/pages.rb @@ -17,9 +17,9 @@ module API delete ':id/pages' do authorize! :remove_pages, user_project - status 204 - ::Pages::DeleteService.new(user_project, current_user).execute + + no_content! end end end diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index 9f8c1e4f916..4c3d2d131ac 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -148,8 +148,9 @@ module API delete ":id/pages/domains/:domain", requirements: PAGES_DOMAINS_ENDPOINT_REQUIREMENTS do authorize! :update_pages, user_project - status 204 pages_domain.destroy + + no_content! end end end diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index aebf7d5fae1..8643854a655 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -69,7 +69,7 @@ module API milestone = user_project.milestones.find(params[:milestone_id]) Milestones::DestroyService.new(user_project, current_user).execute(milestone) - status(204) + no_content! end desc 'Get all issues for a single project milestone' do diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 3e61b3c7f3b..2271131ced3 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -447,7 +447,7 @@ module API ::Projects::UnlinkForkService.new(user_project, current_user).execute end - result ? status(204) : not_modified! + not_modified! unless result end desc 'Share the project with a group' do diff --git a/lib/api/users.rb b/lib/api/users.rb index b8c60f1969c..bf1fe4fc4a8 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -346,8 +346,9 @@ module API key = user.gpg_keys.find_by(id: params[:key_id]) not_found!('GPG Key') unless key - status 204 key.destroy + + no_content! end # rubocop: enable CodeReuse/ActiveRecord @@ -760,8 +761,9 @@ module API key = current_user.gpg_keys.find_by(id: params[:key_id]) not_found!('GPG Key') unless key - status 204 key.destroy + + no_content! end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/variables.rb b/lib/api/variables.rb index f022b9e665a..192b06b8a1b 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -111,9 +111,10 @@ module API variable = user_project.variables.find_by(key: params[:key]) not_found!('Variable') unless variable - # Variables don't have any timestamp. Therfore, destroy unconditionally. - status 204 + # Variables don't have a timestamp. Therefore, destroy unconditionally. variable.destroy + + no_content! end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index eb36779e1d7..a2146406690 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -107,8 +107,9 @@ module API delete ':id/wikis/:slug' do authorize! :admin_wiki, user_project - status 204 WikiPages::DestroyService.new(user_project, current_user).execute(wiki_page) + + no_content! end desc 'Upload an attachment to the wiki repository' do diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb new file mode 100644 index 00000000000..c0748a4b8e6 --- /dev/null +++ b/lib/gitlab/danger/commit_linter.rb @@ -0,0 +1,232 @@ +# frozen_string_literal: true + +emoji_checker_path = File.expand_path('emoji_checker', __dir__) +defined?(Rails) ? require_dependency(emoji_checker_path) : require_relative(emoji_checker_path) + +module Gitlab + module Danger + class CommitLinter + MIN_SUBJECT_WORDS_COUNT = 3 + MAX_LINE_LENGTH = 72 + WARN_SUBJECT_LENGTH = 50 + URL_LIMIT_SUBJECT = "https://chris.beams.io/posts/git-commit/#limit-50" + MAX_CHANGED_FILES_IN_COMMIT = 3 + MAX_CHANGED_LINES_IN_COMMIT = 30 + SHORT_REFERENCE_REGEX = %r{([\w\-\/]+)?(#|!|&|%)\d+\b}.freeze + DEFAULT_SUBJECT_DESCRIPTION = 'commit subject' + PROBLEMS = { + subject_too_short: "The %s must contain at least #{MIN_SUBJECT_WORDS_COUNT} words", + subject_too_long: "The %s may not be longer than #{MAX_LINE_LENGTH} characters", + subject_above_warning: "The %s length is acceptable, but please try to [reduce it to #{WARN_SUBJECT_LENGTH} characters](#{URL_LIMIT_SUBJECT}).", + subject_starts_with_lowercase: "The %s must start with a capital letter", + subject_ends_with_a_period: "The %s must not end with a period", + separator_missing: "The commit subject and body must be separated by a blank line", + details_too_many_changes: "Commits that change #{MAX_CHANGED_LINES_IN_COMMIT} or more lines across " \ + "at least #{MAX_CHANGED_FILES_IN_COMMIT} files must describe these changes in the commit body", + details_line_too_long: "The commit body should not contain more than #{MAX_LINE_LENGTH} characters per line", + message_contains_text_emoji: "Avoid the use of Markdown Emoji such as `:+1:`. These add limited value " \ + "to the commit message, and are displayed as plain text outside of GitLab", + message_contains_unicode_emoji: "Avoid the use of Unicode Emoji. These add no value to the commit " \ + "message, and may not be displayed properly everywhere", + message_contains_short_reference: "Use full URLs instead of short references (`gitlab-org/gitlab#123` or " \ + "`!123`), as short references are displayed as plain text outside of GitLab" + }.freeze + + attr_reader :commit, :problems + + def initialize(commit) + @commit = commit + @problems = {} + @linted = false + end + + def fixup? + commit.message.start_with?('fixup!', 'squash!') + end + + def suggestion? + commit.message.start_with?('Apply suggestion to') + end + + def merge? + commit.message.start_with?('Merge branch') + end + + def revert? + commit.message.start_with?('Revert "') + end + + def multi_line? + !details.nil? && !details.empty? + end + + def failed? + problems.any? + end + + def add_problem(problem_key, *args) + @problems[problem_key] = sprintf(PROBLEMS[problem_key], *args) + end + + def lint(subject_description = "commit subject") + return self if @linted + + @linted = true + lint_subject(subject_description) + lint_separator + lint_details + lint_message + + self + end + + def lint_subject(subject_description) + if subject_too_short? + add_problem(:subject_too_short, subject_description) + end + + if subject_too_long? + add_problem(:subject_too_long, subject_description) + elsif subject_above_warning? + add_problem(:subject_above_warning, subject_description) + end + + if subject_starts_with_lowercase? + add_problem(:subject_starts_with_lowercase, subject_description) + end + + if subject_ends_with_a_period? + add_problem(:subject_ends_with_a_period, subject_description) + end + + self + end + + private + + def lint_separator + return self unless separator && !separator.empty? + + add_problem(:separator_missing) + + self + end + + def lint_details + if !multi_line? && many_changes? + add_problem(:details_too_many_changes) + end + + details&.each_line do |line| + line = line.strip + + next unless line_too_long?(line) + + url_size = line.scan(%r((https?://\S+))).sum { |(url)| url.length } # rubocop:disable CodeReuse/ActiveRecord + + # If the line includes a URL, we'll allow it to exceed MAX_LINE_LENGTH characters, but + # only if the line _without_ the URL does not exceed this limit. + next unless line_too_long?(line.length - url_size) + + add_problem(:details_line_too_long) + break + end + + self + end + + def lint_message + if message_contains_text_emoji? + add_problem(:message_contains_text_emoji) + end + + if message_contains_unicode_emoji? + add_problem(:message_contains_unicode_emoji) + end + + if message_contains_short_reference? + add_problem(:message_contains_short_reference) + end + + self + end + + def files_changed + commit.diff_parent.stats[:total][:files] + end + + def lines_changed + commit.diff_parent.stats[:total][:lines] + end + + def many_changes? + files_changed > MAX_CHANGED_FILES_IN_COMMIT && lines_changed > MAX_CHANGED_LINES_IN_COMMIT + end + + def subject + message_parts[0] + end + + def separator + message_parts[1] + end + + def details + message_parts[2] + end + + def line_too_long?(line) + case line + when String + line.length > MAX_LINE_LENGTH + when Integer + line > MAX_LINE_LENGTH + else + raise ArgumentError, "The line argument (#{line}) should be a String or an Integer! #{line.class} given." + end + end + + def subject_too_short? + subject.split(' ').length < MIN_SUBJECT_WORDS_COUNT + end + + def subject_too_long? + line_too_long?(subject) + end + + def subject_above_warning? + subject.length > WARN_SUBJECT_LENGTH + end + + def subject_starts_with_lowercase? + first_char = subject[0] + + first_char.downcase == first_char + end + + def subject_ends_with_a_period? + subject.end_with?('.') + end + + def message_contains_text_emoji? + emoji_checker.includes_text_emoji?(commit.message) + end + + def message_contains_unicode_emoji? + emoji_checker.includes_unicode_emoji?(commit.message) + end + + def message_contains_short_reference? + commit.message.match?(SHORT_REFERENCE_REGEX) + end + + def emoji_checker + @emoji_checker ||= Gitlab::Danger::EmojiChecker.new + end + + def message_parts + @message_parts ||= commit.message.split("\n", 3) + end + end + end +end diff --git a/lib/gitlab/danger/emoji_checker.rb b/lib/gitlab/danger/emoji_checker.rb new file mode 100644 index 00000000000..e31a6ae5011 --- /dev/null +++ b/lib/gitlab/danger/emoji_checker.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'json' + +module Gitlab + module Danger + class EmojiChecker + DIGESTS = File.expand_path('../../../fixtures/emojis/digests.json', __dir__) + ALIASES = File.expand_path('../../../fixtures/emojis/aliases.json', __dir__) + + # A regex that indicates a piece of text _might_ include an Emoji. The regex + # alone is not enough, as we'd match `:foo:bar:baz`. Instead, we use this + # regex to save us from having to check for all possible emoji names when we + # know one definitely is not included. + LIKELY_EMOJI = /:[\+a-z0-9_\-]+:/.freeze + + UNICODE_EMOJI_REGEX = %r{( + [\u{1F300}-\u{1F5FF}] | + [\u{1F1E6}-\u{1F1FF}] | + [\u{2700}-\u{27BF}] | + [\u{1F900}-\u{1F9FF}] | + [\u{1F600}-\u{1F64F}] | + [\u{1F680}-\u{1F6FF}] | + [\u{2600}-\u{26FF}] + )}x.freeze + + def initialize + names = JSON.parse(File.read(DIGESTS)).keys + + JSON.parse(File.read(ALIASES)).keys + + @emoji = names.map { |name| ":#{name}:" } + end + + def includes_text_emoji?(text) + return false unless text.match?(LIKELY_EMOJI) + + @emoji.any? { |emoji| text.include?(emoji) } + end + + def includes_unicode_emoji?(text) + text.match?(UNICODE_EMOJI_REGEX) + end + end + end +end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 90cef384a1b..5363533ace5 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -174,6 +174,10 @@ module Gitlab labels - current_mr_labels end + def sanitize_mr_title(title) + title.gsub(/^WIP: */, '').gsub(/`/, '\\\`') + end + def security_mr? return false unless gitlab_helper diff --git a/lib/gitlab/error_tracking/repo.rb b/lib/gitlab/error_tracking/repo.rb new file mode 100644 index 00000000000..50611943bac --- /dev/null +++ b/lib/gitlab/error_tracking/repo.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class Repo + attr_accessor :status, :integration_id, :project_id + + def initialize(status:, integration_id:, project_id:) + @status = status + @integration_id = integration_id + @project_id = project_id + end + end + end +end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 9a5e01462fb..43f3b673614 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -3,6 +3,8 @@ module Gitlab module ImportExport class RelationFactory + include Gitlab::Utils::StrongMemoize + prepend_if_ee('::EE::Gitlab::ImportExport::RelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule OVERRIDES = { snippets: :project_snippets, @@ -40,7 +42,7 @@ module Gitlab IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - EXISTING_OBJECT_CHECK = %i[ + EXISTING_OBJECT_RELATIONS = %i[ milestone milestones label @@ -58,9 +60,6 @@ module Gitlab TOKEN_RESET_MODELS = %i[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze - # This represents all relations that have unique key on `project_id` - UNIQUE_RELATIONS = %i[project_feature ProjectCiCdSetting container_expiration_policy].freeze - def self.create(*args) new(*args).create end @@ -115,12 +114,18 @@ module Gitlab OVERRIDES end - def self.existing_object_check - EXISTING_OBJECT_CHECK + def self.existing_object_relations + EXISTING_OBJECT_RELATIONS end private + def existing_object? + strong_memoize(:_existing_object) do + self.class.existing_object_relations.include?(@relation_name) || unique_relation? + end + end + def setup_models case @relation_name when :merge_request_diff_files then setup_diff @@ -229,7 +234,7 @@ module Gitlab end def update_group_references - return unless self.class.existing_object_check.include?(@relation_name) + return unless existing_object? return unless @relation_hash['group_id'] @relation_hash['group_id'] = @project.namespace_id @@ -322,7 +327,7 @@ module Gitlab # Only find existing records to avoid mapping tables such as milestones # Otherwise always create the record, skipping the extra SELECT clause. @existing_or_new_object ||= begin - if self.class.existing_object_check.include?(@relation_name) + if existing_object? attribute_hash = attribute_hash_for(['events']) existing_object.assign_attributes(attribute_hash) if attribute_hash.any? @@ -356,8 +361,43 @@ module Gitlab !Object.const_defined?(parsed_relation_hash['type']) end + def unique_relation? + strong_memoize(:unique_relation) do + project_foreign_key.present? && + (has_unique_index_on_project_fk? || uses_project_fk_as_primary_key?) + end + end + + def has_unique_index_on_project_fk? + cache = cached_has_unique_index_on_project_fk + table_name = relation_class.table_name + return cache[table_name] if cache.has_key?(table_name) + + index_exists = + ActiveRecord::Base.connection.index_exists?( + relation_class.table_name, + project_foreign_key, + unique: true) + + cache[table_name] = index_exists + end + + # Avoid unnecessary DB requests + def cached_has_unique_index_on_project_fk + Thread.current[:cached_has_unique_index_on_project_fk] ||= {} + end + + def uses_project_fk_as_primary_key? + relation_class.primary_key == project_foreign_key + end + + # Should be `:project_id` for most of the cases, but this is more general + def project_foreign_key + relation_class.reflect_on_association(:project)&.foreign_key + end + def find_or_create_object! - if UNIQUE_RELATIONS.include?(@relation_name) + if unique_relation? unique_relation_object = relation_class.find_or_create_by(project_id: @project.id) unique_relation_object.assign_attributes(parsed_relation_hash) diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index beceed3fa75..c8932b26925 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -22,11 +22,8 @@ module Gitlab def pool_size # heuristic constant 5 should be a config setting somewhere -- related to CPU count? size = 5 - if Gitlab::Runtime.sidekiq? - # the pool will be used in a multi-threaded context - size += Sidekiq.options[:concurrency] - elsif Gitlab::Runtime.puma? - size += Puma.cli_config.options[:max_threads] + if Gitlab::Runtime.multi_threaded? + size += Gitlab::Runtime.max_threads end size diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index 4893cbc1f45..3dda244233f 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -10,7 +10,7 @@ module Gitlab def self.server_configurator(metrics: true, arguments_logger: true, memory_killer: true, request_store: true) lambda do |chain| chain.add Gitlab::SidekiqMiddleware::Monitor - chain.add Gitlab::SidekiqMiddleware::Metrics if metrics + chain.add Gitlab::SidekiqMiddleware::ServerMetrics if metrics chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if arguments_logger chain.add Gitlab::SidekiqMiddleware::MemoryKiller if memory_killer chain.add Gitlab::SidekiqMiddleware::RequestStoreMiddleware if request_store @@ -27,6 +27,7 @@ module Gitlab def self.client_configurator lambda do |chain| chain.add Gitlab::SidekiqStatus::ClientMiddleware + chain.add Gitlab::SidekiqMiddleware::ClientMetrics chain.add Labkit::Middleware::Sidekiq::Client end end diff --git a/lib/gitlab/sidekiq_middleware/client_metrics.rb b/lib/gitlab/sidekiq_middleware/client_metrics.rb new file mode 100644 index 00000000000..cd11415b55e --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/client_metrics.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class ClientMetrics < SidekiqMiddleware::Metrics + ENQUEUED = :sidekiq_enqueued_jobs_total + + def initialize + @metrics = init_metrics + end + + def call(worker, _job, queue, _redis_pool) + labels = create_labels(worker.class, queue) + + @metrics.fetch(ENQUEUED).increment(labels, 1) + + yield + end + + private + + def init_metrics + { + ENQUEUED => ::Gitlab::Metrics.counter(ENQUEUED, 'Sidekiq jobs enqueued') + } + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/metrics.rb b/lib/gitlab/sidekiq_middleware/metrics.rb index 7bfb0d54d80..9588e9ef19a 100644 --- a/lib/gitlab/sidekiq_middleware/metrics.rb +++ b/lib/gitlab/sidekiq_middleware/metrics.rb @@ -3,68 +3,11 @@ module Gitlab module SidekiqMiddleware class Metrics - # SIDEKIQ_LATENCY_BUCKETS are latency histogram buckets better suited to Sidekiq - # timeframes than the DEFAULT_BUCKET definition. Defined in seconds. - SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60, 300, 600].freeze - TRUE_LABEL = "yes" FALSE_LABEL = "no" - def initialize - @metrics = init_metrics - - @metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i) - end - - def call(worker, job, queue) - labels = create_labels(worker.class, queue) - queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) - - @metrics[:sidekiq_jobs_queue_duration_seconds].observe(labels, queue_duration) if queue_duration - @metrics[:sidekiq_running_jobs].increment(labels, 1) - - if job['retry_count'].present? - @metrics[:sidekiq_jobs_retried_total].increment(labels, 1) - end - - job_succeeded = false - monotonic_time_start = Gitlab::Metrics::System.monotonic_time - job_thread_cputime_start = get_thread_cputime - begin - yield - job_succeeded = true - ensure - monotonic_time_end = Gitlab::Metrics::System.monotonic_time - job_thread_cputime_end = get_thread_cputime - - monotonic_time = monotonic_time_end - monotonic_time_start - job_thread_cputime = job_thread_cputime_end - job_thread_cputime_start - - # sidekiq_running_jobs, sidekiq_jobs_failed_total should not include the job_status label - @metrics[:sidekiq_running_jobs].increment(labels, -1) - @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) unless job_succeeded - - # job_status: done, fail match the job_status attribute in structured logging - labels[:job_status] = job_succeeded ? "done" : "fail" - @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime) - @metrics[:sidekiq_jobs_completion_seconds].observe(labels, monotonic_time) - end - end - private - def init_metrics - { - sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), - sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), - sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all), - sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all) - } - end - def create_labels(worker_class, queue) labels = { queue: queue.to_s, latency_sensitive: FALSE_LABEL, external_dependencies: FALSE_LABEL, feature_category: "", boundary: "" } return labels unless worker_class.include? WorkerAttributes @@ -84,10 +27,6 @@ module Gitlab def bool_as_label(value) value ? TRUE_LABEL : FALSE_LABEL end - - def get_thread_cputime - defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0 - end end end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb new file mode 100644 index 00000000000..fa7f56b8d9c --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class ServerMetrics < SidekiqMiddleware::Metrics + # SIDEKIQ_LATENCY_BUCKETS are latency histogram buckets better suited to Sidekiq + # timeframes than the DEFAULT_BUCKET definition. Defined in seconds. + SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60, 300, 600].freeze + + def initialize + @metrics = init_metrics + + @metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i) + end + + def call(worker, job, queue) + labels = create_labels(worker.class, queue) + queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) + + @metrics[:sidekiq_jobs_queue_duration_seconds].observe(labels, queue_duration) if queue_duration + @metrics[:sidekiq_running_jobs].increment(labels, 1) + + if job['retry_count'].present? + @metrics[:sidekiq_jobs_retried_total].increment(labels, 1) + end + + job_succeeded = false + monotonic_time_start = Gitlab::Metrics::System.monotonic_time + job_thread_cputime_start = get_thread_cputime + begin + yield + job_succeeded = true + ensure + monotonic_time_end = Gitlab::Metrics::System.monotonic_time + job_thread_cputime_end = get_thread_cputime + + monotonic_time = monotonic_time_end - monotonic_time_start + job_thread_cputime = job_thread_cputime_end - job_thread_cputime_start + + # sidekiq_running_jobs, sidekiq_jobs_failed_total should not include the job_status label + @metrics[:sidekiq_running_jobs].increment(labels, -1) + @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) unless job_succeeded + + # job_status: done, fail match the job_status attribute in structured logging + labels[:job_status] = job_succeeded ? "done" : "fail" + @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime) + @metrics[:sidekiq_jobs_completion_seconds].observe(labels, monotonic_time) + end + end + + private + + def init_metrics + { + sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), + sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), + sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all), + sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all) + } + end + + def get_thread_cputime + defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0 + end + end + end +end diff --git a/lib/sentry/client.rb b/lib/sentry/client.rb index 490f82c4678..8898960c24d 100644 --- a/lib/sentry/client.rb +++ b/lib/sentry/client.rb @@ -5,6 +5,8 @@ module Sentry include Sentry::Client::Event include Sentry::Client::Projects include Sentry::Client::Issue + include Sentry::Client::Repo + include Sentry::Client::IssueLink Error = Class.new(StandardError) MissingKeysError = Class.new(StandardError) @@ -79,7 +81,7 @@ module Sentry end def handle_response(response) - unless response.code == 200 + unless response.code.between?(200, 204) raise_error "Sentry response status code: #{response.code}" end diff --git a/lib/sentry/client/issue_link.rb b/lib/sentry/client/issue_link.rb new file mode 100644 index 00000000000..200b1a6b435 --- /dev/null +++ b/lib/sentry/client/issue_link.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Sentry + class Client + module IssueLink + def create_issue_link(integration_id, sentry_issue_identifier, issue) + issue_link_url = issue_link_api_url(integration_id, sentry_issue_identifier) + + params = { + project: issue.project.id, + externalIssue: "#{issue.project.id}##{issue.iid}" + } + + http_put(issue_link_url, params) + end + + private + + def issue_link_api_url(integration_id, sentry_issue_identifier) + issue_link_url = URI(url) + issue_link_url.path = "/api/0/groups/#{sentry_issue_identifier}/integrations/#{integration_id}/" + + issue_link_url + end + end + end +end diff --git a/lib/sentry/client/repo.rb b/lib/sentry/client/repo.rb new file mode 100644 index 00000000000..9a0ed3c7342 --- /dev/null +++ b/lib/sentry/client/repo.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Sentry + class Client + module Repo + def repos(organization_slug) + repos_url = repos_api_url(organization_slug) + + repos = http_get(repos_url)[:body] + + handle_mapping_exceptions do + map_to_repos(repos) + end + end + + private + + def repos_api_url(organization_slug) + repos_url = URI(url) + repos_url.path = "/api/0/organizations/#{organization_slug}/repos/" + + repos_url + end + + def map_to_repos(repos) + repos.map(&method(:map_to_repo)) + end + + def map_to_repo(repo) + Gitlab::ErrorTracking::Repo.new( + status: repo.fetch('status'), + integration_id: repo.fetch('integrationId'), + project_id: repo.fetch('externalSlug') + ) + end + end + end +end |