diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-12 15:13:54 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-12 15:13:54 +0000 |
commit | 98638cd5e43611aac2193a5c2f80f72374040430 (patch) | |
tree | 6605f0f284efed1d05708b3799f093eb5e305a8f /lib | |
parent | 43d816ebc20da6ff959176248c70d8c4c7c9345a (diff) | |
download | gitlab-ce-98638cd5e43611aac2193a5c2f80f72374040430.tar.gz |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
34 files changed, 210 insertions, 233 deletions
diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index d8f4b228797..0247ce301e2 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -11,7 +11,7 @@ module API desc 'Register a new runner' do detail "Register a new runner for the instance" success Entities::Ci::RunnerRegistrationDetails - failure [[400, 'Bad Request'], [403, 'Forbidden']] + failure [[400, 'Bad Request'], [403, 'Forbidden'], [410, 'Gone']] end params do requires :token, type: String, desc: 'Registration token' @@ -52,9 +52,17 @@ module API attributes[:active] = !attributes.delete(:paused) if attributes.include?(:paused) result = ::Ci::Runners::RegisterRunnerService.new(params[:token], attributes).execute - @runner = result.success? ? result.payload[:runner] : nil - forbidden!(result.message) unless @runner + if result.error? + case result.reason + when :runner_registration_disallowed + render_api_error_with_reason!(410, '410 Gone', result.message) + else + forbidden!(result.message) + end + end + + @runner = result.payload[:runner] if @runner.persisted? present @runner, with: Entities::Ci::RunnerRegistrationDetails else diff --git a/lib/feature/logger.rb b/lib/feature/logger.rb index 95e160273b6..337d3a4ccc2 100644 --- a/lib/feature/logger.rb +++ b/lib/feature/logger.rb @@ -2,6 +2,8 @@ module Feature class Logger < ::Gitlab::JsonLogger + exclude_context! + def self.file_name_noext 'features_json' end diff --git a/lib/gitlab/background_migration/logger.rb b/lib/gitlab/background_migration/logger.rb index 4ea89771eff..d338c214140 100644 --- a/lib/gitlab/background_migration/logger.rb +++ b/lib/gitlab/background_migration/logger.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # Logger that can be used for migrations logging class Logger < ::Gitlab::JsonLogger + exclude_context! + def self.file_name_noext 'migrations' end diff --git a/lib/gitlab/backup_logger.rb b/lib/gitlab/backup_logger.rb index fad36b860ae..ec85c55d4a4 100644 --- a/lib/gitlab/backup_logger.rb +++ b/lib/gitlab/backup_logger.rb @@ -2,6 +2,8 @@ module Gitlab class BackupLogger < Gitlab::JsonLogger + exclude_context! + def self.file_name_noext 'backup_json' end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 4d609de10f6..92d9d170575 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -131,11 +131,6 @@ module Gitlab end def report_uses_deprecated_schema_version? - # Avoid deprecation warnings for GitLab security scanners - # To be removed via https://gitlab.com/gitlab-org/gitlab/-/issues/386798 - return if report_data.dig('scan', 'scanner', 'vendor', 'name')&.downcase == 'gitlab' - return if report_data.dig('scan', 'analyzer', 'vendor', 'name')&.downcase == 'gitlab' - DEPRECATED_VERSIONS[report_type].include?(report_version) end diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index cfade84a533..f16c28e7b60 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -52,6 +52,7 @@ cache: - gitlab-terraform apply resource_group: ${TF_STATE_NAME} rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $TF_AUTO_DEPLOY == "true" - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH when: manual diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 57caf1b07e6..093667dc624 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -18,7 +18,7 @@ module Gitlab # Minimum PostgreSQL version requirement per documentation: # https://docs.gitlab.com/ee/install/requirements.html#postgresql-requirements - MINIMUM_POSTGRES_VERSION = 12 + MINIMUM_POSTGRES_VERSION = 13 # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html MAX_INT_VALUE = 2147483647 diff --git a/lib/gitlab/database/load_balancing/logger.rb b/lib/gitlab/database/load_balancing/logger.rb index ee67ffcc99c..df6ae47bb84 100644 --- a/lib/gitlab/database/load_balancing/logger.rb +++ b/lib/gitlab/database/load_balancing/logger.rb @@ -4,6 +4,8 @@ module Gitlab module Database module LoadBalancing class Logger < ::Gitlab::JsonLogger + exclude_context! + def self.file_name_noext 'database_load_balancing' end diff --git a/lib/gitlab/database/obsolete_ignored_columns.rb b/lib/gitlab/database/obsolete_ignored_columns.rb index 2b88ab12380..c172d15301a 100644 --- a/lib/gitlab/database/obsolete_ignored_columns.rb +++ b/lib/gitlab/database/obsolete_ignored_columns.rb @@ -2,15 +2,15 @@ module Gitlab module Database - # Checks which `ignored_columns` can be safely removed by scanning - # the current schema for all `ApplicationRecord` descendants. + # Checks which `ignored_columns` definitions can be safely removed by + # scanning the current schema for all `ApplicationRecord` descendants. class ObsoleteIgnoredColumns def initialize(base = ApplicationRecord) @base = base end def execute - @base.descendants.map do |klass| + @base.descendants.filter_map do |klass| next if klass.abstract_class? safe_to_remove = ignored_columns_safe_to_remove_for(klass) diff --git a/lib/gitlab/deprecation_json_logger.rb b/lib/gitlab/deprecation_json_logger.rb index 9796b24868b..5b0900a86dd 100644 --- a/lib/gitlab/deprecation_json_logger.rb +++ b/lib/gitlab/deprecation_json_logger.rb @@ -2,6 +2,8 @@ module Gitlab class DeprecationJsonLogger < Gitlab::JsonLogger + exclude_context! + def self.file_name_noext 'deprecation_json' end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index a907af6891b..77d2ba315a8 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -24,8 +24,6 @@ module Gitlab end end - InstrumentationStorage = ::Gitlab::Instrumentation::Storage - SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION' MAXIMUM_GITALY_CALLS = 30 CLIENT_NAME = (Gitlab::Runtime.sidekiq? ? 'gitlab-sidekiq' : 'gitlab-web').freeze @@ -188,15 +186,15 @@ module Gitlab end def self.query_time - query_time = InstrumentationStorage[:gitaly_query_time] || 0 + query_time = Gitlab::SafeRequestStore[:gitaly_query_time] || 0 query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION) end def self.add_query_time(duration) - return unless InstrumentationStorage.active? + return unless Gitlab::SafeRequestStore.active? - InstrumentationStorage[:gitaly_query_time] ||= 0 - InstrumentationStorage[:gitaly_query_time] += duration + Gitlab::SafeRequestStore[:gitaly_query_time] ||= 0 + Gitlab::SafeRequestStore[:gitaly_query_time] += duration end # For some time related tasks we can't rely on `Time.now` since it will be @@ -255,8 +253,9 @@ module Gitlab # forced to route all requests to the primary node which has injected the # quarantine object directory to us. def self.route_to_primary - return {} unless InstrumentationStorage.active? - return {} if InstrumentationStorage[:gitlab_git_env].blank? + return {} unless Gitlab::SafeRequestStore.active? + + return {} if Gitlab::SafeRequestStore[:gitlab_git_env].blank? { 'gitaly-route-repository-accessor-policy' => 'primary-only' } end @@ -279,7 +278,7 @@ module Gitlab private_class_method :request_deadline def self.session_id - InstrumentationStorage[:gitaly_session_id] ||= SecureRandom.uuid + Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid end def self.token(storage) @@ -291,8 +290,8 @@ module Gitlab # Ensures that Gitaly is not being abuse through n+1 misuse etc def self.enforce_gitaly_request_limits(call_site) - # Only count limits in live environments - return unless InstrumentationStorage.active? + # Only count limits in request-response environments + return unless Gitlab::SafeRequestStore.active? # This is this actual number of times this call was made. Used for information purposes only actual_call_count = increment_call_count("gitaly_#{call_site}_actual") @@ -330,7 +329,7 @@ module Gitlab private_class_method :enforce_gitaly_request_limits? def self.allow_n_plus_1_calls - return yield unless InstrumentationStorage.active? + return yield unless Gitlab::SafeRequestStore.active? begin increment_call_count(:gitaly_call_count_exception_block_depth) @@ -345,34 +344,34 @@ module Gitlab # afterwards. However, for read-only requests that never mutate the # branch, this method allows caching of the ref name directly. def self.allow_ref_name_caching - return yield unless InstrumentationStorage.active? + return yield unless Gitlab::SafeRequestStore.active? return yield if ref_name_caching_allowed? begin - InstrumentationStorage[:allow_ref_name_caching] = true + Gitlab::SafeRequestStore[:allow_ref_name_caching] = true yield ensure - InstrumentationStorage[:allow_ref_name_caching] = false + Gitlab::SafeRequestStore[:allow_ref_name_caching] = false end end def self.ref_name_caching_allowed? - InstrumentationStorage[:allow_ref_name_caching] + Gitlab::SafeRequestStore[:allow_ref_name_caching] end def self.get_call_count(key) - InstrumentationStorage[key] || 0 + Gitlab::SafeRequestStore[key] || 0 end private_class_method :get_call_count def self.increment_call_count(key) - InstrumentationStorage[key] ||= 0 - InstrumentationStorage[key] += 1 + Gitlab::SafeRequestStore[key] ||= 0 + Gitlab::SafeRequestStore[key] += 1 end private_class_method :increment_call_count def self.decrement_call_count(key) - InstrumentationStorage[key] -= 1 + Gitlab::SafeRequestStore[key] -= 1 end private_class_method :decrement_call_count @@ -382,21 +381,21 @@ module Gitlab end def self.reset_counts - return unless InstrumentationStorage.active? + return unless Gitlab::SafeRequestStore.active? - InstrumentationStorage["gitaly_call_actual"] = 0 - InstrumentationStorage["gitaly_call_permitted"] = 0 + Gitlab::SafeRequestStore["gitaly_call_actual"] = 0 + Gitlab::SafeRequestStore["gitaly_call_permitted"] = 0 end def self.add_call_details(details) - InstrumentationStorage['gitaly_call_details'] ||= [] - InstrumentationStorage['gitaly_call_details'] << details + Gitlab::SafeRequestStore['gitaly_call_details'] ||= [] + Gitlab::SafeRequestStore['gitaly_call_details'] << details end def self.list_call_details return [] unless Gitlab::PerformanceBar.enabled_for_request? - InstrumentationStorage['gitaly_call_details'] || [] + Gitlab::SafeRequestStore['gitaly_call_details'] || [] end def self.expected_server_version @@ -481,22 +480,22 @@ module Gitlab # Count a stack. Used for n+1 detection def self.count_stack - return unless InstrumentationStorage.active? + return unless Gitlab::SafeRequestStore.active? stack_string = Gitlab::BacktraceCleaner.clean_backtrace(caller).drop(1).join("\n") - InstrumentationStorage[:stack_counter] ||= {} + Gitlab::SafeRequestStore[:stack_counter] ||= {} - count = InstrumentationStorage[:stack_counter][stack_string] || 0 - InstrumentationStorage[:stack_counter][stack_string] = count + 1 + count = Gitlab::SafeRequestStore[:stack_counter][stack_string] || 0 + Gitlab::SafeRequestStore[:stack_counter][stack_string] = count + 1 end private_class_method :count_stack # Returns a count for the stack which called Gitaly the most times. Used for n+1 detection def self.max_call_count - return 0 unless InstrumentationStorage.active? + return 0 unless Gitlab::SafeRequestStore.active? - stack_counter = InstrumentationStorage[:stack_counter] + stack_counter = Gitlab::SafeRequestStore[:stack_counter] return 0 unless stack_counter stack_counter.values.max @@ -505,9 +504,9 @@ module Gitlab # Returns the stacks that calls Gitaly the most times. Used for n+1 detection def self.max_stacks - return unless InstrumentationStorage.active? + return unless Gitlab::SafeRequestStore.active? - stack_counter = InstrumentationStorage[:stack_counter] + stack_counter = Gitlab::SafeRequestStore[:stack_counter] return unless stack_counter max = max_call_count @@ -545,8 +544,8 @@ module Gitlab end def self.feature_flag_actors - if InstrumentationStorage.active? - InstrumentationStorage[:gitaly_feature_flag_actors] ||= {} + if Gitlab::SafeRequestStore.active? + Gitlab::SafeRequestStore[:gitaly_feature_flag_actors] ||= {} else Thread.current[:gitaly_feature_flag_actors] ||= {} end diff --git a/lib/gitlab/graphql/calls_gitaly/field_extension.rb b/lib/gitlab/graphql/calls_gitaly/field_extension.rb index 014ce9fb0ee..32530b47ce3 100644 --- a/lib/gitlab/graphql/calls_gitaly/field_extension.rb +++ b/lib/gitlab/graphql/calls_gitaly/field_extension.rb @@ -67,14 +67,14 @@ module Gitlab end def accounted_for(count = nil) - return 0 unless ::Gitlab::Instrumentation::Storage.active? + return 0 unless Gitlab::SafeRequestStore.active? - ::Gitlab::Instrumentation::Storage["graphql_gitaly_accounted_for"] ||= 0 + Gitlab::SafeRequestStore["graphql_gitaly_accounted_for"] ||= 0 if count.nil? - ::Gitlab::Instrumentation::Storage["graphql_gitaly_accounted_for"] + Gitlab::SafeRequestStore["graphql_gitaly_accounted_for"] else - ::Gitlab::Instrumentation::Storage["graphql_gitaly_accounted_for"] += count + Gitlab::SafeRequestStore["graphql_gitaly_accounted_for"] += count end end diff --git a/lib/gitlab/instrumentation/elasticsearch_transport.rb b/lib/gitlab/instrumentation/elasticsearch_transport.rb index 791cf691112..4bef043ecb0 100644 --- a/lib/gitlab/instrumentation/elasticsearch_transport.rb +++ b/lib/gitlab/instrumentation/elasticsearch_transport.rb @@ -4,8 +4,6 @@ require 'elasticsearch-transport' module Gitlab module Instrumentation - InstrumentationStorage = ::Gitlab::Instrumentation::Storage - module ElasticsearchTransportInterceptor def perform_request(method, path, params = {}, body = nil, headers = nil) start = Time.now @@ -13,7 +11,7 @@ module Gitlab .reverse_merge({ 'X-Opaque-Id': Labkit::Correlation::CorrelationId.current_or_new_id }) response = super ensure - if InstrumentationStorage.active? + if ::Gitlab::SafeRequestStore.active? duration = (Time.now - start) ::Gitlab::Instrumentation::ElasticsearchTransport.increment_request_count @@ -35,35 +33,35 @@ module Gitlab ELASTICSEARCH_TIMED_OUT_COUNT = :elasticsearch_timed_out_count def self.get_request_count - InstrumentationStorage[ELASTICSEARCH_REQUEST_COUNT] || 0 + ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] || 0 end def self.increment_request_count - InstrumentationStorage[ELASTICSEARCH_REQUEST_COUNT] ||= 0 - InstrumentationStorage[ELASTICSEARCH_REQUEST_COUNT] += 1 + ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] ||= 0 + ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] += 1 end def self.detail_store - InstrumentationStorage[ELASTICSEARCH_CALL_DETAILS] ||= [] + ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DETAILS] ||= [] end def self.query_time - query_time = InstrumentationStorage[ELASTICSEARCH_CALL_DURATION] || 0 + query_time = ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] || 0 query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) end def self.add_duration(duration) - InstrumentationStorage[ELASTICSEARCH_CALL_DURATION] ||= 0 - InstrumentationStorage[ELASTICSEARCH_CALL_DURATION] += duration + ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] ||= 0 + ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] += duration end def self.increment_timed_out_count - InstrumentationStorage[ELASTICSEARCH_TIMED_OUT_COUNT] ||= 0 - InstrumentationStorage[ELASTICSEARCH_TIMED_OUT_COUNT] += 1 + ::Gitlab::SafeRequestStore[ELASTICSEARCH_TIMED_OUT_COUNT] ||= 0 + ::Gitlab::SafeRequestStore[ELASTICSEARCH_TIMED_OUT_COUNT] += 1 end def self.get_timed_out_count - InstrumentationStorage[ELASTICSEARCH_TIMED_OUT_COUNT] || 0 + ::Gitlab::SafeRequestStore[ELASTICSEARCH_TIMED_OUT_COUNT] || 0 end def self.add_call_details(duration, method, path, params, body) diff --git a/lib/gitlab/instrumentation/global_search_api.rb b/lib/gitlab/instrumentation/global_search_api.rb index d475a58c36c..ea2f5702364 100644 --- a/lib/gitlab/instrumentation/global_search_api.rb +++ b/lib/gitlab/instrumentation/global_search_api.rb @@ -8,22 +8,20 @@ module Gitlab SCOPE = 'meta.search.scope' SEARCH_DURATION_S = :global_search_duration_s - InstrumentationStorage = ::Gitlab::Instrumentation::Storage - def self.get_type - InstrumentationStorage[TYPE] + ::Gitlab::SafeRequestStore[TYPE] end def self.get_level - InstrumentationStorage[LEVEL] + ::Gitlab::SafeRequestStore[LEVEL] end def self.get_scope - InstrumentationStorage[SCOPE] + ::Gitlab::SafeRequestStore[SCOPE] end def self.get_search_duration_s - InstrumentationStorage[SEARCH_DURATION_S] + ::Gitlab::SafeRequestStore[SEARCH_DURATION_S] end def self.payload @@ -36,11 +34,11 @@ module Gitlab end def self.set_information(type:, level:, scope:, search_duration_s:) - if InstrumentationStorage.active? - InstrumentationStorage[TYPE] = type - InstrumentationStorage[LEVEL] = level - InstrumentationStorage[SCOPE] = scope - InstrumentationStorage[SEARCH_DURATION_S] = search_duration_s + if ::Gitlab::SafeRequestStore.active? + ::Gitlab::SafeRequestStore[TYPE] = type + ::Gitlab::SafeRequestStore[LEVEL] = level + ::Gitlab::SafeRequestStore[SCOPE] = scope + ::Gitlab::SafeRequestStore[SEARCH_DURATION_S] = search_duration_s end end end diff --git a/lib/gitlab/instrumentation/rate_limiting_gates.rb b/lib/gitlab/instrumentation/rate_limiting_gates.rb index 01b362f0cf4..960b6995030 100644 --- a/lib/gitlab/instrumentation/rate_limiting_gates.rb +++ b/lib/gitlab/instrumentation/rate_limiting_gates.rb @@ -5,11 +5,9 @@ module Gitlab class RateLimitingGates GATES = :rate_limiting_gates - InstrumentationStorage = ::Gitlab::Instrumentation::Storage - class << self def track(key) - if InstrumentationStorage.active? + if ::Gitlab::SafeRequestStore.active? gates_set << key end end @@ -27,7 +25,7 @@ module Gitlab private def gates_set - InstrumentationStorage[GATES] ||= Set.new + ::Gitlab::SafeRequestStore[GATES] ||= Set.new end end end diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index 39290c109d7..00a7387afe2 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -9,8 +9,6 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize include ::Gitlab::Instrumentation::RedisPayload - InstrumentationStorage = ::Gitlab::Instrumentation::Storage - # TODO: To be used by https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/395 # as a 'label' alias. def storage_key @@ -18,10 +16,8 @@ module Gitlab end def add_duration(duration) - return unless InstrumentationStorage.active? - - InstrumentationStorage[call_duration_key] ||= 0 - InstrumentationStorage[call_duration_key] += duration + ::RequestStore[call_duration_key] ||= 0 + ::RequestStore[call_duration_key] += duration end def add_call_details(duration, commands) @@ -35,66 +31,56 @@ module Gitlab end def increment_request_count(amount = 1) - return unless InstrumentationStorage.active? - - InstrumentationStorage[request_count_key] ||= 0 - InstrumentationStorage[request_count_key] += amount + ::RequestStore[request_count_key] ||= 0 + ::RequestStore[request_count_key] += amount end def increment_read_bytes(num_bytes) - return unless InstrumentationStorage.active? - - InstrumentationStorage[read_bytes_key] ||= 0 - InstrumentationStorage[read_bytes_key] += num_bytes + ::RequestStore[read_bytes_key] ||= 0 + ::RequestStore[read_bytes_key] += num_bytes end def increment_write_bytes(num_bytes) - return unless InstrumentationStorage.active? - - InstrumentationStorage[write_bytes_key] ||= 0 - InstrumentationStorage[write_bytes_key] += num_bytes + ::RequestStore[write_bytes_key] ||= 0 + ::RequestStore[write_bytes_key] += num_bytes end def increment_cross_slot_request_count(amount = 1) - return unless InstrumentationStorage.active? - - InstrumentationStorage[cross_slots_key] ||= 0 - InstrumentationStorage[cross_slots_key] += amount + ::RequestStore[cross_slots_key] ||= 0 + ::RequestStore[cross_slots_key] += amount end def increment_allowed_cross_slot_request_count(amount = 1) - return unless InstrumentationStorage.active? - - InstrumentationStorage[allowed_cross_slots_key] ||= 0 - InstrumentationStorage[allowed_cross_slots_key] += amount + ::RequestStore[allowed_cross_slots_key] ||= 0 + ::RequestStore[allowed_cross_slots_key] += amount end def get_request_count - InstrumentationStorage[request_count_key] || 0 + ::RequestStore[request_count_key] || 0 end def read_bytes - InstrumentationStorage[read_bytes_key] || 0 + ::RequestStore[read_bytes_key] || 0 end def write_bytes - InstrumentationStorage[write_bytes_key] || 0 + ::RequestStore[write_bytes_key] || 0 end def detail_store - InstrumentationStorage[call_details_key] ||= [] + ::RequestStore[call_details_key] ||= [] end def get_cross_slot_request_count - InstrumentationStorage[cross_slots_key] || 0 + ::RequestStore[cross_slots_key] || 0 end def get_allowed_cross_slot_request_count - InstrumentationStorage[allowed_cross_slots_key] || 0 + ::RequestStore[allowed_cross_slots_key] || 0 end def query_time - query_time = InstrumentationStorage[call_duration_key] || 0 + query_time = ::RequestStore[call_duration_key] || 0 query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) end diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index c81f070219e..b3fbe30e583 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -55,7 +55,7 @@ module Gitlab commands.each { instrumentation_class.instance_observe_duration(duration / commands.size) } end - if ::Gitlab::Instrumentation::Storage.active? + if ::RequestStore.active? # These metrics measure total Redis usage per Rails request / job. instrumentation_class.increment_request_count(commands.size) instrumentation_class.add_duration(duration) diff --git a/lib/gitlab/instrumentation/storage.rb b/lib/gitlab/instrumentation/storage.rb deleted file mode 100644 index 20c1b69acdd..00000000000 --- a/lib/gitlab/instrumentation/storage.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Instrumentation - module Storage - extend self - - delegate :active?, to: ::Gitlab::SafeRequestStore - delegate :[], :[]=, to: :storage - - def clear! - storage.clear - end - - private - - def storage - ::Gitlab::SafeRequestStore.fetch(:instrumentation) { {} } - end - end - end -end diff --git a/lib/gitlab/instrumentation/throttle.rb b/lib/gitlab/instrumentation/throttle.rb index 837313127e7..0b7e990fb2e 100644 --- a/lib/gitlab/instrumentation/throttle.rb +++ b/lib/gitlab/instrumentation/throttle.rb @@ -3,16 +3,14 @@ module Gitlab module Instrumentation class Throttle - InstrumentationStorage = ::Gitlab::Instrumentation::Storage - KEY = :instrumentation_throttle_safelist def self.safelist - InstrumentationStorage[KEY] + Gitlab::SafeRequestStore[KEY] end def self.safelist=(name) - InstrumentationStorage[KEY] = name + Gitlab::SafeRequestStore[KEY] = name end end end diff --git a/lib/gitlab/instrumentation/uploads.rb b/lib/gitlab/instrumentation/uploads.rb index 226c4e73d01..02e457453cd 100644 --- a/lib/gitlab/instrumentation/uploads.rb +++ b/lib/gitlab/instrumentation/uploads.rb @@ -6,21 +6,19 @@ module Gitlab UPLOAD_DURATION = :uploaded_file_upload_duration_s UPLOADED_FILE_SIZE = :uploaded_file_size_bytes - InstrumentationStorage = ::Gitlab::Instrumentation::Storage - def self.track(uploaded_file) - if InstrumentationStorage.active? - InstrumentationStorage[UPLOAD_DURATION] = uploaded_file.upload_duration - InstrumentationStorage[UPLOADED_FILE_SIZE] = uploaded_file.size + if ::Gitlab::SafeRequestStore.active? + ::Gitlab::SafeRequestStore[UPLOAD_DURATION] = uploaded_file.upload_duration + ::Gitlab::SafeRequestStore[UPLOADED_FILE_SIZE] = uploaded_file.size end end def self.get_upload_duration - InstrumentationStorage[UPLOAD_DURATION] + ::Gitlab::SafeRequestStore[UPLOAD_DURATION] end def self.get_uploaded_file_size - InstrumentationStorage[UPLOADED_FILE_SIZE] + ::Gitlab::SafeRequestStore[UPLOADED_FILE_SIZE] end def self.payload diff --git a/lib/gitlab/instrumentation/zoekt.rb b/lib/gitlab/instrumentation/zoekt.rb index 4d2933680c5..cd9b15bcee8 100644 --- a/lib/gitlab/instrumentation/zoekt.rb +++ b/lib/gitlab/instrumentation/zoekt.rb @@ -3,34 +3,32 @@ module Gitlab module Instrumentation class Zoekt - InstrumentationStorage = ::Gitlab::Instrumentation::Storage - ZOEKT_REQUEST_COUNT = :zoekt_request_count ZOEKT_CALL_DURATION = :zoekt_call_duration ZOEKT_CALL_DETAILS = :zoekt_call_details class << self def get_request_count - InstrumentationStorage[ZOEKT_REQUEST_COUNT] || 0 + ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] || 0 end def increment_request_count - InstrumentationStorage[ZOEKT_REQUEST_COUNT] ||= 0 - InstrumentationStorage[ZOEKT_REQUEST_COUNT] += 1 + ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] ||= 0 + ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] += 1 end def detail_store - InstrumentationStorage[ZOEKT_CALL_DETAILS] ||= [] + ::Gitlab::SafeRequestStore[ZOEKT_CALL_DETAILS] ||= [] end def query_time - query_time = InstrumentationStorage[ZOEKT_CALL_DURATION] || 0 + query_time = ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] || 0 query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) end def add_duration(duration) - InstrumentationStorage[ZOEKT_CALL_DURATION] ||= 0 - InstrumentationStorage[ZOEKT_CALL_DURATION] += duration + ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] ||= 0 + ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] += duration end def add_call_details(duration:, method:, path:, params: nil, body: nil) diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 686142e338a..2a3c4db5ffa 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -6,18 +6,8 @@ module Gitlab DURATION_PRECISION = 6 # microseconds - def init_instrumentation_data(request_ip: nil) - ::Gitlab::Instrumentation::Storage.clear! - - # Set `request_start_time` only if this is request - # This is done, as `request_start_time` imply `request_deadline` - if request_ip - Gitlab::RequestContext.instance.client_ip = request_ip - Gitlab::RequestContext.instance.request_start_time = Gitlab::Metrics::System.real_time - end - - Gitlab::RequestContext.instance.start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time - Gitlab::RequestContext.instance.thread_memory_allocations = Gitlab::Memory::Instrumentation.start_thread_memory_allocations + def init_instrumentation_data + Gitlab::RequestContext.start_thread_context end def add_instrumentation_data(payload) diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index dd99d4d770c..10bb358a292 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -21,8 +21,6 @@ module Gitlab SQL_WAL_LOCATION_REGEX = /(pg_current_wal_insert_lsn\(\)::text|pg_last_wal_replay_lsn\(\)::text)/.freeze - InstrumentationStorage = ::Gitlab::Instrumentation::Storage - # This event is published from ActiveRecordBaseTransactionMetrics and # used to record a database transaction duration when calling # ApplicationRecord.transaction {} block. @@ -58,20 +56,20 @@ module Gitlab end def self.db_counter_payload - return {} unless InstrumentationStorage.active? + return {} unless Gitlab::SafeRequestStore.active? {}.tap do |payload| db_counter_keys.each do |key| - payload[key] = InstrumentationStorage[key].to_i + payload[key] = Gitlab::SafeRequestStore[key].to_i end - if InstrumentationStorage.active? + if ::Gitlab::SafeRequestStore.active? load_balancing_metric_counter_keys.each do |counter| - payload[counter] = InstrumentationStorage[counter].to_i + payload[counter] = ::Gitlab::SafeRequestStore[counter].to_i end load_balancing_metric_duration_keys.each do |duration| - payload[duration] = InstrumentationStorage[duration].to_f.round(3) + payload[duration] = ::Gitlab::SafeRequestStore[duration].to_f.round(3) end end end @@ -102,16 +100,16 @@ module Gitlab buckets ::Gitlab::Metrics::Subscribers::ActiveRecord::SQL_DURATION_BUCKET end - return unless InstrumentationStorage.active? + return unless ::Gitlab::SafeRequestStore.active? duration = event.duration / 1000.0 duration_key = compose_metric_key(:duration_s, db_role) - InstrumentationStorage[duration_key] = (InstrumentationStorage[duration_key].presence || 0) + duration + ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration # Per database metrics db_config_name = db_config_name(event.payload) duration_key = compose_metric_key(:duration_s, nil, db_config_name) - InstrumentationStorage[duration_key] = (InstrumentationStorage[duration_key].presence || 0) + duration + ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration end def ignored_query?(payload) @@ -137,14 +135,14 @@ module Gitlab current_transaction&.increment(prometheus_key, 1, { db_config_name: db_config_name }) - InstrumentationStorage[log_key] = InstrumentationStorage[log_key].to_i + 1 + Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1 # To avoid confusing log keys we only log the db_config_name metrics # when we are also logging the db_role. Otherwise it will be hard to # tell if the log key is referring to a db_role OR a db_config_name. if db_role.present? && db_config_name.present? log_key = compose_metric_key(counter, nil, db_config_name) - InstrumentationStorage[log_key] = InstrumentationStorage[log_key].to_i + 1 + Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1 end end diff --git a/lib/gitlab/metrics/subscribers/external_http.rb b/lib/gitlab/metrics/subscribers/external_http.rb index a5bfc80b3bf..87756b14887 100644 --- a/lib/gitlab/metrics/subscribers/external_http.rb +++ b/lib/gitlab/metrics/subscribers/external_http.rb @@ -6,8 +6,6 @@ module Gitlab # Class for tracking the total time spent in external HTTP # See more at https://gitlab.com/gitlab-org/labkit-ruby/-/blob/v0.14.0/lib/gitlab-labkit.rb#L18 class ExternalHttp < ActiveSupport::Subscriber - InstrumentationStorage = ::Gitlab::Instrumentation::Storage - attach_to :external_http DEFAULT_STATUS_CODE = 'undefined' @@ -21,19 +19,19 @@ module Gitlab MAX_SLOW_REQUESTS = 10 def self.detail_store - InstrumentationStorage[DETAIL_STORE] ||= [] + ::Gitlab::SafeRequestStore[DETAIL_STORE] ||= [] end def self.duration - InstrumentationStorage[DURATION].to_f + Gitlab::SafeRequestStore[DURATION].to_f end def self.request_count - InstrumentationStorage[COUNTER].to_i + Gitlab::SafeRequestStore[COUNTER].to_i end def self.slow_requests - InstrumentationStorage[SLOW_REQUESTS] + Gitlab::SafeRequestStore[SLOW_REQUESTS] end def self.top_slowest_requests @@ -84,14 +82,14 @@ module Gitlab end def add_to_request_store(payload) - return unless InstrumentationStorage.active? + return unless Gitlab::SafeRequestStore.active? - InstrumentationStorage[COUNTER] = InstrumentationStorage[COUNTER].to_i + 1 - InstrumentationStorage[DURATION] = InstrumentationStorage[DURATION].to_f + payload[:duration].to_f + Gitlab::SafeRequestStore[COUNTER] = Gitlab::SafeRequestStore[COUNTER].to_i + 1 + Gitlab::SafeRequestStore[DURATION] = Gitlab::SafeRequestStore[DURATION].to_f + payload[:duration].to_f if payload[:duration].to_f > THRESHOLD_SLOW_REQUEST_S - InstrumentationStorage[SLOW_REQUESTS] ||= [] - InstrumentationStorage[SLOW_REQUESTS] << { + Gitlab::SafeRequestStore[SLOW_REQUESTS] ||= [] + Gitlab::SafeRequestStore[SLOW_REQUESTS] << { method: payload[:method], host: payload[:host], port: payload[:port], diff --git a/lib/gitlab/metrics/subscribers/ldap.rb b/lib/gitlab/metrics/subscribers/ldap.rb index 409ecbb6e8a..3dae2d1fd88 100644 --- a/lib/gitlab/metrics/subscribers/ldap.rb +++ b/lib/gitlab/metrics/subscribers/ldap.rb @@ -8,8 +8,6 @@ module Gitlab # at the end of the event key, e.g. `open.net_ldap` attach_to :net_ldap - InstrumentationStorage = ::Gitlab::Instrumentation::Storage - COUNTER = :net_ldap_count DURATION = :net_ldap_duration_s @@ -28,12 +26,12 @@ module Gitlab class << self # @return [Integer] the total number of LDAP requests def count - InstrumentationStorage[COUNTER].to_i + Gitlab::SafeRequestStore[COUNTER].to_i end # @return [Float] the total duration spent on LDAP requests def duration - InstrumentationStorage[DURATION].to_f + Gitlab::SafeRequestStore[DURATION].to_f end # Used in Gitlab::InstrumentationHelper to merge the LDAP stats @@ -73,10 +71,10 @@ module Gitlab # Track these events as statistics for the current requests, for logging purposes def add_to_request_store(event) - return unless InstrumentationStorage.active? + return unless Gitlab::SafeRequestStore.active? - InstrumentationStorage[COUNTER] = self.class.count + 1 - InstrumentationStorage[DURATION] = self.class.duration + convert_to_seconds(event.duration) + Gitlab::SafeRequestStore[COUNTER] = self.class.count + 1 + Gitlab::SafeRequestStore[DURATION] = self.class.duration + convert_to_seconds(event.duration) end # Converts the observed events into Prometheus metrics diff --git a/lib/gitlab/metrics/subscribers/load_balancing.rb b/lib/gitlab/metrics/subscribers/load_balancing.rb index d7fe33dbe89..bd77e8c3c3f 100644 --- a/lib/gitlab/metrics/subscribers/load_balancing.rb +++ b/lib/gitlab/metrics/subscribers/load_balancing.rb @@ -6,13 +6,11 @@ module Gitlab class LoadBalancing < ActiveSupport::Subscriber attach_to :load_balancing - InstrumentationStorage = ::Gitlab::Instrumentation::Storage - PROMETHEUS_COUNTER = :gitlab_transaction_caught_up_replica_pick_count_total LOG_COUNTERS = { true => :caught_up_replica_pick_ok, false => :caught_up_replica_pick_fail }.freeze def caught_up_replica_pick(event) - return unless InstrumentationStorage.active? + return unless Gitlab::SafeRequestStore.active? result = event.payload[:result] counter_name = counter(result) @@ -22,17 +20,17 @@ module Gitlab # we want to update Prometheus counter after the controller/action are set def web_transaction_completed(_event) - return unless InstrumentationStorage.active? + return unless Gitlab::SafeRequestStore.active? LOG_COUNTERS.keys.each { |result| increment_prometheus_for_result_label(result) } end def self.load_balancing_payload - return {} unless InstrumentationStorage.active? + return {} unless Gitlab::SafeRequestStore.active? {}.tap do |payload| LOG_COUNTERS.values.each do |counter| - value = InstrumentationStorage[counter] + value = Gitlab::SafeRequestStore[counter] payload[counter] = value.to_i if value end @@ -42,12 +40,12 @@ module Gitlab private def increment(counter) - InstrumentationStorage[counter] = InstrumentationStorage[counter].to_i + 1 + Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1 end def increment_prometheus_for_result_label(label_value) counter_name = counter(label_value) - return unless (counter_value = InstrumentationStorage[counter_name]) + return unless (counter_value = Gitlab::SafeRequestStore[counter_name]) increment_prometheus(labels: { result: label_value }, value: counter_value.to_i) end diff --git a/lib/gitlab/metrics/subscribers/rack_attack.rb b/lib/gitlab/metrics/subscribers/rack_attack.rb index 705536039ed..2196122df01 100644 --- a/lib/gitlab/metrics/subscribers/rack_attack.rb +++ b/lib/gitlab/metrics/subscribers/rack_attack.rb @@ -13,12 +13,10 @@ module Gitlab class RackAttack < ActiveSupport::Subscriber attach_to 'rack_attack' - InstrumentationStorage = ::Gitlab::Instrumentation::Storage - INSTRUMENTATION_STORE_KEY = :rack_attack_instrumentation def self.payload - InstrumentationStorage[INSTRUMENTATION_STORE_KEY] ||= { + Gitlab::SafeRequestStore[INSTRUMENTATION_STORE_KEY] ||= { rack_attack_redis_count: 0, rack_attack_redis_duration_s: 0.0 } diff --git a/lib/gitlab/middleware/request_context.rb b/lib/gitlab/middleware/request_context.rb index 07f6f87a68c..f609002007c 100644 --- a/lib/gitlab/middleware/request_context.rb +++ b/lib/gitlab/middleware/request_context.rb @@ -8,15 +8,9 @@ module Gitlab end def call(env) - # We should be using ActionDispatch::Request instead of - # Rack::Request to be consistent with Rails, but due to a Rails - # bug described in - # https://gitlab.com/gitlab-org/gitlab-foss/issues/58573#note_149799010 - # hosts behind a load balancer will only see 127.0.0.1 for the - # load balancer's IP. - req = Rack::Request.new(env) - - ::Gitlab::InstrumentationHelper.init_instrumentation_data(request_ip: req.ip) + request = ActionDispatch::Request.new(env) + Gitlab::RequestContext.start_request_context(request: request) + Gitlab::RequestContext.start_thread_context @app.call(env) end diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb index c9eefe9a647..813468ece90 100644 --- a/lib/gitlab/request_context.rb +++ b/lib/gitlab/request_context.rb @@ -7,12 +7,28 @@ module Gitlab RequestDeadlineExceeded = Class.new(StandardError) - attr_accessor :client_ip, :start_thread_cpu_time, :request_start_time, :thread_memory_allocations + attr_accessor :client_ip, :spam_params, :start_thread_cpu_time, :request_start_time, :thread_memory_allocations class << self def instance Gitlab::SafeRequestStore[:request_context] ||= new end + + def start_request_context(request:) + # We need to use Rack::Request to be consistent with Rails due to a Rails bug described in + # https://gitlab.com/gitlab-org/gitlab-foss/issues/58573#note_149799010 + # Hosts behind a load balancer will only see 127.0.0.1 for the load balancer's IP. + rack_req = Rack::Request.new(request.env) + instance.client_ip = rack_req.ip + + instance.spam_params = ::Spam::SpamParams.new_from_request(request: request) + instance.request_start_time = Gitlab::Metrics::System.real_time + end + + def start_thread_context + instance.start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time + instance.thread_memory_allocations = Gitlab::Memory::Instrumentation.start_thread_memory_allocations + end end def request_deadline diff --git a/lib/gitlab/rugged_instrumentation.rb b/lib/gitlab/rugged_instrumentation.rb index b768e89b1e4..36a3a491de6 100644 --- a/lib/gitlab/rugged_instrumentation.rb +++ b/lib/gitlab/rugged_instrumentation.rb @@ -2,16 +2,14 @@ module Gitlab module RuggedInstrumentation - InstrumentationStorage = ::Gitlab::Instrumentation::Storage - def self.query_time - query_time = InstrumentationStorage[:rugged_query_time] || 0 + query_time = SafeRequestStore[:rugged_query_time] || 0 query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION) end def self.add_query_time(duration) - InstrumentationStorage[:rugged_query_time] ||= 0 - InstrumentationStorage[:rugged_query_time] += duration + SafeRequestStore[:rugged_query_time] ||= 0 + SafeRequestStore[:rugged_query_time] += duration end def self.query_time_ms @@ -19,29 +17,29 @@ module Gitlab end def self.query_count - InstrumentationStorage[:rugged_call_count] ||= 0 + SafeRequestStore[:rugged_call_count] ||= 0 end def self.increment_query_count - InstrumentationStorage[:rugged_call_count] ||= 0 - InstrumentationStorage[:rugged_call_count] += 1 + SafeRequestStore[:rugged_call_count] ||= 0 + SafeRequestStore[:rugged_call_count] += 1 end def self.active? - InstrumentationStorage.active? + SafeRequestStore.active? end def self.add_call_details(details) return unless Gitlab::PerformanceBar.enabled_for_request? - InstrumentationStorage[:rugged_call_details] ||= [] - InstrumentationStorage[:rugged_call_details] << details + Gitlab::SafeRequestStore[:rugged_call_details] ||= [] + Gitlab::SafeRequestStore[:rugged_call_details] << details end def self.list_call_details return [] unless Gitlab::PerformanceBar.enabled_for_request? - InstrumentationStorage[:rugged_call_details] || [] + Gitlab::SafeRequestStore[:rugged_call_details] || [] end end end diff --git a/lib/sidebars/projects/menus/confluence_menu.rb b/lib/sidebars/projects/menus/confluence_menu.rb index 0fd42a57da3..43ef7ac73c4 100644 --- a/lib/sidebars/projects/menus/confluence_menu.rb +++ b/lib/sidebars/projects/menus/confluence_menu.rb @@ -42,6 +42,14 @@ module Sidebars def active_routes { controller: :confluences } end + + override :serialize_as_menu_item_args + def serialize_as_menu_item_args + super.merge({ + item_id: :confluence, + super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::PlanMenu + }) + end end end end diff --git a/lib/sidebars/projects/menus/external_issue_tracker_menu.rb b/lib/sidebars/projects/menus/external_issue_tracker_menu.rb index 136d30f38c3..f088ccce9f5 100644 --- a/lib/sidebars/projects/menus/external_issue_tracker_menu.rb +++ b/lib/sidebars/projects/menus/external_issue_tracker_menu.rb @@ -48,6 +48,14 @@ module Sidebars external_issue_tracker.present? end + override :serialize_as_menu_item_args + def serialize_as_menu_item_args + super.merge({ + item_id: :external_issue_tracker, + super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::PlanMenu + }) + end + private def external_issue_tracker diff --git a/lib/sidebars/projects/menus/external_wiki_menu.rb b/lib/sidebars/projects/menus/external_wiki_menu.rb index 825f0ca5e8b..1af9abc33ff 100644 --- a/lib/sidebars/projects/menus/external_wiki_menu.rb +++ b/lib/sidebars/projects/menus/external_wiki_menu.rb @@ -41,6 +41,14 @@ module Sidebars external_wiki.present? end + override :serialize_as_menu_item_args + def serialize_as_menu_item_args + super.merge({ + item_id: :external_wiki, + super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::PlanMenu + }) + end + private def external_wiki diff --git a/lib/tasks/db_obsolete_ignored_columns.rake b/lib/tasks/db_obsolete_ignored_columns.rake index a689a9bf2d8..c71e3169723 100644 --- a/lib/tasks/db_obsolete_ignored_columns.rake +++ b/lib/tasks/db_obsolete_ignored_columns.rake @@ -5,9 +5,9 @@ task 'db:obsolete_ignored_columns' => :environment do list = Gitlab::Database::ObsoleteIgnoredColumns.new.execute if list.empty? - puts 'No obsolete `ignored_columns` found.' + puts 'No obsolete `ignored_columns` definitions found.' else - puts 'The following `ignored_columns` are obsolete and can be removed:' + puts 'The following `ignored_columns` definitions are obsolete and can be removed:' list.each do |name, ignored_columns| puts "#{name}:" |