diff options
Diffstat (limited to 'lib')
24 files changed, 296 insertions, 157 deletions
diff --git a/lib/api/entities/package.rb b/lib/api/entities/package.rb index ab6cc0fcb0a..1c22a1a36fa 100644 --- a/lib/api/entities/package.rb +++ b/lib/api/entities/package.rb @@ -7,6 +7,8 @@ module API include ::Routing::PackagesHelper extend ::API::Entities::EntityHelpers + EMPTY_PIPELINES = [].freeze + expose :id, documentation: { type: 'integer', example: 1 } expose :name, documentation: { type: 'string', example: '@foo/bar' } do |package| @@ -44,7 +46,9 @@ module API expose :tags expose :pipeline, if: ->(package) { package.original_build_info }, using: Package::Pipeline - expose :pipelines, if: ->(package) { package.pipelines.present? }, using: Package::Pipeline + expose :pipelines, if: ->(package) { package.pipelines.present? }, using: Package::Pipeline do |_| + EMPTY_PIPELINES + end expose :versions, using: ::API::Entities::PackageVersion, unless: ->(_, opts) { opts[:collection] } diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb index f38fabc9586..072431a24ab 100644 --- a/lib/api/helpers/integrations_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -593,6 +593,12 @@ module API }, { required: true, + name: :jira_auth_type, + type: Integer, + desc: 'The authorization type for Jira' + }, + { + required: false, name: :username, type: String, desc: 'The username of the user created to be used with GitLab/Jira' diff --git a/lib/api/helpers/packages/maven/basic_auth_helpers.rb b/lib/api/helpers/packages/maven/basic_auth_helpers.rb new file mode 100644 index 00000000000..c9ef95adc33 --- /dev/null +++ b/lib/api/helpers/packages/maven/basic_auth_helpers.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module API + module Helpers + module Packages + module Maven + module BasicAuthHelpers + include ::API::Helpers::Packages::BasicAuthHelpers + extend ::Gitlab::Utils::Override + + # override so that we can receive the job token either by headers or + # basic auth. + override :find_user_from_job_token + def find_user_from_job_token + super || find_user_from_basic_auth_job + end + end + end + end + end +end diff --git a/lib/api/integrations.rb b/lib/api/integrations.rb index 0b1a339a864..3ec0a723808 100644 --- a/lib/api/integrations.rb +++ b/lib/api/integrations.rb @@ -139,8 +139,14 @@ module API destroy_conditionally!(integration) do attrs = integration_attributes(integration).index_with do |attr| - column = integration.column_for_attribute(attr) - if column.is_a?(ActiveRecord::ConnectionAdapters::NullColumn) + column = if integration.attribute_present?(attr) + integration.column_for_attribute(attr) + elsif integration.data_fields_present? + integration.data_fields.column_for_attribute(attr) + end + + case column + when nil, ActiveRecord::ConnectionAdapters::NullColumn nil else column.default diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index ed3479fce2d..e075a917fa9 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -23,8 +23,14 @@ module API helpers ::API::Helpers::PackagesHelpers helpers ::API::Helpers::Packages::DependencyProxyHelpers + helpers ::API::Helpers::Packages::Maven::BasicAuthHelpers helpers do + params :path_and_file_name do + requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' } + requires :file_name, type: String, desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' } + end + def path_exists?(path) return false if path.blank? @@ -159,10 +165,9 @@ module API tags %w[maven_packages] end params do - requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' } - requires :file_name, type: String, desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' } + use :path_and_file_name end - route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true + route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true, basic_auth_personal_access_token: true get 'packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do # return a similar failure to authorize_read_package!(project) @@ -214,13 +219,12 @@ module API end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do params do - requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' } - requires :file_name, type: String, desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' } + use :path_and_file_name end - route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true + route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true, basic_auth_personal_access_token: true get ':id/-/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do # return a similar failure to group = find_group(params[:id]) - group = find_group(params[:id]) + group = find_authorized_group! if Feature.disabled?(:maven_central_request_forwarding, group&.root_ancestor) not_found!('Group') unless path_exists?(params[:path]) @@ -241,7 +245,7 @@ module API requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Download the maven package file' do + desc 'Download the maven package file at a project level' do detail 'This feature was introduced in GitLab 11.3' success [ { code: 200 }, @@ -255,12 +259,11 @@ module API tags %w[maven_packages] end params do - requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' } - requires :file_name, type: String, desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' } + use :path_and_file_name end - route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true + route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true, basic_auth_personal_access_token: true get ':id/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do - project = user_project(action: :read_package) + project = authorized_user_project(action: :read_package) # return a similar failure to user_project unless Feature.enabled?(:maven_central_request_forwarding, project&.root_ancestor) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 77d2ba315a8..a907af6891b 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -24,6 +24,8 @@ module Gitlab end end + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION' MAXIMUM_GITALY_CALLS = 30 CLIENT_NAME = (Gitlab::Runtime.sidekiq? ? 'gitlab-sidekiq' : 'gitlab-web').freeze @@ -186,15 +188,15 @@ module Gitlab end def self.query_time - query_time = Gitlab::SafeRequestStore[:gitaly_query_time] || 0 + query_time = InstrumentationStorage[:gitaly_query_time] || 0 query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION) end def self.add_query_time(duration) - return unless Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? - Gitlab::SafeRequestStore[:gitaly_query_time] ||= 0 - Gitlab::SafeRequestStore[:gitaly_query_time] += duration + InstrumentationStorage[:gitaly_query_time] ||= 0 + InstrumentationStorage[:gitaly_query_time] += duration end # For some time related tasks we can't rely on `Time.now` since it will be @@ -253,9 +255,8 @@ module Gitlab # forced to route all requests to the primary node which has injected the # quarantine object directory to us. def self.route_to_primary - return {} unless Gitlab::SafeRequestStore.active? - - return {} if Gitlab::SafeRequestStore[:gitlab_git_env].blank? + return {} unless InstrumentationStorage.active? + return {} if InstrumentationStorage[:gitlab_git_env].blank? { 'gitaly-route-repository-accessor-policy' => 'primary-only' } end @@ -278,7 +279,7 @@ module Gitlab private_class_method :request_deadline def self.session_id - Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid + InstrumentationStorage[:gitaly_session_id] ||= SecureRandom.uuid end def self.token(storage) @@ -290,8 +291,8 @@ module Gitlab # Ensures that Gitaly is not being abuse through n+1 misuse etc def self.enforce_gitaly_request_limits(call_site) - # Only count limits in request-response environments - return unless Gitlab::SafeRequestStore.active? + # Only count limits in live environments + return unless InstrumentationStorage.active? # This is this actual number of times this call was made. Used for information purposes only actual_call_count = increment_call_count("gitaly_#{call_site}_actual") @@ -329,7 +330,7 @@ module Gitlab private_class_method :enforce_gitaly_request_limits? def self.allow_n_plus_1_calls - return yield unless Gitlab::SafeRequestStore.active? + return yield unless InstrumentationStorage.active? begin increment_call_count(:gitaly_call_count_exception_block_depth) @@ -344,34 +345,34 @@ module Gitlab # afterwards. However, for read-only requests that never mutate the # branch, this method allows caching of the ref name directly. def self.allow_ref_name_caching - return yield unless Gitlab::SafeRequestStore.active? + return yield unless InstrumentationStorage.active? return yield if ref_name_caching_allowed? begin - Gitlab::SafeRequestStore[:allow_ref_name_caching] = true + InstrumentationStorage[:allow_ref_name_caching] = true yield ensure - Gitlab::SafeRequestStore[:allow_ref_name_caching] = false + InstrumentationStorage[:allow_ref_name_caching] = false end end def self.ref_name_caching_allowed? - Gitlab::SafeRequestStore[:allow_ref_name_caching] + InstrumentationStorage[:allow_ref_name_caching] end def self.get_call_count(key) - Gitlab::SafeRequestStore[key] || 0 + InstrumentationStorage[key] || 0 end private_class_method :get_call_count def self.increment_call_count(key) - Gitlab::SafeRequestStore[key] ||= 0 - Gitlab::SafeRequestStore[key] += 1 + InstrumentationStorage[key] ||= 0 + InstrumentationStorage[key] += 1 end private_class_method :increment_call_count def self.decrement_call_count(key) - Gitlab::SafeRequestStore[key] -= 1 + InstrumentationStorage[key] -= 1 end private_class_method :decrement_call_count @@ -381,21 +382,21 @@ module Gitlab end def self.reset_counts - return unless Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? - Gitlab::SafeRequestStore["gitaly_call_actual"] = 0 - Gitlab::SafeRequestStore["gitaly_call_permitted"] = 0 + InstrumentationStorage["gitaly_call_actual"] = 0 + InstrumentationStorage["gitaly_call_permitted"] = 0 end def self.add_call_details(details) - Gitlab::SafeRequestStore['gitaly_call_details'] ||= [] - Gitlab::SafeRequestStore['gitaly_call_details'] << details + InstrumentationStorage['gitaly_call_details'] ||= [] + InstrumentationStorage['gitaly_call_details'] << details end def self.list_call_details return [] unless Gitlab::PerformanceBar.enabled_for_request? - Gitlab::SafeRequestStore['gitaly_call_details'] || [] + InstrumentationStorage['gitaly_call_details'] || [] end def self.expected_server_version @@ -480,22 +481,22 @@ module Gitlab # Count a stack. Used for n+1 detection def self.count_stack - return unless Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? stack_string = Gitlab::BacktraceCleaner.clean_backtrace(caller).drop(1).join("\n") - Gitlab::SafeRequestStore[:stack_counter] ||= {} + InstrumentationStorage[:stack_counter] ||= {} - count = Gitlab::SafeRequestStore[:stack_counter][stack_string] || 0 - Gitlab::SafeRequestStore[:stack_counter][stack_string] = count + 1 + count = InstrumentationStorage[:stack_counter][stack_string] || 0 + InstrumentationStorage[:stack_counter][stack_string] = count + 1 end private_class_method :count_stack # Returns a count for the stack which called Gitaly the most times. Used for n+1 detection def self.max_call_count - return 0 unless Gitlab::SafeRequestStore.active? + return 0 unless InstrumentationStorage.active? - stack_counter = Gitlab::SafeRequestStore[:stack_counter] + stack_counter = InstrumentationStorage[:stack_counter] return 0 unless stack_counter stack_counter.values.max @@ -504,9 +505,9 @@ module Gitlab # Returns the stacks that calls Gitaly the most times. Used for n+1 detection def self.max_stacks - return unless Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? - stack_counter = Gitlab::SafeRequestStore[:stack_counter] + stack_counter = InstrumentationStorage[:stack_counter] return unless stack_counter max = max_call_count @@ -544,8 +545,8 @@ module Gitlab end def self.feature_flag_actors - if Gitlab::SafeRequestStore.active? - Gitlab::SafeRequestStore[:gitaly_feature_flag_actors] ||= {} + if InstrumentationStorage.active? + InstrumentationStorage[:gitaly_feature_flag_actors] ||= {} else Thread.current[:gitaly_feature_flag_actors] ||= {} end diff --git a/lib/gitlab/graphql/calls_gitaly/field_extension.rb b/lib/gitlab/graphql/calls_gitaly/field_extension.rb index 32530b47ce3..014ce9fb0ee 100644 --- a/lib/gitlab/graphql/calls_gitaly/field_extension.rb +++ b/lib/gitlab/graphql/calls_gitaly/field_extension.rb @@ -67,14 +67,14 @@ module Gitlab end def accounted_for(count = nil) - return 0 unless Gitlab::SafeRequestStore.active? + return 0 unless ::Gitlab::Instrumentation::Storage.active? - Gitlab::SafeRequestStore["graphql_gitaly_accounted_for"] ||= 0 + ::Gitlab::Instrumentation::Storage["graphql_gitaly_accounted_for"] ||= 0 if count.nil? - Gitlab::SafeRequestStore["graphql_gitaly_accounted_for"] + ::Gitlab::Instrumentation::Storage["graphql_gitaly_accounted_for"] else - Gitlab::SafeRequestStore["graphql_gitaly_accounted_for"] += count + ::Gitlab::Instrumentation::Storage["graphql_gitaly_accounted_for"] += count end end diff --git a/lib/gitlab/instrumentation/elasticsearch_transport.rb b/lib/gitlab/instrumentation/elasticsearch_transport.rb index 4bef043ecb0..791cf691112 100644 --- a/lib/gitlab/instrumentation/elasticsearch_transport.rb +++ b/lib/gitlab/instrumentation/elasticsearch_transport.rb @@ -4,6 +4,8 @@ require 'elasticsearch-transport' module Gitlab module Instrumentation + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + module ElasticsearchTransportInterceptor def perform_request(method, path, params = {}, body = nil, headers = nil) start = Time.now @@ -11,7 +13,7 @@ module Gitlab .reverse_merge({ 'X-Opaque-Id': Labkit::Correlation::CorrelationId.current_or_new_id }) response = super ensure - if ::Gitlab::SafeRequestStore.active? + if InstrumentationStorage.active? duration = (Time.now - start) ::Gitlab::Instrumentation::ElasticsearchTransport.increment_request_count @@ -33,35 +35,35 @@ module Gitlab ELASTICSEARCH_TIMED_OUT_COUNT = :elasticsearch_timed_out_count def self.get_request_count - ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] || 0 + InstrumentationStorage[ELASTICSEARCH_REQUEST_COUNT] || 0 end def self.increment_request_count - ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] ||= 0 - ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] += 1 + InstrumentationStorage[ELASTICSEARCH_REQUEST_COUNT] ||= 0 + InstrumentationStorage[ELASTICSEARCH_REQUEST_COUNT] += 1 end def self.detail_store - ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DETAILS] ||= [] + InstrumentationStorage[ELASTICSEARCH_CALL_DETAILS] ||= [] end def self.query_time - query_time = ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] || 0 + query_time = InstrumentationStorage[ELASTICSEARCH_CALL_DURATION] || 0 query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) end def self.add_duration(duration) - ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] ||= 0 - ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] += duration + InstrumentationStorage[ELASTICSEARCH_CALL_DURATION] ||= 0 + InstrumentationStorage[ELASTICSEARCH_CALL_DURATION] += duration end def self.increment_timed_out_count - ::Gitlab::SafeRequestStore[ELASTICSEARCH_TIMED_OUT_COUNT] ||= 0 - ::Gitlab::SafeRequestStore[ELASTICSEARCH_TIMED_OUT_COUNT] += 1 + InstrumentationStorage[ELASTICSEARCH_TIMED_OUT_COUNT] ||= 0 + InstrumentationStorage[ELASTICSEARCH_TIMED_OUT_COUNT] += 1 end def self.get_timed_out_count - ::Gitlab::SafeRequestStore[ELASTICSEARCH_TIMED_OUT_COUNT] || 0 + InstrumentationStorage[ELASTICSEARCH_TIMED_OUT_COUNT] || 0 end def self.add_call_details(duration, method, path, params, body) diff --git a/lib/gitlab/instrumentation/global_search_api.rb b/lib/gitlab/instrumentation/global_search_api.rb index ea2f5702364..d475a58c36c 100644 --- a/lib/gitlab/instrumentation/global_search_api.rb +++ b/lib/gitlab/instrumentation/global_search_api.rb @@ -8,20 +8,22 @@ module Gitlab SCOPE = 'meta.search.scope' SEARCH_DURATION_S = :global_search_duration_s + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + def self.get_type - ::Gitlab::SafeRequestStore[TYPE] + InstrumentationStorage[TYPE] end def self.get_level - ::Gitlab::SafeRequestStore[LEVEL] + InstrumentationStorage[LEVEL] end def self.get_scope - ::Gitlab::SafeRequestStore[SCOPE] + InstrumentationStorage[SCOPE] end def self.get_search_duration_s - ::Gitlab::SafeRequestStore[SEARCH_DURATION_S] + InstrumentationStorage[SEARCH_DURATION_S] end def self.payload @@ -34,11 +36,11 @@ module Gitlab end def self.set_information(type:, level:, scope:, search_duration_s:) - if ::Gitlab::SafeRequestStore.active? - ::Gitlab::SafeRequestStore[TYPE] = type - ::Gitlab::SafeRequestStore[LEVEL] = level - ::Gitlab::SafeRequestStore[SCOPE] = scope - ::Gitlab::SafeRequestStore[SEARCH_DURATION_S] = search_duration_s + if InstrumentationStorage.active? + InstrumentationStorage[TYPE] = type + InstrumentationStorage[LEVEL] = level + InstrumentationStorage[SCOPE] = scope + InstrumentationStorage[SEARCH_DURATION_S] = search_duration_s end end end diff --git a/lib/gitlab/instrumentation/rate_limiting_gates.rb b/lib/gitlab/instrumentation/rate_limiting_gates.rb index 960b6995030..01b362f0cf4 100644 --- a/lib/gitlab/instrumentation/rate_limiting_gates.rb +++ b/lib/gitlab/instrumentation/rate_limiting_gates.rb @@ -5,9 +5,11 @@ module Gitlab class RateLimitingGates GATES = :rate_limiting_gates + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + class << self def track(key) - if ::Gitlab::SafeRequestStore.active? + if InstrumentationStorage.active? gates_set << key end end @@ -25,7 +27,7 @@ module Gitlab private def gates_set - ::Gitlab::SafeRequestStore[GATES] ||= Set.new + InstrumentationStorage[GATES] ||= Set.new end end end diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index 00a7387afe2..39290c109d7 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -9,6 +9,8 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize include ::Gitlab::Instrumentation::RedisPayload + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + # TODO: To be used by https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/395 # as a 'label' alias. def storage_key @@ -16,8 +18,10 @@ module Gitlab end def add_duration(duration) - ::RequestStore[call_duration_key] ||= 0 - ::RequestStore[call_duration_key] += duration + return unless InstrumentationStorage.active? + + InstrumentationStorage[call_duration_key] ||= 0 + InstrumentationStorage[call_duration_key] += duration end def add_call_details(duration, commands) @@ -31,56 +35,66 @@ module Gitlab end def increment_request_count(amount = 1) - ::RequestStore[request_count_key] ||= 0 - ::RequestStore[request_count_key] += amount + return unless InstrumentationStorage.active? + + InstrumentationStorage[request_count_key] ||= 0 + InstrumentationStorage[request_count_key] += amount end def increment_read_bytes(num_bytes) - ::RequestStore[read_bytes_key] ||= 0 - ::RequestStore[read_bytes_key] += num_bytes + return unless InstrumentationStorage.active? + + InstrumentationStorage[read_bytes_key] ||= 0 + InstrumentationStorage[read_bytes_key] += num_bytes end def increment_write_bytes(num_bytes) - ::RequestStore[write_bytes_key] ||= 0 - ::RequestStore[write_bytes_key] += num_bytes + return unless InstrumentationStorage.active? + + InstrumentationStorage[write_bytes_key] ||= 0 + InstrumentationStorage[write_bytes_key] += num_bytes end def increment_cross_slot_request_count(amount = 1) - ::RequestStore[cross_slots_key] ||= 0 - ::RequestStore[cross_slots_key] += amount + return unless InstrumentationStorage.active? + + InstrumentationStorage[cross_slots_key] ||= 0 + InstrumentationStorage[cross_slots_key] += amount end def increment_allowed_cross_slot_request_count(amount = 1) - ::RequestStore[allowed_cross_slots_key] ||= 0 - ::RequestStore[allowed_cross_slots_key] += amount + return unless InstrumentationStorage.active? + + InstrumentationStorage[allowed_cross_slots_key] ||= 0 + InstrumentationStorage[allowed_cross_slots_key] += amount end def get_request_count - ::RequestStore[request_count_key] || 0 + InstrumentationStorage[request_count_key] || 0 end def read_bytes - ::RequestStore[read_bytes_key] || 0 + InstrumentationStorage[read_bytes_key] || 0 end def write_bytes - ::RequestStore[write_bytes_key] || 0 + InstrumentationStorage[write_bytes_key] || 0 end def detail_store - ::RequestStore[call_details_key] ||= [] + InstrumentationStorage[call_details_key] ||= [] end def get_cross_slot_request_count - ::RequestStore[cross_slots_key] || 0 + InstrumentationStorage[cross_slots_key] || 0 end def get_allowed_cross_slot_request_count - ::RequestStore[allowed_cross_slots_key] || 0 + InstrumentationStorage[allowed_cross_slots_key] || 0 end def query_time - query_time = ::RequestStore[call_duration_key] || 0 + query_time = InstrumentationStorage[call_duration_key] || 0 query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) end diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index b3fbe30e583..c81f070219e 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -55,7 +55,7 @@ module Gitlab commands.each { instrumentation_class.instance_observe_duration(duration / commands.size) } end - if ::RequestStore.active? + if ::Gitlab::Instrumentation::Storage.active? # These metrics measure total Redis usage per Rails request / job. instrumentation_class.increment_request_count(commands.size) instrumentation_class.add_duration(duration) diff --git a/lib/gitlab/instrumentation/storage.rb b/lib/gitlab/instrumentation/storage.rb new file mode 100644 index 00000000000..20c1b69acdd --- /dev/null +++ b/lib/gitlab/instrumentation/storage.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Instrumentation + module Storage + extend self + + delegate :active?, to: ::Gitlab::SafeRequestStore + delegate :[], :[]=, to: :storage + + def clear! + storage.clear + end + + private + + def storage + ::Gitlab::SafeRequestStore.fetch(:instrumentation) { {} } + end + end + end +end diff --git a/lib/gitlab/instrumentation/throttle.rb b/lib/gitlab/instrumentation/throttle.rb index 0b7e990fb2e..837313127e7 100644 --- a/lib/gitlab/instrumentation/throttle.rb +++ b/lib/gitlab/instrumentation/throttle.rb @@ -3,14 +3,16 @@ module Gitlab module Instrumentation class Throttle + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + KEY = :instrumentation_throttle_safelist def self.safelist - Gitlab::SafeRequestStore[KEY] + InstrumentationStorage[KEY] end def self.safelist=(name) - Gitlab::SafeRequestStore[KEY] = name + InstrumentationStorage[KEY] = name end end end diff --git a/lib/gitlab/instrumentation/uploads.rb b/lib/gitlab/instrumentation/uploads.rb index 02e457453cd..226c4e73d01 100644 --- a/lib/gitlab/instrumentation/uploads.rb +++ b/lib/gitlab/instrumentation/uploads.rb @@ -6,19 +6,21 @@ module Gitlab UPLOAD_DURATION = :uploaded_file_upload_duration_s UPLOADED_FILE_SIZE = :uploaded_file_size_bytes + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + def self.track(uploaded_file) - if ::Gitlab::SafeRequestStore.active? - ::Gitlab::SafeRequestStore[UPLOAD_DURATION] = uploaded_file.upload_duration - ::Gitlab::SafeRequestStore[UPLOADED_FILE_SIZE] = uploaded_file.size + if InstrumentationStorage.active? + InstrumentationStorage[UPLOAD_DURATION] = uploaded_file.upload_duration + InstrumentationStorage[UPLOADED_FILE_SIZE] = uploaded_file.size end end def self.get_upload_duration - ::Gitlab::SafeRequestStore[UPLOAD_DURATION] + InstrumentationStorage[UPLOAD_DURATION] end def self.get_uploaded_file_size - ::Gitlab::SafeRequestStore[UPLOADED_FILE_SIZE] + InstrumentationStorage[UPLOADED_FILE_SIZE] end def self.payload diff --git a/lib/gitlab/instrumentation/zoekt.rb b/lib/gitlab/instrumentation/zoekt.rb index cd9b15bcee8..4d2933680c5 100644 --- a/lib/gitlab/instrumentation/zoekt.rb +++ b/lib/gitlab/instrumentation/zoekt.rb @@ -3,32 +3,34 @@ module Gitlab module Instrumentation class Zoekt + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + ZOEKT_REQUEST_COUNT = :zoekt_request_count ZOEKT_CALL_DURATION = :zoekt_call_duration ZOEKT_CALL_DETAILS = :zoekt_call_details class << self def get_request_count - ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] || 0 + InstrumentationStorage[ZOEKT_REQUEST_COUNT] || 0 end def increment_request_count - ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] ||= 0 - ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] += 1 + InstrumentationStorage[ZOEKT_REQUEST_COUNT] ||= 0 + InstrumentationStorage[ZOEKT_REQUEST_COUNT] += 1 end def detail_store - ::Gitlab::SafeRequestStore[ZOEKT_CALL_DETAILS] ||= [] + InstrumentationStorage[ZOEKT_CALL_DETAILS] ||= [] end def query_time - query_time = ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] || 0 + query_time = InstrumentationStorage[ZOEKT_CALL_DURATION] || 0 query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) end def add_duration(duration) - ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] ||= 0 - ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] += duration + InstrumentationStorage[ZOEKT_CALL_DURATION] ||= 0 + InstrumentationStorage[ZOEKT_CALL_DURATION] += duration end def add_call_details(duration:, method:, path:, params: nil, body: nil) diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 1b81ec012d1..686142e338a 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -7,6 +7,8 @@ module Gitlab DURATION_PRECISION = 6 # microseconds def init_instrumentation_data(request_ip: nil) + ::Gitlab::Instrumentation::Storage.clear! + # Set `request_start_time` only if this is request # This is done, as `request_start_time` imply `request_deadline` if request_ip diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 10bb358a292..dd99d4d770c 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -21,6 +21,8 @@ module Gitlab SQL_WAL_LOCATION_REGEX = /(pg_current_wal_insert_lsn\(\)::text|pg_last_wal_replay_lsn\(\)::text)/.freeze + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + # This event is published from ActiveRecordBaseTransactionMetrics and # used to record a database transaction duration when calling # ApplicationRecord.transaction {} block. @@ -56,20 +58,20 @@ module Gitlab end def self.db_counter_payload - return {} unless Gitlab::SafeRequestStore.active? + return {} unless InstrumentationStorage.active? {}.tap do |payload| db_counter_keys.each do |key| - payload[key] = Gitlab::SafeRequestStore[key].to_i + payload[key] = InstrumentationStorage[key].to_i end - if ::Gitlab::SafeRequestStore.active? + if InstrumentationStorage.active? load_balancing_metric_counter_keys.each do |counter| - payload[counter] = ::Gitlab::SafeRequestStore[counter].to_i + payload[counter] = InstrumentationStorage[counter].to_i end load_balancing_metric_duration_keys.each do |duration| - payload[duration] = ::Gitlab::SafeRequestStore[duration].to_f.round(3) + payload[duration] = InstrumentationStorage[duration].to_f.round(3) end end end @@ -100,16 +102,16 @@ module Gitlab buckets ::Gitlab::Metrics::Subscribers::ActiveRecord::SQL_DURATION_BUCKET end - return unless ::Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? duration = event.duration / 1000.0 duration_key = compose_metric_key(:duration_s, db_role) - ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration + InstrumentationStorage[duration_key] = (InstrumentationStorage[duration_key].presence || 0) + duration # Per database metrics db_config_name = db_config_name(event.payload) duration_key = compose_metric_key(:duration_s, nil, db_config_name) - ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration + InstrumentationStorage[duration_key] = (InstrumentationStorage[duration_key].presence || 0) + duration end def ignored_query?(payload) @@ -135,14 +137,14 @@ module Gitlab current_transaction&.increment(prometheus_key, 1, { db_config_name: db_config_name }) - Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1 + InstrumentationStorage[log_key] = InstrumentationStorage[log_key].to_i + 1 # To avoid confusing log keys we only log the db_config_name metrics # when we are also logging the db_role. Otherwise it will be hard to # tell if the log key is referring to a db_role OR a db_config_name. if db_role.present? && db_config_name.present? log_key = compose_metric_key(counter, nil, db_config_name) - Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1 + InstrumentationStorage[log_key] = InstrumentationStorage[log_key].to_i + 1 end end diff --git a/lib/gitlab/metrics/subscribers/external_http.rb b/lib/gitlab/metrics/subscribers/external_http.rb index 87756b14887..a5bfc80b3bf 100644 --- a/lib/gitlab/metrics/subscribers/external_http.rb +++ b/lib/gitlab/metrics/subscribers/external_http.rb @@ -6,6 +6,8 @@ module Gitlab # Class for tracking the total time spent in external HTTP # See more at https://gitlab.com/gitlab-org/labkit-ruby/-/blob/v0.14.0/lib/gitlab-labkit.rb#L18 class ExternalHttp < ActiveSupport::Subscriber + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + attach_to :external_http DEFAULT_STATUS_CODE = 'undefined' @@ -19,19 +21,19 @@ module Gitlab MAX_SLOW_REQUESTS = 10 def self.detail_store - ::Gitlab::SafeRequestStore[DETAIL_STORE] ||= [] + InstrumentationStorage[DETAIL_STORE] ||= [] end def self.duration - Gitlab::SafeRequestStore[DURATION].to_f + InstrumentationStorage[DURATION].to_f end def self.request_count - Gitlab::SafeRequestStore[COUNTER].to_i + InstrumentationStorage[COUNTER].to_i end def self.slow_requests - Gitlab::SafeRequestStore[SLOW_REQUESTS] + InstrumentationStorage[SLOW_REQUESTS] end def self.top_slowest_requests @@ -82,14 +84,14 @@ module Gitlab end def add_to_request_store(payload) - return unless Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? - Gitlab::SafeRequestStore[COUNTER] = Gitlab::SafeRequestStore[COUNTER].to_i + 1 - Gitlab::SafeRequestStore[DURATION] = Gitlab::SafeRequestStore[DURATION].to_f + payload[:duration].to_f + InstrumentationStorage[COUNTER] = InstrumentationStorage[COUNTER].to_i + 1 + InstrumentationStorage[DURATION] = InstrumentationStorage[DURATION].to_f + payload[:duration].to_f if payload[:duration].to_f > THRESHOLD_SLOW_REQUEST_S - Gitlab::SafeRequestStore[SLOW_REQUESTS] ||= [] - Gitlab::SafeRequestStore[SLOW_REQUESTS] << { + InstrumentationStorage[SLOW_REQUESTS] ||= [] + InstrumentationStorage[SLOW_REQUESTS] << { method: payload[:method], host: payload[:host], port: payload[:port], diff --git a/lib/gitlab/metrics/subscribers/ldap.rb b/lib/gitlab/metrics/subscribers/ldap.rb index 3dae2d1fd88..409ecbb6e8a 100644 --- a/lib/gitlab/metrics/subscribers/ldap.rb +++ b/lib/gitlab/metrics/subscribers/ldap.rb @@ -8,6 +8,8 @@ module Gitlab # at the end of the event key, e.g. `open.net_ldap` attach_to :net_ldap + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + COUNTER = :net_ldap_count DURATION = :net_ldap_duration_s @@ -26,12 +28,12 @@ module Gitlab class << self # @return [Integer] the total number of LDAP requests def count - Gitlab::SafeRequestStore[COUNTER].to_i + InstrumentationStorage[COUNTER].to_i end # @return [Float] the total duration spent on LDAP requests def duration - Gitlab::SafeRequestStore[DURATION].to_f + InstrumentationStorage[DURATION].to_f end # Used in Gitlab::InstrumentationHelper to merge the LDAP stats @@ -71,10 +73,10 @@ module Gitlab # Track these events as statistics for the current requests, for logging purposes def add_to_request_store(event) - return unless Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? - Gitlab::SafeRequestStore[COUNTER] = self.class.count + 1 - Gitlab::SafeRequestStore[DURATION] = self.class.duration + convert_to_seconds(event.duration) + InstrumentationStorage[COUNTER] = self.class.count + 1 + InstrumentationStorage[DURATION] = self.class.duration + convert_to_seconds(event.duration) end # Converts the observed events into Prometheus metrics diff --git a/lib/gitlab/metrics/subscribers/load_balancing.rb b/lib/gitlab/metrics/subscribers/load_balancing.rb index bd77e8c3c3f..d7fe33dbe89 100644 --- a/lib/gitlab/metrics/subscribers/load_balancing.rb +++ b/lib/gitlab/metrics/subscribers/load_balancing.rb @@ -6,11 +6,13 @@ module Gitlab class LoadBalancing < ActiveSupport::Subscriber attach_to :load_balancing + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + PROMETHEUS_COUNTER = :gitlab_transaction_caught_up_replica_pick_count_total LOG_COUNTERS = { true => :caught_up_replica_pick_ok, false => :caught_up_replica_pick_fail }.freeze def caught_up_replica_pick(event) - return unless Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? result = event.payload[:result] counter_name = counter(result) @@ -20,17 +22,17 @@ module Gitlab # we want to update Prometheus counter after the controller/action are set def web_transaction_completed(_event) - return unless Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? LOG_COUNTERS.keys.each { |result| increment_prometheus_for_result_label(result) } end def self.load_balancing_payload - return {} unless Gitlab::SafeRequestStore.active? + return {} unless InstrumentationStorage.active? {}.tap do |payload| LOG_COUNTERS.values.each do |counter| - value = Gitlab::SafeRequestStore[counter] + value = InstrumentationStorage[counter] payload[counter] = value.to_i if value end @@ -40,12 +42,12 @@ module Gitlab private def increment(counter) - Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1 + InstrumentationStorage[counter] = InstrumentationStorage[counter].to_i + 1 end def increment_prometheus_for_result_label(label_value) counter_name = counter(label_value) - return unless (counter_value = Gitlab::SafeRequestStore[counter_name]) + return unless (counter_value = InstrumentationStorage[counter_name]) increment_prometheus(labels: { result: label_value }, value: counter_value.to_i) end diff --git a/lib/gitlab/metrics/subscribers/rack_attack.rb b/lib/gitlab/metrics/subscribers/rack_attack.rb index 2196122df01..705536039ed 100644 --- a/lib/gitlab/metrics/subscribers/rack_attack.rb +++ b/lib/gitlab/metrics/subscribers/rack_attack.rb @@ -13,10 +13,12 @@ module Gitlab class RackAttack < ActiveSupport::Subscriber attach_to 'rack_attack' + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + INSTRUMENTATION_STORE_KEY = :rack_attack_instrumentation def self.payload - Gitlab::SafeRequestStore[INSTRUMENTATION_STORE_KEY] ||= { + InstrumentationStorage[INSTRUMENTATION_STORE_KEY] ||= { rack_attack_redis_count: 0, rack_attack_redis_duration_s: 0.0 } diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb index da3f67dde51..c770260a66e 100644 --- a/lib/gitlab/project_authorizations.rb +++ b/lib/gitlab/project_authorizations.rb @@ -12,7 +12,12 @@ module Gitlab end def calculate - cte = recursive_cte + cte = if Feature.enabled?(:linear_project_authorization, user) + linear_cte + else + recursive_cte + end + cte_alias = cte.table.alias(Group.table_name) projects = Project.arel_table links = ProjectGroupLink.arel_table @@ -47,11 +52,18 @@ module Gitlab .where('p_ns.share_with_group_lock IS FALSE') ] - ProjectAuthorization - .unscoped - .with - .recursive(cte.to_arel) - .select_from_union(relations) + if Feature.enabled?(:linear_project_authorization, user) + ProjectAuthorization + .unscoped + .with(cte.to_arel) + .select_from_union(relations) + else + ProjectAuthorization + .unscoped + .with + .recursive(cte.to_arel) + .select_from_union(relations) + end end private @@ -89,6 +101,30 @@ module Gitlab cte end + def linear_cte + # Groups shared with user and their parent groups + shared_groups = Group + .select("namespaces.id, MAX(LEAST(members.access_level, group_group_links.group_access)) as access_level") + .joins("INNER JOIN group_group_links ON group_group_links.shared_group_id = namespaces.id + OR namespaces.traversal_ids @> ARRAY[group_group_links.shared_group_id::int]") + .joins("INNER JOIN members ON group_group_links.shared_with_group_id = members.source_id") + .merge(user.group_members) + .merge(GroupMember.active_state) + .group("namespaces.id") + + # Groups the user is a member of and their parent groups. + lateral_query = Group.as_ids.where("namespaces.traversal_ids @> ARRAY [members.source_id]") + member_groups_with_ancestors = GroupMember.select("namespaces.id, MAX(members.access_level) as access_level") + .joins("CROSS JOIN LATERAL (#{lateral_query.to_sql}) as namespaces") + .group("namespaces.id") + .merge(user.group_members) + .merge(GroupMember.active_state) + + union = Namespace.from_union([shared_groups, member_groups_with_ancestors]) + + Gitlab::SQL::CTE.new(:linear_namespaces_cte, union) + end + # Builds a LEFT JOIN to join optional memberships onto the CTE. def join_members_on_namespaces members = Member.arel_table diff --git a/lib/gitlab/rugged_instrumentation.rb b/lib/gitlab/rugged_instrumentation.rb index 36a3a491de6..b768e89b1e4 100644 --- a/lib/gitlab/rugged_instrumentation.rb +++ b/lib/gitlab/rugged_instrumentation.rb @@ -2,14 +2,16 @@ module Gitlab module RuggedInstrumentation + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + def self.query_time - query_time = SafeRequestStore[:rugged_query_time] || 0 + query_time = InstrumentationStorage[:rugged_query_time] || 0 query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION) end def self.add_query_time(duration) - SafeRequestStore[:rugged_query_time] ||= 0 - SafeRequestStore[:rugged_query_time] += duration + InstrumentationStorage[:rugged_query_time] ||= 0 + InstrumentationStorage[:rugged_query_time] += duration end def self.query_time_ms @@ -17,29 +19,29 @@ module Gitlab end def self.query_count - SafeRequestStore[:rugged_call_count] ||= 0 + InstrumentationStorage[:rugged_call_count] ||= 0 end def self.increment_query_count - SafeRequestStore[:rugged_call_count] ||= 0 - SafeRequestStore[:rugged_call_count] += 1 + InstrumentationStorage[:rugged_call_count] ||= 0 + InstrumentationStorage[:rugged_call_count] += 1 end def self.active? - SafeRequestStore.active? + InstrumentationStorage.active? end def self.add_call_details(details) return unless Gitlab::PerformanceBar.enabled_for_request? - Gitlab::SafeRequestStore[:rugged_call_details] ||= [] - Gitlab::SafeRequestStore[:rugged_call_details] << details + InstrumentationStorage[:rugged_call_details] ||= [] + InstrumentationStorage[:rugged_call_details] << details end def self.list_call_details return [] unless Gitlab::PerformanceBar.enabled_for_request? - Gitlab::SafeRequestStore[:rugged_call_details] || [] + InstrumentationStorage[:rugged_call_details] || [] end end end |