diff options
Diffstat (limited to 'lib/gitlab')
259 files changed, 3978 insertions, 1325 deletions
diff --git a/lib/gitlab/allowable.rb b/lib/gitlab/allowable.rb index 4518c8a862c..879247d0174 100644 --- a/lib/gitlab/allowable.rb +++ b/lib/gitlab/allowable.rb @@ -2,8 +2,8 @@ module Gitlab module Allowable - def can?(*args) - Ability.allowed?(*args) + def can?(...) + Ability.allowed?(...) end end end diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb index dd8149aba94..41f94e79f91 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb @@ -24,7 +24,7 @@ module Gitlab def initialize(stage:, params: {}) @stage = stage @params = params - @root_ancestor = stage.parent.root_ancestor + @root_ancestor = stage.namespace.root_ancestor @stage_event_model = MODEL_CLASSES.fetch(stage.subject_class.to_s) end @@ -90,7 +90,7 @@ module Gitlab end def filter_by_stage_parent(query) - query.by_project_id(stage.parent_id) + query.by_project_id(stage.namespace.project.id) end def base_query diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb index 3abf380d461..8b40a8c2b26 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb @@ -5,7 +5,7 @@ module Gitlab module CycleAnalytics module Aggregated # Arguments: - # stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::Stage + # stage - an instance of CycleAnalytics::Stage # params: # current_user: an instance of User # from: DateTime diff --git a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb index c7987d63153..ca8b4a3a890 100644 --- a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb +++ b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb @@ -57,9 +57,14 @@ module Gitlab end def add_parent_model_params!(finder_params) - raise(ArgumentError, "unknown parent_class: #{parent_class}") unless parent_class.eql?(Project) - - finder_params[:project_id] = stage.parent_id + case stage.parent + when Namespaces::ProjectNamespace + finder_params[:project_id] = stage.parent.project.id + when Project + finder_params[:project_id] = stage.parent_id + else + raise(ArgumentError, "unknown parent_class: #{parent_class}") + end end def add_time_range_params!(finder_params, from, to) diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb index 0db027b9861..ffafafda4ac 100644 --- a/lib/gitlab/analytics/cycle_analytics/data_collector.rb +++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb @@ -4,7 +4,7 @@ module Gitlab module Analytics module CycleAnalytics # Arguments: - # stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::Stage + # stage - an instance of CycleAnalytics::Stage # params: # current_user: an instance of User # from: DateTime diff --git a/lib/gitlab/analytics/cycle_analytics/default_stages.rb b/lib/gitlab/analytics/cycle_analytics/default_stages.rb index 43683ae174e..0f1380125fa 100644 --- a/lib/gitlab/analytics/cycle_analytics/default_stages.rb +++ b/lib/gitlab/analytics/cycle_analytics/default_stages.rb @@ -6,7 +6,7 @@ # Example: # # params = Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_issue_stage -# Analytics::CycleAnalytics::ProjectStage.new(params) +# Analytics::CycleAnalytics::Stage.new(params) module Gitlab module Analytics module CycleAnalytics diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb index d058782ae87..2df3680db5f 100644 --- a/lib/gitlab/analytics/cycle_analytics/request_params.rb +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -196,7 +196,7 @@ module Gitlab return unless value_stream strong_memoize(:stage) do - ::Analytics::CycleAnalytics::StageFinder.new(parent: project || group, stage_id: stage_id).execute if stage_id + ::Analytics::CycleAnalytics::StageFinder.new(parent: project&.project_namespace || group, stage_id: stage_id).execute if stage_id end end end diff --git a/lib/gitlab/api_authentication/token_resolver.rb b/lib/gitlab/api_authentication/token_resolver.rb index afada055928..3f8025c408a 100644 --- a/lib/gitlab/api_authentication/token_resolver.rb +++ b/lib/gitlab/api_authentication/token_resolver.rb @@ -165,7 +165,9 @@ module Gitlab end def with_deploy_token(raw, &block) - raise ::Gitlab::Auth::UnauthorizedError if Gitlab::ExternalAuthorization.enabled? + unless Gitlab::ExternalAuthorization.allow_deploy_tokens_and_deploy_keys? + raise ::Gitlab::Auth::UnauthorizedError + end token = ::DeployToken.active.find_by_token(raw.password) return unless token diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index a788586ebec..466538df56e 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -55,7 +55,8 @@ module Gitlab phone_verification_verify_code: { threshold: 10, interval: 10.minutes }, namespace_exists: { threshold: 20, interval: 1.minute }, fetch_google_ip_list: { threshold: 10, interval: 1.minute }, - jobs_index: { threshold: 600, interval: 1.minute } + jobs_index: { threshold: 600, interval: 1.minute }, + bulk_import: { threshold: 6, interval: 1.minute } }.freeze end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index c97ef5a10ef..06bdb2c1ddc 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -32,7 +32,7 @@ module Gitlab # Scopes used for GitLab as admin SUDO_SCOPE = :sudo ADMIN_MODE_SCOPE = :admin_mode - ADMIN_SCOPES = [SUDO_SCOPE].freeze + ADMIN_SCOPES = [SUDO_SCOPE, ADMIN_MODE_SCOPE].freeze # Default scopes for OAuth applications that don't define their own DEFAULT_SCOPES = [API_SCOPE].freeze @@ -366,7 +366,10 @@ module Gitlab def available_scopes_for(current_user) scopes = non_admin_available_scopes - scopes += ADMIN_SCOPES if current_user.admin? + + if current_user.admin? # rubocop: disable Cop/UserAdmin + scopes += Feature.enabled?(:admin_mode_for_api) ? ADMIN_SCOPES : [SUDO_SCOPE] + end scopes end diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index 16bee187c87..c69462b12de 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -121,6 +121,7 @@ module Gitlab # It is also used by GraphQL/API requests. # And to allow accessing /archive programatically as it was a big pain point # for users https://gitlab.com/gitlab-org/gitlab/-/issues/28978. + # Used for release downloading as well def find_user_from_web_access_token(request_format, scopes: [:api]) return unless access_token && valid_web_access_format?(request_format) @@ -147,7 +148,7 @@ module Gitlab # deploy tokens are accepted with deploy token headers and basic auth headers def deploy_token_from_request return unless route_authentication_setting[:deploy_token_allowed] - return if Gitlab::ExternalAuthorization.enabled? + return unless Gitlab::ExternalAuthorization.allow_deploy_tokens_and_deploy_keys? token = current_request.env[DEPLOY_TOKEN_HEADER].presence || parsed_oauth_token @@ -301,6 +302,8 @@ module Gitlab api_request? when :archive archive_request? + when :download + download_request? end end @@ -352,6 +355,10 @@ module Gitlab current_request.path.include?('/-/archive/') end + def download_request? + current_request.path.include?('/downloads/') + end + def blob_request? current_request.path.include?('/raw/') end diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index 9aedc131e92..0201f1f8725 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -33,8 +33,8 @@ module Gitlab users_search(options) end - def user(*args) - users(*args).first + def user(...) + users(...).first end def dn_matches_filter?(dn, filter) diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 242390c3e89..01e126ec2f5 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -258,7 +258,7 @@ module Gitlab metadata = gl_user.build_user_synced_attributes_metadata if sync_profile_from_provider? - UserSyncedAttributesMetadata::SYNCABLE_ATTRIBUTES.each do |key| + UserSyncedAttributesMetadata.syncable_attributes.each do |key| if auth_hash.has_attribute?(key) && gl_user.sync_attribute?(key) gl_user.public_send("#{key}=".to_sym, auth_hash.public_send(key)) # rubocop:disable GitlabSecurity/PublicSend metadata.set_attribute_synced(key, true) diff --git a/lib/gitlab/background_migration/add_namespaces_emails_enabled_column_data.rb b/lib/gitlab/background_migration/add_namespaces_emails_enabled_column_data.rb new file mode 100644 index 00000000000..46e2d5cb930 --- /dev/null +++ b/lib/gitlab/background_migration/add_namespaces_emails_enabled_column_data.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Iterates through the namespaces table and attempts to set the + # opposite of the value of the column "emails_disabled" to a new + # column in namespace_settings called emails_enabled + class AddNamespacesEmailsEnabledColumnData < BatchedMigrationJob + feature_category :database + operation_name :add_namespaces_emails_enabled_column_data + + # Targeted table + class NamespaceSetting < ApplicationRecord + self.table_name = 'namespace_settings' + end + + def perform + each_sub_batch do |sub_batch| + plucked_list = sub_batch.where('NOT emails_disabled IS NULL').pluck(:id, :emails_disabled) + next if plucked_list.empty? + + ApplicationRecord.connection.execute <<~SQL + UPDATE namespace_settings + SET emails_enabled= NOT subquery.emails_enabled + FROM (SELECT * FROM (#{Arel::Nodes::ValuesList.new(plucked_list).to_sql}) AS updates(namespace_id, emails_enabled)) AS subquery + WHERE namespace_settings.namespace_id=subquery.namespace_id + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/add_projects_emails_enabled_column_data.rb b/lib/gitlab/background_migration/add_projects_emails_enabled_column_data.rb new file mode 100644 index 00000000000..a0ce5d22597 --- /dev/null +++ b/lib/gitlab/background_migration/add_projects_emails_enabled_column_data.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Iterates through the Projects table and attempts to set the + # opposite of the value of the column "emails_disabled" to a new + # column in project_settings called emails_enabled + class AddProjectsEmailsEnabledColumnData < BatchedMigrationJob + feature_category :database + operation_name :add_projects_emails_enabled_column_data + + # Targeted table + class ProjectSetting < ApplicationRecord + self.table_name = 'project_settings' + end + + def perform + each_sub_batch do |sub_batch| + plucked_list = sub_batch.where('NOT emails_disabled IS NULL').pluck(:id, :emails_disabled) + next if plucked_list.empty? + + ApplicationRecord.connection.execute <<~SQL + UPDATE project_settings + SET emails_enabled=NOT subquery.emails_enabled + FROM (SELECT * FROM (#{Arel::Nodes::ValuesList.new(plucked_list).to_sql}) AS updates(project_id, emails_enabled)) AS subquery + WHERE project_settings.project_id=subquery.project_id + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb b/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb deleted file mode 100644 index 669e5338dd1..00000000000 --- a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb +++ /dev/null @@ -1,86 +0,0 @@ -# frozen_string_literal: true - -# Based on https://community.developer.atlassian.com/t/get-rest-api-3-filter-search/29459/2, -# it's enough at the moment to simply notice if the url is from `atlassian.net` -module Gitlab - module BackgroundMigration - # Backfill the deployment_type in jira_tracker_data table - class BackfillJiraTrackerDeploymentType2 - # Migration only version of jira_tracker_data table - class JiraTrackerDataTemp < ApplicationRecord - self.table_name = 'jira_tracker_data' - - def self.encryption_options - { - key: Settings.attr_encrypted_db_key_base_32, - encode: true, - mode: :per_attribute_iv, - algorithm: 'aes-256-gcm' - } - end - - attr_encrypted :url, encryption_options - attr_encrypted :api_url, encryption_options - - enum deployment_type: { unknown: 0, server: 1, cloud: 2 }, _prefix: :deployment - end - - # Migration only version of services table - class JiraServiceTemp < ApplicationRecord - self.table_name = 'services' - self.inheritance_column = :_type_disabled - end - - def perform(start_id, stop_id) - @server_ids = [] - @cloud_ids = [] - - JiraTrackerDataTemp - .where(id: start_id..stop_id, deployment_type: 0) - .each do |jira_tracker_data| - collect_deployment_type(jira_tracker_data) - end - - unless cloud_ids.empty? - JiraTrackerDataTemp.where(id: cloud_ids) - .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:cloud]) - end - - unless server_ids.empty? - JiraTrackerDataTemp.where(id: server_ids) - .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:server]) - end - - mark_jobs_as_succeeded(start_id, stop_id) - end - - private - - attr_reader :server_ids, :cloud_ids - - def client_url(jira_tracker_data) - jira_tracker_data.api_url.presence || jira_tracker_data.url.presence - end - - def server_type(url) - url.downcase.include?('.atlassian.net') ? :cloud : :server - end - - def collect_deployment_type(jira_tracker_data) - url = client_url(jira_tracker_data) - return unless url - - case server_type(url) - when :cloud - cloud_ids << jira_tracker_data.id - else - server_ids << jira_tracker_data.id - end - end - - def mark_jobs_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name.demodulize, arguments) - end - end - end -end diff --git a/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb b/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb new file mode 100644 index 00000000000..b6e22e481fa --- /dev/null +++ b/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Migration to make sure that all the prevously saved tokens have their encrypted values in the db. + class EncryptCiTriggerToken < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :continuous_integration + scope_to ->(relation) { relation.where(encrypted_token: nil) } + operation_name :update + # Class that is imitating Ci::Trigger + class CiTrigger < ::Ci::ApplicationRecord + ALGORITHM = 'aes-256-gcm' + + self.table_name = 'ci_triggers' + + attr_encrypted :encrypted_token_tmp, + attribute: :encrypted_token, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_32, + encode: false, + encode_vi: false + + before_save :copy_token_to_encrypted_token + + def copy_token_to_encrypted_token + self.encrypted_token_tmp = token + end + end + + def perform + each_sub_batch do |sub_batch| + sub_batch.each do |trigger| + Gitlab::BackgroundMigration::EncryptCiTriggerToken::CiTrigger.find(trigger.id).save! + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/encrypt_integration_properties.rb b/lib/gitlab/background_migration/encrypt_integration_properties.rb index 3843356af69..c9582da2a51 100644 --- a/lib/gitlab/background_migration/encrypt_integration_properties.rb +++ b/lib/gitlab/background_migration/encrypt_integration_properties.rb @@ -31,7 +31,7 @@ module Gitlab def encrypt_properties data = ::Gitlab::Json.parse(properties) iv = generate_iv(ALGORITHM) - ep = self.class.encrypt(:encrypted_properties_tmp, data, { iv: iv }) + ep = self.class.attr_encrypt(:encrypted_properties_tmp, data, { iv: iv }) [ep, iv] end diff --git a/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb b/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb new file mode 100644 index 00000000000..4b6bb12c91b --- /dev/null +++ b/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A migration that will: + # * get all project statistics where packages_size is non zero + # * check the coherence with the related package files + # * fix non coherent packages_size values + class FixIncoherentPackagesSizeOnProjectStatistics < BatchedMigrationJob + MIGRATOR = 'FixIncoherentPackagesSizeOnProjectStatistics' + + feature_category :package_registry + + operation_name :fix_incorrect_packages_size + + def perform + each_sub_batch do |sub_batch| + fix_packages_size(sub_batch) + end + end + + private + + def fix_packages_size(project_statistics) + statistics_table = FixIncoherentPackagesSizeOnProjectStatistics::ProjectStatistics.arel_table + from = [ + statistics_table, + FixIncoherentPackagesSizeOnProjectStatistics::PackageFile.sum_query.arel.lateral.as('size_sum') + ] + size_sum_table = ::Arel::Table.new(:size_sum) + + project_statistics.select(:id, :project_id, :packages_size, size_sum_table[:total]) + .from(from) + .where.not(statistics_table[:packages_size].eq(size_sum_table[:total])) + .each do |stat| + increment = stat[:total].to_i - stat[:packages_size] - buffered_update(stat) + next if increment == 0 + + ::Gitlab::BackgroundMigration::Logger.info( + migrator: MIGRATOR, + project_id: stat[:project_id], + old_size: stat[:packages_size], + new_size: stat[:total].to_i + ) + + stat.becomes(FixIncoherentPackagesSizeOnProjectStatistics::ProjectStatistics) # rubocop:disable Cop/AvoidBecomes + .increment(increment) + end + end + + def buffered_update(stat) + key = "project:{#{stat[:project_id]}}:counters:ProjectStatistics:#{stat[:id]}:packages_size" + + Gitlab::Redis::SharedState.with do |redis| + redis.get(key).to_i + end + end + + # rubocop:disable Style/Documentation + class ProjectStatistics < ::ApplicationRecord + self.table_name = 'project_statistics' + + def increment(amount) + FixIncoherentPackagesSizeOnProjectStatistics::BufferedCounter.new(self).increment(amount) + end + end + + class Package < ::ApplicationRecord + self.table_name = 'packages_packages' + + has_many :package_files, + class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::PackageFile' # rubocop:disable Layout/LineLength + end + + class PackageFile < ::ApplicationRecord + self.table_name = 'packages_package_files' + + belongs_to :package, + class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::Package' # rubocop:disable Layout/LineLength + + def self.sum_query + packages = FixIncoherentPackagesSizeOnProjectStatistics::Package.arel_table + stats = FixIncoherentPackagesSizeOnProjectStatistics::ProjectStatistics.arel_table + + joins(:package) + .where(packages[:project_id].eq(stats[:project_id])) + .where.not(size: nil) + .select('SUM(packages_package_files.size) as total') + end + end + + class BufferedCounter + WORKER_DELAY = 10.minutes + + def initialize(stat) + @stat = stat + end + + def key + "project:{#{@stat.project_id}}:counters:ProjectStatistics:#{@stat.id}:packages_size" + end + + def increment(amount) + Gitlab::Redis::SharedState.with do |redis| + redis.incrby(key, amount) + end + + FlushCounterIncrementsWorker.perform_in( + WORKER_DELAY, + 'ProjectStatistics', + @stat.id, + :packages_size + ) + end + end + # rubocop:enable Style/Documentation + end + end +end diff --git a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb index 914ababa5c2..db3f98bc2ba 100644 --- a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb +++ b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb @@ -68,7 +68,7 @@ module Gitlab def valid_json?(metadata) Oj.load(metadata) true - rescue Oj::ParseError, EncodingError, JSON::ParserError, Encoding::UndefinedConversionError + rescue Oj::ParseError, EncodingError, JSON::ParserError, JSON::GeneratorError, Encoding::UndefinedConversionError false end diff --git a/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb new file mode 100644 index 00000000000..592ef3220ff --- /dev/null +++ b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to nullify `projects.creator_id` column of projects who creator + # does not exist in `users` table anymore. + class NullifyCreatorIdColumnOfOrphanedProjects < BatchedMigrationJob + scope_to ->(relation) do + relation.where.not(creator_id: nil) + .joins('LEFT OUTER JOIN users ON users.id = projects.creator_id') + .where(users: { id: nil }) + end + + operation_name :update_all + feature_category :projects + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all(creator_id: nil) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/rebalance_partition_id.rb b/lib/gitlab/background_migration/rebalance_partition_id.rb new file mode 100644 index 00000000000..7000ae5a856 --- /dev/null +++ b/lib/gitlab/background_migration/rebalance_partition_id.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This rebalances partition_id to fix invalid records in production + class RebalancePartitionId < BatchedMigrationJob + INVALID_PARTITION_ID = 101 + VALID_PARTITION_ID = 100 + + scope_to ->(relation) { relation.where(partition_id: INVALID_PARTITION_ID) } + operation_name :update_all + feature_category :continuous_integration + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all(partition_id: VALID_PARTITION_ID) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/sanitize_confidential_todos.rb b/lib/gitlab/background_migration/sanitize_confidential_todos.rb index 2df0b8a4d93..8215e92cbeb 100644 --- a/lib/gitlab/background_migration/sanitize_confidential_todos.rb +++ b/lib/gitlab/background_migration/sanitize_confidential_todos.rb @@ -10,43 +10,14 @@ module Gitlab # to extract all related logic. # Details in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87908#note_952459215 class SanitizeConfidentialTodos < BatchedMigrationJob - scope_to ->(relation) { relation.where(confidential: true) } - operation_name :delete_invalid_todos feature_category :database def perform - each_sub_batch do |sub_batch| - delete_ids = invalid_todo_ids(sub_batch) - - Todo.where(id: delete_ids).delete_all if delete_ids.present? - end - end - - private - - def invalid_todo_ids(notes_batch) - todos = Todo.where(note_id: notes_batch.select(:id)).includes(:note, :user) - - todos.each_with_object([]) do |todo, ids| - ids << todo.id if invalid_todo?(todo) - end - end - - def invalid_todo?(todo) - return false unless todo.note - return false if Ability.allowed?(todo.user, :read_todo, todo) - - logger.info( - message: "#{self.class.name} deleting invalid todo", - attributes: todo.attributes - ) - - true - end - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build + # no-op: this BG migration is left here only for compatibility reasons, + # but it's not scheduled from any migration anymore. + # It was a temporary migration which used not-isolated code. + # https://gitlab.com/gitlab-org/gitlab/-/issues/382557 end end end diff --git a/lib/gitlab/background_migration/third_recount_epic_cache_counts.rb b/lib/gitlab/background_migration/third_recount_epic_cache_counts.rb new file mode 100644 index 00000000000..24080a06c46 --- /dev/null +++ b/lib/gitlab/background_migration/third_recount_epic_cache_counts.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class ThirdRecountEpicCacheCounts < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :database + + def perform; end + end + # rubocop: enable Style/Documentation + end +end + +# rubocop: disable Layout/LineLength +# we just want to re-enqueue the previous BackfillEpicCacheCounts migration, +# because it's a EE-only migation and it's a module, we just prepend new +# RecountEpicCacheCounts with existing batched migration module (which is same in both cases) +Gitlab::BackgroundMigration::ThirdRecountEpicCacheCounts.prepend_mod_with('Gitlab::BackgroundMigration::BackfillEpicCacheCounts') +# rubocop: enable Layout/LineLength diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 49b8ab760f3..3dafe7c8962 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -92,15 +92,15 @@ module Gitlab def import_issues return unless repo.issues_enabled? - # If a user creates an issue while the import is in progress, this can lead to an import failure. - # The workaround is to allocate IIDs before starting the importer. - allocate_issues_internal_id!(project, client) - create_labels issue_type_id = ::WorkItems::Type.default_issue_type.id - client.issues(repo).each do |issue| + client.issues(repo).each_with_index do |issue, index| + # If a user creates an issue while the import is in progress, this can lead to an import failure. + # The workaround is to allocate IIDs before starting the importer. + allocate_issues_internal_id!(project, client) if index == 0 + import_issue(issue, issue_type_id) end end diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index 242979da367..ea9b79c12fd 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -55,7 +55,7 @@ module Gitlab handle_errors metrics.track_finished_import - log_info(stage: "complete") + log_info(import_stage: "complete") Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT) true @@ -139,16 +139,16 @@ module Gitlab end def import_repository - log_info(stage: 'import_repository', message: 'starting import') + log_info(import_stage: 'import_repository', message: 'starting import') project.repository.import_repository(project.import_url) project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap) - log_info(stage: 'import_repository', message: 'finished import') + log_info(import_stage: 'import_repository', message: 'finished import') rescue ::Gitlab::Git::CommandError => e Gitlab::ErrorTracking.log_exception( e, - stage: 'import_repository', message: 'failed import', error: e.message + import_stage: 'import_repository', message: 'failed import', error: e.message ) # Expire cache to prevent scenarios such as: @@ -179,10 +179,10 @@ module Gitlab def import_pull_requests page = 0 - log_info(stage: 'import_pull_requests', message: "starting") + log_info(import_stage: 'import_pull_requests', message: "starting") loop do - log_debug(stage: 'import_pull_requests', message: "importing page #{page} and batch-size #{BATCH_SIZE} from #{page * BATCH_SIZE} to #{(page + 1) * BATCH_SIZE}") + log_debug(import_stage: 'import_pull_requests', message: "importing page #{page} and batch-size #{BATCH_SIZE} from #{page * BATCH_SIZE} to #{(page + 1) * BATCH_SIZE}") pull_requests = client.pull_requests(project_key, repository_slug, page_offset: page, limit: BATCH_SIZE).to_a @@ -196,21 +196,21 @@ module Gitlab pull_requests.each do |pull_request| if already_imported?(pull_request) - log_info(stage: 'import_pull_requests', message: 'already imported', iid: pull_request.iid) + log_info(import_stage: 'import_pull_requests', message: 'already imported', iid: pull_request.iid) else import_bitbucket_pull_request(pull_request) end rescue StandardError => e Gitlab::ErrorTracking.log_exception( e, - stage: 'import_pull_requests', iid: pull_request.iid, error: e.message + import_stage: 'import_pull_requests', iid: pull_request.iid, error: e.message ) backtrace = Gitlab::BacktraceCleaner.clean_backtrace(e.backtrace) errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, backtrace: backtrace.join("\n"), raw_response: pull_request.raw } end - log_debug(stage: 'import_pull_requests', message: "finished page #{page} and batch-size #{BATCH_SIZE}") + log_debug(import_stage: 'import_pull_requests', message: "finished page #{page} and batch-size #{BATCH_SIZE}") page += 1 end end @@ -235,7 +235,7 @@ module Gitlab rescue BitbucketServer::Connection::ConnectionError => e Gitlab::ErrorTracking.log_exception( e, - stage: 'delete_temp_branches', branch: branch.name, error: e.message + import_stage: 'delete_temp_branches', branch: branch.name, error: e.message ) @errors << { type: :delete_temp_branches, branch_name: branch.name, errors: e.message } @@ -243,7 +243,7 @@ module Gitlab end def import_bitbucket_pull_request(pull_request) - log_info(stage: 'import_bitbucket_pull_requests', message: 'starting', iid: pull_request.iid) + log_info(import_stage: 'import_bitbucket_pull_requests', message: 'starting', iid: pull_request.iid) description = '' description += author_line(pull_request) @@ -274,12 +274,12 @@ module Gitlab metrics.merge_requests_counter.increment end - log_info(stage: 'import_bitbucket_pull_requests', message: 'finished', iid: pull_request.iid) + log_info(import_stage: 'import_bitbucket_pull_requests', message: 'finished', iid: pull_request.iid) mark_as_imported(pull_request) end def import_pull_request_comments(pull_request, merge_request) - log_info(stage: 'import_pull_request_comments', message: 'starting', iid: merge_request.iid) + log_info(import_stage: 'import_pull_request_comments', message: 'starting', iid: merge_request.iid) comments, other_activities = client.activities(project_key, repository_slug, pull_request.iid).partition(&:comment?) @@ -291,7 +291,7 @@ module Gitlab import_inline_comments(inline_comments.map(&:comment), merge_request) import_standalone_pr_comments(pr_comments.map(&:comment), merge_request) - log_info(stage: 'import_pull_request_comments', message: 'finished', iid: merge_request.iid, + log_info(import_stage: 'import_pull_request_comments', message: 'finished', iid: merge_request.iid, merge_event_found: merge_event.present?, inline_comments_count: inline_comments.count, standalone_pr_comments: pr_comments.count) @@ -299,7 +299,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def import_merge_event(merge_request, merge_event) - log_info(stage: 'import_merge_event', message: 'starting', iid: merge_request.iid) + log_info(import_stage: 'import_merge_event', message: 'starting', iid: merge_request.iid) committer = merge_event.committer_email @@ -309,12 +309,12 @@ module Gitlab metric = MergeRequest::Metrics.find_or_initialize_by(merge_request: merge_request) metric.update(merged_by_id: user_id, merged_at: timestamp) - log_info(stage: 'import_merge_event', message: 'finished', iid: merge_request.iid) + log_info(import_stage: 'import_merge_event', message: 'finished', iid: merge_request.iid) end # rubocop: enable CodeReuse/ActiveRecord def import_inline_comments(inline_comments, merge_request) - log_info(stage: 'import_inline_comments', message: 'starting', iid: merge_request.iid) + log_info(import_stage: 'import_inline_comments', message: 'starting', iid: merge_request.iid) inline_comments.each do |comment| position = build_position(merge_request, comment) @@ -329,7 +329,7 @@ module Gitlab end end - log_info(stage: 'import_inline_comments', message: 'finished', iid: merge_request.iid) + log_info(import_stage: 'import_inline_comments', message: 'finished', iid: merge_request.iid) end def create_diff_note(merge_request, comment, position, discussion_id = nil) @@ -344,7 +344,7 @@ module Gitlab return note end - log_info(stage: 'create_diff_note', message: 'creating fallback DiffNote', iid: merge_request.iid) + log_info(import_stage: 'create_diff_note', message: 'creating fallback DiffNote', iid: merge_request.iid) # Bitbucket Server supports the ability to comment on any line, not just the # line in the diff. If we can't add the note as a DiffNote, fallback to creating @@ -353,7 +353,7 @@ module Gitlab rescue StandardError => e Gitlab::ErrorTracking.log_exception( e, - stage: 'create_diff_note', comment_id: comment.id, error: e.message + import_stage: 'create_diff_note', comment_id: comment.id, error: e.message ) errors << { type: :pull_request, id: comment.id, errors: e.message } @@ -394,7 +394,7 @@ module Gitlab rescue StandardError => e Gitlab::ErrorTracking.log_exception( e, - stage: 'import_standalone_pr_comments', merge_request_id: merge_request.id, comment_id: comment.id, error: e.message + import_stage: 'import_standalone_pr_comments', merge_request_id: merge_request.id, comment_id: comment.id, error: e.message ) errors << { type: :pull_request, comment_id: comment.id, errors: e.message } diff --git a/lib/gitlab/cache/helpers.rb b/lib/gitlab/cache/helpers.rb index 024fa48c066..0fc0b1504af 100644 --- a/lib/gitlab/cache/helpers.rb +++ b/lib/gitlab/cache/helpers.rb @@ -45,7 +45,14 @@ module Gitlab def contextual_cache_key(presenter, object, context) return object.cache_key if context.nil? - [presenter.class.name, object.cache_key, context.call(object)].flatten.join(":") + [presenter_class_name(presenter), object.cache_key, context.call(object)].flatten.join(":") + end + + def presenter_class_name(presenter) + return presenter.class.name if presenter.is_a?(BaseSerializer) + return presenter.name if presenter.is_a?(Class) && presenter < Grape::Entity + + raise ArgumentError, "presenter #{presenter} is not supported" end # Used for fetching or rendering a single object diff --git a/lib/gitlab/cache/metadata.rb b/lib/gitlab/cache/metadata.rb new file mode 100644 index 00000000000..d6c89b5b2c3 --- /dev/null +++ b/lib/gitlab/cache/metadata.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + # Value object for cache metadata + class Metadata + VALID_BACKING_RESOURCES = [:cpu, :database, :gitaly, :memory, :unknown].freeze + DEFAULT_BACKING_RESOURCE = :unknown + + def initialize( + cache_identifier:, + feature_category:, + caller_id: Gitlab::ApplicationContext.current_context_attribute(:caller_id), + backing_resource: DEFAULT_BACKING_RESOURCE + ) + @cache_identifier = cache_identifier + @feature_category = Gitlab::FeatureCategories.default.get!(feature_category) + @caller_id = caller_id + @backing_resource = fetch_backing_resource!(backing_resource) + end + + attr_reader :caller_id, :cache_identifier, :feature_category, :backing_resource + + private + + def fetch_backing_resource!(resource) + return resource if VALID_BACKING_RESOURCES.include?(resource) + + raise "Unknown backing resource: #{resource}" if Gitlab.dev_or_test_env? + + DEFAULT_BACKING_RESOURCE + end + end + end +end diff --git a/lib/gitlab/cache/metrics.rb b/lib/gitlab/cache/metrics.rb index 0143052beb1..00d4e6e4d4e 100644 --- a/lib/gitlab/cache/metrics.rb +++ b/lib/gitlab/cache/metrics.rb @@ -5,19 +5,9 @@ module Gitlab module Cache class Metrics DEFAULT_BUCKETS = [0, 1, 5].freeze - VALID_BACKING_RESOURCES = [:cpu, :database, :gitaly, :memory, :unknown].freeze - DEFAULT_BACKING_RESOURCE = :unknown - def initialize( - caller_id:, - cache_identifier:, - feature_category: ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT, - backing_resource: DEFAULT_BACKING_RESOURCE - ) - @caller_id = caller_id - @cache_identifier = cache_identifier - @feature_category = Gitlab::FeatureCategories.default.get!(feature_category) - @backing_resource = fetch_backing_resource!(backing_resource) + def initialize(cache_metadata) + @cache_metadata = cache_metadata end # Increase cache hit counter @@ -51,7 +41,7 @@ module Gitlab private - attr_reader :caller_id, :cache_identifier, :feature_category, :backing_resource + attr_reader :cache_metadata def counter @counter ||= Gitlab::Metrics.counter(:redis_hit_miss_operations_total, "Hit/miss Redis cache counter") @@ -68,20 +58,12 @@ module Gitlab def labels @labels ||= { - caller_id: caller_id, - cache_identifier: cache_identifier, - feature_category: feature_category, - backing_resource: backing_resource + caller_id: cache_metadata.caller_id, + cache_identifier: cache_metadata.cache_identifier, + feature_category: cache_metadata.feature_category, + backing_resource: cache_metadata.backing_resource } end - - def fetch_backing_resource!(resource) - return resource if VALID_BACKING_RESOURCES.include?(resource) - - raise "Unknown backing resource: #{resource}" if Gitlab.dev_or_test_env? - - DEFAULT_BACKING_RESOURCE - end end end end diff --git a/lib/gitlab/chat/responder.rb b/lib/gitlab/chat/responder.rb index 53a625e9d43..478be5bd350 100644 --- a/lib/gitlab/chat/responder.rb +++ b/lib/gitlab/chat/responder.rb @@ -11,10 +11,21 @@ module Gitlab # # build - A `Ci::Build` that executed a chat command. def self.responder_for(build) - integration = build.pipeline.chat_data&.chat_name&.integration + if Feature.enabled?(:use_response_url_for_chat_responder) + response_url = build.pipeline.chat_data&.response_url + return unless response_url - if (responder = integration.try(:chat_responder)) - responder.new(build) + if response_url.start_with?('https://hooks.slack.com/') + Gitlab::Chat::Responder::Slack.new(build) + else + Gitlab::Chat::Responder::Mattermost.new(build) + end + else + integration = build.pipeline.chat_data&.chat_name&.integration + + if (responder = integration.try(:chat_responder)) + responder.new(build) + end end end end diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index abe2f272ca7..21fc2980cdc 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -26,7 +26,7 @@ module Gitlab # Without forcing the encoding to UTF-8 and then replacing # invalid UTF-8 sequences we can get an error when serializing # the Hash to JSON. - # Encoding::UndefinedConversionError: + # Encoding::UndefinedConversionError (or possibly JSON::GeneratorError in json 2.6.1+): # "\xE2" from ASCII-8BIT to UTF-8 { text: encode_utf8_no_detect(text) }.tap do |result| result[:style] = style.to_s if style.set? diff --git a/lib/gitlab/ci/artifacts/logger.rb b/lib/gitlab/ci/artifacts/logger.rb index 628f4129df4..63064118232 100644 --- a/lib/gitlab/ci/artifacts/logger.rb +++ b/lib/gitlab/ci/artifacts/logger.rb @@ -29,17 +29,19 @@ module Gitlab ) end - def self.log_created(artifact) - payload = Gitlab::ApplicationContext.current.merge( - message: 'Artifact created', - job_artifact_id: artifact.id, - size: artifact.size, - type: artifact.file_type, - build_id: artifact.job_id, - project_id: artifact.project_id - ) + def self.log_created(job_artifacts) + Array(job_artifacts).each do |artifact| + payload = Gitlab::ApplicationContext.current.merge( + message: 'Artifact created', + job_artifact_id: artifact.id, + size: artifact.size, + file_type: artifact.file_type, + build_id: artifact.job_id, + project_id: artifact.project_id + ) - Gitlab::AppLogger.info(payload) + Gitlab::AppLogger.info(payload) + end end def self.log_deleted(job_artifacts, method) @@ -49,7 +51,7 @@ module Gitlab job_artifact_id: artifact.id, expire_at: artifact.expire_at, size: artifact.size, - type: artifact.file_type, + file_type: artifact.file_type, build_id: artifact.job_id, project_id: artifact.project_id, method: method diff --git a/lib/gitlab/ci/build/auto_retry.rb b/lib/gitlab/ci/build/auto_retry.rb index 4950a7616c8..453c293f6cd 100644 --- a/lib/gitlab/ci/build/auto_retry.rb +++ b/lib/gitlab/ci/build/auto_retry.rb @@ -47,7 +47,9 @@ class Gitlab::Ci::Build::AutoRetry end def options_retry_when - options_retry.fetch(:when, ['always']) + default = ['always'] + + options_retry.fetch(:when, default) || default end def retry_on_reason_or_always? diff --git a/lib/gitlab/ci/build/context/build.rb b/lib/gitlab/ci/build/context/build.rb index 1025e1cc2d7..81efbdb297b 100644 --- a/lib/gitlab/ci/build/context/build.rb +++ b/lib/gitlab/ci/build/context/build.rb @@ -9,24 +9,19 @@ module Gitlab attr_reader :attributes - def initialize(pipeline, attributes = {}, build = nil) + def initialize(pipeline, attributes = {}) super(pipeline) - @build = build @attributes = attributes end def variables - build.scoped_variables + stub_build.scoped_variables end strong_memoize_attr :variables private - def build - @build || stub_build - end - def stub_build # This is a temporary piece of technical debt to allow us access # to the CI variables to evaluate rules before we persist a Build diff --git a/lib/gitlab/ci/components/instance_path.rb b/lib/gitlab/ci/components/instance_path.rb new file mode 100644 index 00000000000..010ce57d2a0 --- /dev/null +++ b/lib/gitlab/ci/components/instance_path.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Components + class InstancePath + include Gitlab::Utils::StrongMemoize + + def self.match?(address) + address.include?('@') && address.start_with?(Settings.gitlab_ci['component_fqdn']) + end + + attr_reader :host + + def initialize(address:, content_filename:) + @full_path, @version = address.to_s.split('@', 2) + @content_filename = content_filename + @host = Settings.gitlab_ci['component_fqdn'] + end + + def fetch_content!(current_user:) + return unless project + return unless sha + + raise Gitlab::Access::AccessDeniedError unless Ability.allowed?(current_user, :download_code, project) + + project.repository.blob_data_at(sha, project_file_path) + end + + def project + find_project_by_component_path(instance_path) + end + strong_memoize_attr :project + + def project_file_path + return unless project + + component_dir = instance_path.delete_prefix(project.full_path) + File.join(component_dir, @content_filename).delete_prefix('/') + end + + # TODO: Add support when version is a released tag and "~latest" moving target + def sha + return unless project + + project.commit(version)&.id + end + strong_memoize_attr :sha + + private + + attr_reader :version, :path + + def instance_path + @full_path.delete_prefix(host) + end + + # Given a path like "my-org/sub-group/the-project/path/to/component" + # find the project "my-org/sub-group/the-project" by looking at all possible paths. + def find_project_by_component_path(path) + possible_paths = [path] + + while index = path.rindex('/') # find index of last `/` in a path + possible_paths << (path = path[0..index - 1]) + end + + # remove shortest path as it is group + possible_paths.pop + + ::Project.where_full_path_in(possible_paths).take # rubocop: disable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 142f0b8dfd8..585e671ce42 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -117,7 +117,8 @@ module Gitlab def expand_config(config) build_config(config) - rescue Gitlab::Config::Loader::Yaml::DataTooLargeError => e + rescue Gitlab::Config::Loader::Yaml::DataTooLargeError, + Gitlab::Config::Loader::MultiDocYaml::DataTooLargeError => e track_and_raise_for_dev_exception(e) raise Config::ConfigError, e.message diff --git a/lib/gitlab/ci/config/entry/include.rb b/lib/gitlab/ci/config/entry/include.rb index 368d8f07f8d..baab7ebdb79 100644 --- a/lib/gitlab/ci/config/entry/include.rb +++ b/lib/gitlab/ci/config/entry/include.rb @@ -12,7 +12,7 @@ module Gitlab include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[local file remote template artifact job project ref rules].freeze + ALLOWED_KEYS = %i[local file remote template component artifact job project ref rules].freeze validations do validates :config, hash_or_string: true @@ -36,8 +36,8 @@ module Gitlab end entry :rules, ::Gitlab::Ci::Config::Entry::Include::Rules, - description: 'List of evaluable Rules to determine file inclusion.', - inherit: false + description: 'List of evaluable Rules to determine file inclusion.', + inherit: false attributes :rules diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 138e79db331..6eef279d3de 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -10,6 +10,7 @@ module Gitlab TimeoutError = Class.new(StandardError) MAX_INCLUDES = 100 + NEW_MAX_INCLUDES = 150 # Update to MAX_INCLUDES when FF ci_includes_count_duplicates is removed include ::Gitlab::Utils::StrongMemoize @@ -27,10 +28,10 @@ module Gitlab @user = user @parent_pipeline = parent_pipeline @variables = variables || Ci::Variables::Collection.new - @expandset = Set.new + @expandset = Feature.enabled?(:ci_includes_count_duplicates, project) ? [] : Set.new @execution_deadline = 0 @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) - @max_includes = MAX_INCLUDES + @max_includes = Feature.enabled?(:ci_includes_count_duplicates, project) ? NEW_MAX_INCLUDES : MAX_INCLUDES yield self if block_given? end diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb index 140cbfac5c1..0b90d240a15 100644 --- a/lib/gitlab/ci/config/external/file/artifact.rb +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -20,8 +20,6 @@ module Gitlab def content strong_memoize(:content) do - next unless artifact_job - Gitlab::Ci::ArtifactFileReader.new(artifact_job).read(location) rescue Gitlab::Ci::ArtifactFileReader::Error => error errors.push(error.message) @@ -36,8 +34,6 @@ module Gitlab ) end - private - def validate_context! context.logger.instrument(:config_file_artifact_validate_context) do if !creating_child_pipeline? @@ -54,10 +50,10 @@ module Gitlab errors.push("File `#{masked_location}` is empty!") unless content.present? end + private + def artifact_job strong_memoize(:artifact_job) do - next unless creating_child_pipeline? - context.parent_pipeline.find_job_with_archive_artifacts(job_name) end end diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 7899fe0ff73..84f34f2584b 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -46,13 +46,6 @@ module Gitlab expanded_content_hash end - def validate! - validate_location! - validate_context! if valid? - fetch_and_validate_content! if valid? - load_and_validate_expanded_hash! if valid? - end - def metadata { context_project: context.project&.full_path, @@ -68,22 +61,16 @@ module Gitlab [params, context.project&.full_path, context.sha].hash end - protected - - def expanded_content_hash - return unless content_hash - - strong_memoize(:expanded_content_yaml) do - expand_includes(content_hash) + def load_and_validate_expanded_hash! + context.logger.instrument(:config_file_fetch_content_hash) do + content_hash # calling the method loads then memoizes the result end - end - def content_hash - strong_memoize(:content_yaml) do - ::Gitlab::Ci::Config::Yaml.load!(content) + context.logger.instrument(:config_file_expand_content_includes) do + expanded_content_hash # calling the method expands then memoizes the result end - rescue Gitlab::Config::Loader::FormatError - nil + + validate_hash! end def validate_location! @@ -98,34 +85,28 @@ module Gitlab raise NotImplementedError, 'subclass must implement validate_context' end - def fetch_and_validate_content! - context.logger.instrument(:config_file_fetch_content) do - content # calling the method fetches then memoizes the result - end - - return if errors.any? - - context.logger.instrument(:config_file_validate_content) do - validate_content! + def validate_content! + if content.blank? + errors.push("Included file `#{masked_location}` is empty or does not exist!") end end - def load_and_validate_expanded_hash! - context.logger.instrument(:config_file_fetch_content_hash) do - content_hash # calling the method loads then memoizes the result - end + protected - context.logger.instrument(:config_file_expand_content_includes) do - expanded_content_hash # calling the method expands then memoizes the result - end + def expanded_content_hash + return unless content_hash - validate_hash! + strong_memoize(:expanded_content_hash) do + expand_includes(content_hash) + end end - def validate_content! - if content.blank? - errors.push("Included file `#{masked_location}` is empty or does not exist!") + def content_hash + strong_memoize(:content_hash) do + ::Gitlab::Ci::Config::Yaml.load!(content) end + rescue Gitlab::Config::Loader::FormatError + nil end def validate_hash! diff --git a/lib/gitlab/ci/config/external/file/component.rb b/lib/gitlab/ci/config/external/file/component.rb new file mode 100644 index 00000000000..33e7724bf9b --- /dev/null +++ b/lib/gitlab/ci/config/external/file/component.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + module File + class Component < Base + extend ::Gitlab::Utils::Override + include Gitlab::Utils::StrongMemoize + + def initialize(params, context) + @location = params[:component] + super + end + + def matching? + super && ::Feature.enabled?(:ci_include_components, context.project) + end + + def content + return unless component_result.success? + + component_result.payload.fetch(:content) + end + strong_memoize_attr :content + + def metadata + super.merge( + type: :component, + location: masked_location, + blob: masked_blob, + raw: nil, + extra: {} + ) + end + + def validate_location! + return unless invalid_location_type? + + errors.push("Included file `#{masked_location}` needs to be a string") + end + + def validate_context! + return if context.project&.repository + + errors.push('Unable to use components outside of a project context') + end + + def validate_content! + return if content.present? + + errors.push(component_result.message) + end + + private + + attr_reader :path, :version + + def component_result + ::Ci::Components::FetchService.new( + address: location, + current_user: context.user + ).execute + end + strong_memoize_attr :component_result + + override :expand_context_attrs + def expand_context_attrs + { + project: component_path.project, + sha: component_path.sha, + user: context.user, + variables: context.variables + } + end + + def masked_blob + return unless component_path + + context.mask_variables_from( + Gitlab::Routing.url_helpers.project_blob_url( + component_path.project, + ::File.join(component_path.sha, component_path.project_file_path)) + ) + end + strong_memoize_attr :masked_blob + + def component_path + return unless component_result.success? + + component_result.payload.fetch(:path) + end + strong_memoize_attr :component_path + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index 0912a732158..bb1c304d02b 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -10,7 +10,8 @@ module Gitlab include Gitlab::Utils::StrongMemoize def initialize(params, context) - @location = params[:local] + # `Repository#blobs_at` does not support files with the `/` prefix. + @location = Gitlab::Utils.remove_leading_slashes(params[:local]) super end @@ -29,8 +30,6 @@ module Gitlab ) end - private - def validate_context! return if context.project&.repository @@ -45,14 +44,19 @@ module Gitlab end end + private + def fetch_local_content - context.logger.instrument(:config_file_fetch_local_content) do - context.project.repository.blob_data_at(context.sha, location) + BatchLoader.for([context.sha, location]) + .batch(key: context.project) do |locations, loader, args| + context.logger.instrument(:config_file_fetch_local_content) do + args[:key].repository.blobs_at(locations).each do |blob| + loader.call([blob.commit_id, blob.path], blob.data) + end + end + rescue GRPC::InvalidArgument + # no-op end - rescue GRPC::InvalidArgument - errors.push("Sha #{context.sha} is not valid!") - - nil end override :expand_context_attrs @@ -67,6 +71,8 @@ module Gitlab end def masked_blob + return unless valid? + strong_memoize(:masked_blob) do context.mask_variables_from( Gitlab::Routing.url_helpers.project_blob_url(context.project, ::File.join(context.sha, location)) @@ -75,7 +81,7 @@ module Gitlab end def masked_raw - return unless context.project + return unless valid? strong_memoize(:masked_raw) do context.mask_variables_from( diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index 553cbd819ad..f8d4cb27710 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -12,7 +12,9 @@ module Gitlab attr_reader :project_name, :ref_name def initialize(params, context) - @location = params[:file] + # `Repository#blobs_at` does not support files with the `/` prefix. + @location = Gitlab::Utils.remove_leading_slashes(params[:file]) + @project_name = get_project_name(params[:project]) @ref_name = params[:ref] || 'HEAD' @@ -37,8 +39,6 @@ module Gitlab ) end - private - def validate_context! if !can_access_local_content? errors.push("Project `#{masked_project_name}` not found or access denied! Make sure any includes in the pipeline configuration are correctly defined.") @@ -55,6 +55,8 @@ module Gitlab end end + private + def project strong_memoize(:project) do ::Project.find_by_full_path(project_name) @@ -70,19 +72,19 @@ module Gitlab end def fetch_local_content - return unless can_access_local_content? - return unless sha - - context.logger.instrument(:config_file_fetch_project_content) do - project.repository.blob_data_at(sha, location) + BatchLoader.for([sha, location]) + .batch(key: project) do |locations, loader, args| + context.logger.instrument(:config_file_fetch_project_content) do + args[:key].repository.blobs_at(locations).each do |blob| + loader.call([blob.commit_id, blob.path], blob.data) + end + end + rescue GRPC::NotFound, GRPC::Internal + # no-op end - rescue GRPC::NotFound, GRPC::Internal - nil end def sha - return unless project - strong_memoize(:sha) do project.commit(ref_name).try(:sha) end @@ -112,7 +114,7 @@ module Gitlab end def masked_blob - return unless project + return unless valid? strong_memoize(:masked_blob) do context.mask_variables_from( @@ -122,7 +124,7 @@ module Gitlab end def masked_raw - return unless project + return unless valid? strong_memoize(:masked_raw) do context.mask_variables_from( diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index ed37357dc53..bc8cebb8c3e 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -28,8 +28,6 @@ module Gitlab ) end - private - def validate_context! # no-op end @@ -42,6 +40,8 @@ module Gitlab end end + private + def fetch_remote_content begin response = context.logger.instrument(:config_file_fetch_remote_content) do diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb index 53236cb317b..093b945da4b 100644 --- a/lib/gitlab/ci/config/external/file/template.rb +++ b/lib/gitlab/ci/config/external/file/template.rb @@ -31,8 +31,6 @@ module Gitlab ) end - private - def validate_context! # no-op end @@ -45,6 +43,8 @@ module Gitlab end end + private + def template_name return unless template_name_valid? diff --git a/lib/gitlab/ci/config/external/mapper/base.rb b/lib/gitlab/ci/config/external/mapper/base.rb index d2f56d0b8f6..d898ddb356c 100644 --- a/lib/gitlab/ci/config/external/mapper/base.rb +++ b/lib/gitlab/ci/config/external/mapper/base.rb @@ -11,9 +11,9 @@ module Gitlab @context = context end - def process(*args) + def process(...) context.logger.instrument(mapper_instrumentation_key) do - process_without_instrumentation(*args) + process_without_instrumentation(...) end end diff --git a/lib/gitlab/ci/config/external/mapper/matcher.rb b/lib/gitlab/ci/config/external/mapper/matcher.rb index 85e19ff1ced..e59eaa6d324 100644 --- a/lib/gitlab/ci/config/external/mapper/matcher.rb +++ b/lib/gitlab/ci/config/external/mapper/matcher.rb @@ -10,6 +10,7 @@ module Gitlab FILE_CLASSES = [ External::File::Local, External::File::Project, + External::File::Component, External::File::Remote, External::File::Template, External::File::Artifact @@ -29,11 +30,11 @@ module Gitlab matching.first elsif matching.empty? raise Mapper::AmbigiousSpecificationError, - "`#{masked_location(location.to_json)}` does not have a valid subkey for include. " \ - "Valid subkeys are: `#{FILE_SUBKEYS.join('`, `')}`" + "`#{masked_location(location.to_json)}` does not have a valid subkey for include. " \ + "Valid subkeys are: `#{FILE_SUBKEYS.join('`, `')}`" else raise Mapper::AmbigiousSpecificationError, - "Each include must use only one of: `#{FILE_SUBKEYS.join('`, `')}`" + "Each include must use only one of: `#{FILE_SUBKEYS.join('`, `')}`" end end end diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb index 6d6f227b940..2982b0efb6c 100644 --- a/lib/gitlab/ci/config/external/mapper/verifier.rb +++ b/lib/gitlab/ci/config/external/mapper/verifier.rb @@ -10,13 +10,29 @@ module Gitlab private def process_without_instrumentation(files) - files.select do |file| + files.each do |file| + verify_execution_time! + + file.validate_location! + file.validate_context! if file.valid? + file.content if file.valid? + end + + # We do not combine the loops because we need to load the content of all files before continuing + # to call `BatchLoader` for all locations. + files.each do |file| # rubocop:disable Style/CombinableLoops + # Checking the max includes will be changed with https://gitlab.com/gitlab-org/gitlab/-/issues/367150 verify_max_includes! verify_execution_time! - file.validate! + file.validate_content! if file.valid? + file.load_and_validate_expanded_hash! if file.valid? - context.expandset.add(file) + if context.expandset.is_a?(Array) # To be removed when FF 'ci_includes_count_duplicates' is removed + context.expandset << file + else + context.expandset.add(file) + end end end diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb index de833619c8d..94ef0afe7f9 100644 --- a/lib/gitlab/ci/config/yaml.rb +++ b/lib/gitlab/ci/config/yaml.rb @@ -5,12 +5,21 @@ module Gitlab class Config module Yaml AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze + MAX_DOCUMENTS = 2 class << self def load!(content) ensure_custom_tags - Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load! + if ::Feature.enabled?(:ci_multi_doc_yaml) + Gitlab::Config::Loader::MultiDocYaml.new( + content, + max_documents: MAX_DOCUMENTS, + additional_permitted_classes: AVAILABLE_TAGS + ).load!.first + else + Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load! + end end private diff --git a/lib/gitlab/ci/config/yaml/tags/reference.rb b/lib/gitlab/ci/config/yaml/tags/reference.rb index 45787077c91..5ecab033109 100644 --- a/lib/gitlab/ci/config/yaml/tags/reference.rb +++ b/lib/gitlab/ci/config/yaml/tags/reference.rb @@ -16,7 +16,7 @@ module Gitlab def valid? data[:seq].is_a?(Array) && !data[:seq].empty? && - data[:seq].all? { |identifier| identifier.is_a?(String) } + data[:seq].all?(String) end private diff --git a/lib/gitlab/ci/interpolation/access.rb b/lib/gitlab/ci/interpolation/access.rb new file mode 100644 index 00000000000..42598458902 --- /dev/null +++ b/lib/gitlab/ci/interpolation/access.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Interpolation + class Access + attr_reader :content, :errors + + MAX_ACCESS_OBJECTS = 5 + MAX_ACCESS_BYTESIZE = 1024 + + def initialize(access, ctx) + @content = access + @ctx = ctx + @errors = [] + + if objects.count <= 1 # rubocop:disable Style/IfUnlessModifier + @errors.push('invalid interpolation access pattern') + end + + if access.bytesize > MAX_ACCESS_BYTESIZE # rubocop:disable Style/IfUnlessModifier + @errors.push('maximum interpolation expression size exceeded') + end + + evaluate! if valid? + end + + def valid? + errors.none? + end + + def objects + @objects ||= @content.split('.', MAX_ACCESS_OBJECTS) + end + + def value + raise ArgumentError, 'access path invalid' unless valid? + + @value + end + + private + + def evaluate! + raise ArgumentError, 'access path invalid' unless valid? + + @value ||= objects.inject(@ctx) do |memo, value| + memo.fetch(value.to_sym) + end + rescue KeyError => e + @errors.push(e) + end + end + end + end +end diff --git a/lib/gitlab/ci/interpolation/block.rb b/lib/gitlab/ci/interpolation/block.rb new file mode 100644 index 00000000000..389cbf378a2 --- /dev/null +++ b/lib/gitlab/ci/interpolation/block.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Interpolation + class Block + PREFIX = '$[[' + PATTERN = /(?<block>\$\[\[\s*(?<access>.*?)\s*\]\])/.freeze + + attr_reader :block, :data, :ctx + + def initialize(block, data, ctx) + @block = block + @ctx = ctx + @data = data + + @access = Interpolation::Access.new(@data, ctx) + end + + def valid? + errors.none? + end + + def errors + @access.errors + end + + def content + @access.content + end + + def value + raise ArgumentError, 'block invalid' unless valid? + + @access.value + end + + def self.match(data) + return data unless data.is_a?(String) && data.include?(PREFIX) + + data.gsub(PATTERN) do + yield ::Regexp.last_match(1), ::Regexp.last_match(2) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/interpolation/config.rb b/lib/gitlab/ci/interpolation/config.rb new file mode 100644 index 00000000000..32f58521139 --- /dev/null +++ b/lib/gitlab/ci/interpolation/config.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Interpolation + ## + # Interpolation::Config represents a configuration artifact that we want to perform interpolation on. + # + class Config + include Gitlab::Utils::StrongMemoize + ## + # Total number of hash nodes traversed. For example, loading a YAML below would result in a hash having 12 nodes + # instead of 9, because hash values are being counted before we recursively traverse them. + # + # test: + # spec: + # env: $[[ inputs.env ]] + # + # $[[ inputs.key ]]: + # name: $[[ inputs.key ]] + # script: my-value + # + # According to our benchmarks performed when developing this code, the worst-case scenario of processing + # a hash with 500_000 nodes takes around 1 second and consumes around 225 megabytes of memory. + # + # The typical scenario, using just a few interpolations takes 250ms and consumes around 20 megabytes of memory. + # + # Given the above the 500_000 nodes should be an upper limit, provided that the are additional safeguard + # present in other parts of the code (example: maximum number of interpolation blocks found). Typical size of a + # YAML configuration with 500k nodes might be around 10 megabytes, which is an order of magnitude higher than + # the 1MB limit for loading YAML on GitLab.com + # + MAX_NODES = 500_000 + MAX_NODE_SIZE = 1024 * 1024 # 1MB + + TooManyNodesError = Class.new(StandardError) + NodeTooLargeError = Class.new(StandardError) + + Visitor = Class.new do + def initialize + @visited = 0 + end + + def visit! + @visited += 1 + + raise Config::TooManyNodesError if @visited > Config::MAX_NODES + end + end + + attr_reader :errors + + def initialize(hash) + @config = hash + @errors = [] + end + + def to_h + @config + end + + ## + # The replace! method will yield a block and replace a each of the hash config nodes with a return value of the + # block. + # + # It returns `nil` if there were errors found during the process. + # + def replace!(&block) + recursive_replace(@config, Visitor.new, &block) + rescue TooManyNodesError + @errors.push('config too large') + nil + rescue NodeTooLargeError + @errors.push('config node too large') + nil + end + strong_memoize_attr :replace! + + def self.fabricate(config) + case config + when Hash + new(config) + when Interpolation::Config + config + else + raise ArgumentError, 'unknown interpolation config' + end + end + + private + + def recursive_replace(config, visitor, &block) + visitor.visit! + + case config + when Hash + {}.tap do |new_hash| + config.each_pair do |key, value| + new_key = recursive_replace(key, visitor, &block) + new_value = recursive_replace(value, visitor, &block) + + if new_key != key + new_hash[new_key] = new_value + else + new_hash[key] = new_value + end + end + end + when Array + config.map { |value| recursive_replace(value, visitor, &block) } + when Symbol + recursive_replace(config.to_s, visitor, &block) + when String + raise NodeTooLargeError if config.bytesize > MAX_NODE_SIZE + + yield config + else + config + end + end + end + end + end +end diff --git a/lib/gitlab/ci/interpolation/context.rb b/lib/gitlab/ci/interpolation/context.rb new file mode 100644 index 00000000000..ce7a86a3c9b --- /dev/null +++ b/lib/gitlab/ci/interpolation/context.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Interpolation + ## + # Interpolation::Context is a class that represents the data that can be used when performing string interpolation + # on a CI configuration. + # + class Context + ContextTooComplexError = Class.new(StandardError) + NotSymbolizedContextError = Class.new(StandardError) + + MAX_DEPTH = 3 + + def initialize(hash) + @context = hash + + raise ContextTooComplexError if depth > MAX_DEPTH + end + + def valid? + errors.none? + end + + ## + # This method is here because `Context` will be responsible for validating specs, inputs and defaults. + # + def errors + [] + end + + def depth + deep_depth(@context) + end + + def fetch(field) + @context.fetch(field) + end + + def to_h + @context.to_h + end + + private + + def deep_depth(context, depth = 0) + values = context.values.map do |value| + if value.is_a?(Hash) + deep_depth(value, depth + 1) + else + depth + 1 + end + end + + values.max + end + + def self.fabricate(context) + case context + when Hash + new(context) + when Interpolation::Context + context + else + raise ArgumentError, 'unknown interpolation context' + end + end + end + end + end +end diff --git a/lib/gitlab/ci/interpolation/template.rb b/lib/gitlab/ci/interpolation/template.rb new file mode 100644 index 00000000000..0211279f266 --- /dev/null +++ b/lib/gitlab/ci/interpolation/template.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Interpolation + class Template + include Gitlab::Utils::StrongMemoize + + attr_reader :blocks, :ctx + + TooManyBlocksError = Class.new(StandardError) + InvalidBlockError = Class.new(StandardError) + + MAX_BLOCKS = 10_000 + + def initialize(config, ctx) + @config = Interpolation::Config.fabricate(config) + @ctx = Interpolation::Context.fabricate(ctx) + @errors = [] + @blocks = {} + + interpolate! if valid? + end + + def valid? + errors.none? + end + + def errors + @errors + @config.errors + @ctx.errors + @blocks.values.flat_map(&:errors) + end + + def size + @blocks.size + end + + def interpolated + @result if valid? + end + + private + + def interpolate! + @result = @config.replace! do |data| + Interpolation::Block.match(data) do |block, data| + evaluate_block(block, data) + end + end + rescue TooManyBlocksError + @errors.push('too many interpolation blocks') + rescue InvalidBlockError + @errors.push('interpolation interrupted by errors') + end + strong_memoize_attr :interpolate! + + def evaluate_block(block, data) + block = (@blocks[block] ||= Interpolation::Block.new(block, data, ctx)) + + raise TooManyBlocksError if @blocks.count > MAX_BLOCKS + raise InvalidBlockError unless block.valid? + + block.value + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/instrumentation.rb b/lib/gitlab/ci/parsers/instrumentation.rb index ab4a923d9aa..5e97b22ecdf 100644 --- a/lib/gitlab/ci/parsers/instrumentation.rb +++ b/lib/gitlab/ci/parsers/instrumentation.rb @@ -6,7 +6,7 @@ module Gitlab module Instrumentation BUCKETS = [0.25, 1, 5, 10].freeze - def parse!(*args) + def parse!(...) parser_result = nil duration = Benchmark.realtime do diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 67817c9f832..1b9afc92d6b 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -282,7 +282,12 @@ module Gitlab return data['name'] if data['name'].present? identifier = identifiers.find(&:cve?) || identifiers.find(&:cwe?) || identifiers.first - "#{identifier.name} in #{location&.fingerprint_path}" + + if location&.fingerprint_path + "#{identifier.name} in #{location.fingerprint_path}" + else + identifier.name.to_s + end end def calculate_uuid_v5(primary_identifier, location_fingerprint) diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index 53c8a7ac122..dcaaefee98f 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -5,59 +5,13 @@ module Gitlab module Pipeline module Chain class CancelPendingPipelines < Chain::Base - include Chain::Helpers - - BATCH_SIZE = 25 - - # rubocop: disable CodeReuse/ActiveRecord def perform! - return if pipeline.parent_pipeline? # skip if child pipeline - return unless project.auto_cancel_pending_pipelines? - - Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines, name: 'cancel_pending_pipelines') do |cancelables| - cancelables.select(:id).each_batch(of: BATCH_SIZE) do |cancelables_batch| - auto_cancel_interruptible_pipelines(cancelables_batch.ids) - end - end + ::Ci::CancelRedundantPipelinesWorker.perform_async(pipeline.id) end - # rubocop: enable CodeReuse/ActiveRecord def break? false end - - private - - def auto_cancelable_pipelines - project.all_pipelines - .created_after(1.week.ago) - .ci_and_parent_sources - .for_ref(pipeline.ref) - .where_not_sha(project.commit(pipeline.ref).try(:id)) - .alive_or_scheduled - .id_not_in(pipeline.id) - end - - def auto_cancel_interruptible_pipelines(pipeline_ids) - ::Ci::Pipeline - .id_in(pipeline_ids) - .with_only_interruptible_builds - .each do |cancelable_pipeline| - Gitlab::AppLogger.info( - class: self.class.name, - message: "Pipeline #{pipeline.id} auto-canceling pipeline #{cancelable_pipeline.id}", - canceled_pipeline_id: cancelable_pipeline.id, - canceled_by_pipeline_id: pipeline.id, - canceled_by_pipeline_source: pipeline.source - ) - - # cascade_to_children not needed because we iterate through descendants here - cancelable_pipeline.cancel_running( - auto_canceled_by_pipeline_id: pipeline.id, - cascade_to_children: false - ) - end - end end end end diff --git a/lib/gitlab/ci/pipeline/chain/create_deployments.rb b/lib/gitlab/ci/pipeline/chain/create_deployments.rb deleted file mode 100644 index 99e438ddbae..00000000000 --- a/lib/gitlab/ci/pipeline/chain/create_deployments.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - class CreateDeployments < Chain::Base - def perform! - create_deployments! if Feature.disabled?(:move_create_deployments_to_worker, pipeline.project) - end - - def break? - false - end - - private - - def create_deployments! - pipeline.stages.map(&:statuses).flatten.map(&method(:create_deployment)) - end - - def create_deployment(build) - ::Deployments::CreateForBuildService.new.execute(build) - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb index b17ae77d445..b886aa22ba3 100644 --- a/lib/gitlab/ci/pipeline/chain/metrics.rb +++ b/lib/gitlab/ci/pipeline/chain/metrics.rb @@ -6,15 +6,27 @@ module Gitlab module Chain class Metrics < Chain::Base def perform! - counter.increment(source: @pipeline.source) + increment_pipeline_created_counter + create_snowplow_event_for_pipeline_name end def break? false end - def counter - ::Gitlab::Ci::Pipeline::Metrics.pipelines_created_counter + def increment_pipeline_created_counter + ::Gitlab::Ci::Pipeline::Metrics.pipelines_created_counter.increment(source: @pipeline.source) + end + + def create_snowplow_event_for_pipeline_name + return unless @pipeline.pipeline_metadata&.name + + Gitlab::Tracking.event( + self.class.name, + 'create_pipeline_with_name', + project: @pipeline.project, + user: @pipeline.user, + namespace: @pipeline.project.namespace) end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 684b58474ad..484e18c6979 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -9,13 +9,12 @@ module Gitlab delegate :dig, to: :@seed_attributes - def initialize(context, attributes, stages_for_needs_lookup, stage) + def initialize(context, attributes, stages_for_needs_lookup) @context = context @pipeline = context.pipeline @seed_attributes = attributes @stages_for_needs_lookup = stages_for_needs_lookup.compact @needs_attributes = dig(:needs_attributes) - @stage = stage @resource_group_key = attributes.delete(:resource_group_key) @job_variables = @seed_attributes.delete(:job_variables) @root_variables_inheritance = @seed_attributes.delete(:root_variables_inheritance) { true } @@ -34,8 +33,6 @@ module Gitlab .new(attributes.delete(:cache), @pipeline) calculate_yaml_variables! - - @processable = initialize_processable end def name @@ -66,20 +63,13 @@ module Gitlab end strong_memoize_attr :errors - # TODO: Method used only in specs. Replace with `to_resource.attributes` when - # the feature flag ci_reuse_build_in_seed_context is removed. - # Then remove this method. def attributes - if reuse_build_in_seed_context? - initial_attributes.deep_merge(evaluated_attributes) - else - @seed_attributes - .deep_merge(pipeline_attributes) - .deep_merge(rules_attributes) - .deep_merge(allow_failure_criteria_attributes) - .deep_merge(@cache.cache_attributes) - .deep_merge(runner_tags) - end + @seed_attributes + .deep_merge(pipeline_attributes) + .deep_merge(rules_attributes) + .deep_merge(allow_failure_criteria_attributes) + .deep_merge(@cache.cache_attributes) + .deep_merge(runner_tags) end def bridge? @@ -91,17 +81,10 @@ module Gitlab def to_resource logger.instrument(:pipeline_seed_build_to_resource) do - if reuse_build_in_seed_context? - # The `options` attribute need to be entirely reassigned because they may - # be overridden by evaluated_attributes. - # We also don't want to reassign all the `initial_attributes` since those - # can affect performance. We only want to assign what's changed. - assignable_attributes = initial_attributes.slice(:options) - .deep_merge(evaluated_attributes) - processable.assign_attributes(assignable_attributes) - processable + if bridge? + ::Ci::Bridge.new(attributes) else - legacy_initialize_processable + ::Ci::Build.new(attributes) end end end @@ -109,18 +92,8 @@ module Gitlab private - attr_reader :processable - delegate :logger, to: :@context - def legacy_initialize_processable - if bridge? - ::Ci::Bridge.new(attributes) - else - ::Ci::Build.new(attributes) - end - end - def initialize_processable return unless reuse_build_in_seed_context? @@ -131,19 +104,6 @@ module Gitlab end end - def initial_attributes - @seed_attributes - .deep_merge(pipeline_attributes) - .deep_merge(ci_stage: @stage) - .deep_merge(@cache.cache_attributes) - end - - def evaluated_attributes - rules_attributes - .deep_merge(allow_failure_criteria_attributes) - .deep_merge(runner_tags) - end - def all_of_only? @only.all? { |spec| spec.satisfied_by?(@pipeline, evaluate_context) } end @@ -223,11 +183,7 @@ module Gitlab strong_memoize_attr :rules_errors def evaluate_context - if reuse_build_in_seed_context? - Gitlab::Ci::Build::Context::Build.new(@pipeline, @seed_attributes, processable) - else - Gitlab::Ci::Build::Context::Build.new(@pipeline, @seed_attributes) - end + Gitlab::Ci::Build::Context::Build.new(@pipeline, @seed_attributes) end strong_memoize_attr :evaluate_context @@ -257,11 +213,6 @@ module Gitlab from: @context.root_variables, to: @job_variables, inheritance: @root_variables_inheritance ) end - - def reuse_build_in_seed_context? - Feature.enabled?(:ci_reuse_build_in_seed_context, @pipeline.project) - end - strong_memoize_attr :reuse_build_in_seed_context? end end end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index c3e94529634..42970d8f5ac 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -10,25 +10,25 @@ module Gitlab delegate :size, to: :seeds delegate :dig, to: :seeds - attr_reader :attributes - - def initialize(context, stage_attributes, previous_stages) - pipeline = context.pipeline - @attributes = { - name: stage_attributes.fetch(:name), - position: stage_attributes.fetch(:index), - pipeline: pipeline, - project: pipeline.project, - partition_id: pipeline.partition_id - } - - @stage = ::Ci::Stage.new(@attributes) - - @builds = stage_attributes.fetch(:builds).map do |build_attributes| - Seed::Build.new(context, build_attributes, previous_stages + [self], @stage) + def initialize(context, attributes, previous_stages) + @context = context + @pipeline = context.pipeline + @attributes = attributes + @previous_stages = previous_stages + + @builds = attributes.fetch(:builds).map do |attributes| + Seed::Build.new(context, attributes, previous_stages + [self]) end end + def attributes + { name: @attributes.fetch(:name), + position: @attributes.fetch(:index), + pipeline: @pipeline, + project: @pipeline.project, + partition_id: @pipeline.partition_id } + end + def seeds @builds.select(&:included?) end @@ -49,8 +49,9 @@ module Gitlab end def to_resource - @stage.statuses = seeds.map(&:to_resource) - @stage + ::Ci::Stage.new(attributes).tap do |stage| + stage.statuses = seeds.map(&:to_resource) + end end strong_memoize_attr :to_resource end diff --git a/lib/gitlab/ci/reports/codequality_reports.rb b/lib/gitlab/ci/reports/codequality_reports.rb index 3196bf3fc6d..aba2d2e8b19 100644 --- a/lib/gitlab/ci/reports/codequality_reports.rb +++ b/lib/gitlab/ci/reports/codequality_reports.rb @@ -42,6 +42,18 @@ module Gitlab rescue StandardError => _ false end + + def code_quality_report_summary + report_degradations = @degradations.presence + return if report_degradations.nil? + + summary = ::Gitlab::Ci::Reports::CodequalityReports::SEVERITY_PRIORITIES.keys.index_with(0) + report_degradations.each_value do |degradation| + summary[degradation[:severity]] += 1 + end + summary['count'] = summary.values.sum + summary + end end end end diff --git a/lib/gitlab/ci/reports/sbom/component.rb b/lib/gitlab/ci/reports/sbom/component.rb index 5188304f4ed..08307580987 100644 --- a/lib/gitlab/ci/reports/sbom/component.rb +++ b/lib/gitlab/ci/reports/sbom/component.rb @@ -31,8 +31,10 @@ module Gitlab end def supported_purl_type? + # the purl type is not required as per the spec: https://cyclonedx.org/docs/1.4/json/#components_items_purl return true unless purl + # however, if the purl type is provided, it _must be valid_ ::Enums::Sbom.purl_types.include?(purl.type.to_sym) end end diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb index bcda2fec5ba..1bf015a0aa0 100644 --- a/lib/gitlab/ci/runner_instructions.rb +++ b/lib/gitlab/ci/runner_instructions.rb @@ -47,6 +47,9 @@ module Gitlab kubernetes: { human_readable_name: "Kubernetes", installation_instructions_url: "https://docs.gitlab.com/runner/install/kubernetes.html" + }, + aws: { + human_readable_name: "AWS" } }.freeze diff --git a/lib/gitlab/ci/runner_upgrade_check.rb b/lib/gitlab/ci/runner_upgrade_check.rb index 03130addd6a..19a8dc446d4 100644 --- a/lib/gitlab/ci/runner_upgrade_check.rb +++ b/lib/gitlab/ci/runner_upgrade_check.rb @@ -42,7 +42,7 @@ module Gitlab # Consider update if there's a newer release within the currently deployed GitLab version add_available_runner_release(runner_version, suggestions) - suggestions[runner_version] = :not_available if suggestions.empty? + suggestions[runner_version] = :unavailable if suggestions.empty? suggestions end diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb index d66d4b20bba..8e9eda560a7 100644 --- a/lib/gitlab/ci/status/bridge/common.rb +++ b/lib/gitlab/ci/status/bridge/common.rb @@ -6,7 +6,7 @@ module Gitlab module Bridge module Common def label - subject.description + subject.description.presence || super end def has_details? diff --git a/lib/gitlab/ci/status/bridge/factory.rb b/lib/gitlab/ci/status/bridge/factory.rb index 4d5a94a3beb..fbe2e2282a0 100644 --- a/lib/gitlab/ci/status/bridge/factory.rb +++ b/lib/gitlab/ci/status/bridge/factory.rb @@ -6,11 +6,13 @@ module Gitlab module Bridge class Factory < Status::Factory def self.extended_statuses - [[Status::Bridge::Failed], + [[Status::Bridge::Retryable], + [Status::Bridge::Failed], [Status::Bridge::Manual], [Status::Bridge::WaitingForResource], [Status::Bridge::Play], - [Status::Bridge::Action]] + [Status::Bridge::Action], + [Status::Bridge::Retried]] end def self.common_helpers diff --git a/lib/gitlab/ci/status/bridge/retried.rb b/lib/gitlab/ci/status/bridge/retried.rb new file mode 100644 index 00000000000..101974f10f2 --- /dev/null +++ b/lib/gitlab/ci/status/bridge/retried.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + class Retried < Status::Build::Retried + end + end + end + end +end diff --git a/lib/gitlab/ci/status/bridge/retryable.rb b/lib/gitlab/ci/status/bridge/retryable.rb new file mode 100644 index 00000000000..203a0b8d5ca --- /dev/null +++ b/lib/gitlab/ci/status/bridge/retryable.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + class Retryable < Status::Build::Retryable + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index c66b8ca5654..55e510166b4 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -22,7 +22,7 @@ module Gitlab end def action_button_title - _('Trigger this manual action') + _('Run job') end def action_path diff --git a/lib/gitlab/ci/status/waiting_for_resource.rb b/lib/gitlab/ci/status/waiting_for_resource.rb index 2026148f752..9ced0aadb88 100644 --- a/lib/gitlab/ci/status/waiting_for_resource.rb +++ b/lib/gitlab/ci/status/waiting_for_resource.rb @@ -17,7 +17,7 @@ module Gitlab end def favicon - 'favicon_pending' + 'favicon_status_pending' end def group diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index fc1f4f0cce8..40f5109851b 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.21.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.28.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index fc1f4f0cce8..40f5109851b 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.21.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.28.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 6884a9556b4..aa2356f6a34 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.45.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.46.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index 222f534387a..eb8e5de5b56 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -15,6 +15,7 @@ variables: DS_EXCLUDED_ANALYZERS: "" DS_EXCLUDED_PATHS: "spec, test, tests, tmp" DS_MAJOR_VERSION: 3 + DS_SCHEMA_MODEL: 15 dependency_scanning: stage: test diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml index 67057e916a8..655ac6ee712 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml @@ -15,6 +15,7 @@ variables: DS_EXCLUDED_ANALYZERS: "" DS_EXCLUDED_PATHS: "spec, test, tests, tmp" DS_MAJOR_VERSION: 3 + DS_SCHEMA_MODEL: 15 dependency_scanning: stage: test diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index dc7e5f445d2..372b782c0a0 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.45.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.46.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 9e15b07f5d1..feba2efcf22 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.45.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.46.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Terraform-Module.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform-Module.gitlab-ci.yml new file mode 100644 index 00000000000..4beaaae9d93 --- /dev/null +++ b/lib/gitlab/ci/templates/Terraform-Module.gitlab-ci.yml @@ -0,0 +1,23 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform-Module.gitlab-ci.yml + +include: + - template: Terraform/Module-Base.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml + - template: Jobs/SAST-IaC.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml + +stages: + - validate + - build + - test + - deploy + +fmt: + extends: .terraform-module:fmt + +# See the included job template at `Terraform/Module-Base.gitlab-ci.yml` to learn about supported variables. +deploy: + extends: .terraform-module:deploy + rules: + - if: $CI_COMMIT_TAG diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index 0b6c10293fc..f3f736e96c4 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -24,6 +24,9 @@ validate: build: extends: .terraform:build + environment: + name: $TF_STATE_NAME + action: prepare deploy: extends: .terraform:deploy @@ -31,3 +34,4 @@ deploy: - build environment: name: $TF_STATE_NAME + action: start diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 9c967d48de1..bc23a7c2a95 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -69,6 +69,7 @@ cache: - gitlab-terraform apply resource_group: ${TF_STATE_NAME} rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $TF_AUTO_DEPLOY == "true" - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH when: manual diff --git a/lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml new file mode 100644 index 00000000000..e73e6194760 --- /dev/null +++ b/lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml @@ -0,0 +1,39 @@ +# Terraform/Module-Base +# +# The purpose of this template is to provide flexibility to the user so +# they are able to only include the jobs that they find interesting. +# +# Therefore, this template is not supposed to run any jobs. The idea is to only +# create hidden jobs. See: https://docs.gitlab.com/ee/ci/yaml/#hide-jobs +# +# There is a more opinionated template which we suggest the users to abide, +# which is the lib/gitlab/ci/templates/Terraform-Module.gitlab-ci.yml + +# These variables may be overridden by the pipeline including it to control how the Terraform module is being deployed. +variables: + TERRAFORM_MODULE_DIR: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project. + TERRAFORM_MODULE_NAME: ${CI_PROJECT_NAME} # The name of your Terraform module, must not have any spaces or underscores (will be translated to hyphens). + TERRAFORM_MODULE_SYSTEM: local # The system or provider your Terraform module targets (ex. local, aws, google). + TERRAFORM_MODULE_VERSION: ${CI_COMMIT_TAG} # The version - it's recommended to follow SemVer for Terraform Module Versioning. + +.terraform-module:fmt: + stage: validate + image: $CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/stable:latest + variables: + TF_ROOT: $TERRAFORM_MODULE_DIR + script: + - gitlab-terraform fmt + allow_failure: true + +.terraform-module:deploy: + stage: deploy + image: $CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/stable:latest + script: + - TERRAFORM_MODULE_NAME=$(echo "${TERRAFORM_MODULE_NAME}" | tr " _" -) # module-name must not have spaces or underscores, so translate them to hyphens + # Builds the Terraform module artifact: a gzipped tar archive with the contents from `$TERRAFORM_MODULE_DIR` without a `.git` directory. + - tar -vczf /tmp/${TERRAFORM_MODULE_NAME}-${TERRAFORM_MODULE_SYSTEM}-${TERRAFORM_MODULE_VERSION}.tgz -C ${TERRAFORM_MODULE_DIR} --exclude=./.git . + # Uploads the Terraform module artifact to the GitLab Terraform Module Registry, see + # docs/user/packages/terraform_module_registry/index.html#publish-a-terraform-module + - 'curl --fail-with-body --location --header "JOB-TOKEN: ${CI_JOB_TOKEN}" + --upload-file /tmp/${TERRAFORM_MODULE_NAME}-${TERRAFORM_MODULE_SYSTEM}-${TERRAFORM_MODULE_VERSION}.tgz + ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/terraform/modules/${TERRAFORM_MODULE_NAME}/${TERRAFORM_MODULE_SYSTEM}/${TERRAFORM_MODULE_VERSION}/file' diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 8e18d57b724..89d681c418d 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -8,6 +8,7 @@ module Gitlab def initialize(pipeline) @pipeline = pipeline + @pipeline_variables_builder = Builder::Pipeline.new(pipeline) @instance_variables_builder = Builder::Instance.new @project_variables_builder = Builder::Project.new(project) @group_variables_builder = Builder::Group.new(project&.group) @@ -18,7 +19,7 @@ module Gitlab Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.concat(predefined_variables(job)) variables.concat(project.predefined_variables) - variables.concat(pipeline.predefined_variables) + variables.concat(pipeline_variables_builder.predefined_variables) variables.concat(job.runner.predefined_variables) if job.runnable? && job.runner variables.concat(kubernetes_variables(environment: environment, job: job)) variables.concat(job.yaml_variables) @@ -38,7 +39,7 @@ module Gitlab break variables unless project variables.concat(project.predefined_variables) - variables.concat(pipeline.predefined_variables) + variables.concat(pipeline_variables_builder.predefined_variables) variables.concat(secret_instance_variables) variables.concat(secret_group_variables(environment: nil)) variables.concat(secret_project_variables(environment: nil)) @@ -117,6 +118,7 @@ module Gitlab private attr_reader :pipeline + attr_reader :pipeline_variables_builder attr_reader :instance_variables_builder attr_reader :project_variables_builder attr_reader :group_variables_builder diff --git a/lib/gitlab/ci/variables/builder/pipeline.rb b/lib/gitlab/ci/variables/builder/pipeline.rb new file mode 100644 index 00000000000..96d6f1673b9 --- /dev/null +++ b/lib/gitlab/ci/variables/builder/pipeline.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + class Builder + class Pipeline + include Gitlab::Utils::StrongMemoize + + def initialize(pipeline) + @pipeline = pipeline + end + + def predefined_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'CI_PIPELINE_IID', value: pipeline.iid.to_s) + variables.append(key: 'CI_PIPELINE_SOURCE', value: pipeline.source.to_s) + variables.append(key: 'CI_PIPELINE_CREATED_AT', value: pipeline.created_at&.iso8601) + + variables.concat(predefined_commit_variables) if pipeline.sha.present? + variables.concat(predefined_commit_tag_variables) if pipeline.tag? + variables.concat(predefined_merge_request_variables) if pipeline.merge_request? + + if pipeline.open_merge_requests_refs.any? + variables.append(key: 'CI_OPEN_MERGE_REQUESTS', value: pipeline.open_merge_requests_refs.join(',')) + end + + variables.append(key: 'CI_GITLAB_FIPS_MODE', value: 'true') if Gitlab::FIPS.enabled? + + variables.append(key: 'CI_KUBERNETES_ACTIVE', value: 'true') if pipeline.has_kubernetes_active? + variables.append(key: 'CI_DEPLOY_FREEZE', value: 'true') if pipeline.freeze_period? + + if pipeline.external_pull_request_event? && pipeline.external_pull_request + variables.concat(pipeline.external_pull_request.predefined_variables) + end + end + end + + private + + attr_reader :pipeline + + def predefined_commit_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + next variables unless pipeline.sha.present? + + variables.append(key: 'CI_COMMIT_SHA', value: pipeline.sha) + variables.append(key: 'CI_COMMIT_SHORT_SHA', value: pipeline.short_sha) + variables.append(key: 'CI_COMMIT_BEFORE_SHA', value: pipeline.before_sha) + variables.append(key: 'CI_COMMIT_REF_NAME', value: pipeline.source_ref) + variables.append(key: 'CI_COMMIT_REF_SLUG', value: pipeline.source_ref_slug) + variables.append(key: 'CI_COMMIT_BRANCH', value: pipeline.ref) if pipeline.branch? + variables.append(key: 'CI_COMMIT_MESSAGE', value: pipeline.git_commit_message.to_s) + variables.append(key: 'CI_COMMIT_TITLE', value: pipeline.git_commit_full_title.to_s) + variables.append(key: 'CI_COMMIT_DESCRIPTION', value: pipeline.git_commit_description.to_s) + variables.append(key: 'CI_COMMIT_REF_PROTECTED', value: (!!pipeline.protected_ref?).to_s) + variables.append(key: 'CI_COMMIT_TIMESTAMP', value: pipeline.git_commit_timestamp.to_s) + variables.append(key: 'CI_COMMIT_AUTHOR', value: pipeline.git_author_full_text.to_s) + + variables.concat(legacy_predefined_commit_variables) + end + end + strong_memoize_attr :predefined_commit_variables + + def legacy_predefined_commit_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'CI_BUILD_REF', value: pipeline.sha) + variables.append(key: 'CI_BUILD_BEFORE_SHA', value: pipeline.before_sha) + variables.append(key: 'CI_BUILD_REF_NAME', value: pipeline.source_ref) + variables.append(key: 'CI_BUILD_REF_SLUG', value: pipeline.source_ref_slug) + end + end + strong_memoize_attr :legacy_predefined_commit_variables + + def predefined_commit_tag_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + git_tag = pipeline.project.repository.find_tag(pipeline.ref) + + next variables unless git_tag + + variables.append(key: 'CI_COMMIT_TAG', value: pipeline.ref) + variables.append(key: 'CI_COMMIT_TAG_MESSAGE', value: git_tag.message) + + variables.concat(legacy_predefined_commit_tag_variables) + end + end + strong_memoize_attr :predefined_commit_tag_variables + + def legacy_predefined_commit_tag_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'CI_BUILD_TAG', value: pipeline.ref) + end + end + strong_memoize_attr :legacy_predefined_commit_tag_variables + + def predefined_merge_request_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'CI_MERGE_REQUEST_EVENT_TYPE', value: pipeline.merge_request_event_type.to_s) + variables.append(key: 'CI_MERGE_REQUEST_SOURCE_BRANCH_SHA', value: pipeline.source_sha.to_s) + variables.append(key: 'CI_MERGE_REQUEST_TARGET_BRANCH_SHA', value: pipeline.target_sha.to_s) + + if merge_request_diff.present? + variables.append(key: 'CI_MERGE_REQUEST_DIFF_ID', value: merge_request_diff.id.to_s) + variables.append(key: 'CI_MERGE_REQUEST_DIFF_BASE_SHA', value: merge_request_diff.base_commit_sha) + end + + variables.concat(pipeline.merge_request.predefined_variables) + end + end + strong_memoize_attr :predefined_merge_request_variables + + def merge_request_diff + pipeline.merge_request_diff + end + strong_memoize_attr :merge_request_diff + end + end + end + end +end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index 9e6a3d86e92..fad2260d818 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -98,7 +98,7 @@ module Gitlab private def validate_array_of_hashes(value) - value.is_a?(Array) && value.all? { |obj| obj.is_a?(Hash) } + value.is_a?(Array) && value.all?(Hash) end end diff --git a/lib/gitlab/config/loader/multi_doc_yaml.rb b/lib/gitlab/config/loader/multi_doc_yaml.rb new file mode 100644 index 00000000000..346adc79896 --- /dev/null +++ b/lib/gitlab/config/loader/multi_doc_yaml.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Loader + class MultiDocYaml + TooManyDocumentsError = Class.new(Loader::FormatError) + DataTooLargeError = Class.new(Loader::FormatError) + NotHashError = Class.new(Loader::FormatError) + + MULTI_DOC_DIVIDER = /^---$/.freeze + + def initialize(config, max_documents:, additional_permitted_classes: []) + @max_documents = max_documents + @safe_config = load_config(config, additional_permitted_classes) + end + + def load! + raise TooManyDocumentsError, 'The parsed YAML has too many documents' if too_many_documents? + raise DataTooLargeError, 'The parsed YAML is too big' if too_big? + raise NotHashError, 'Invalid configuration format' unless all_hashes? + + safe_config.map(&:deep_symbolize_keys) + end + + private + + attr_reader :safe_config, :max_documents + + def load_config(config, additional_permitted_classes) + config.split(MULTI_DOC_DIVIDER).filter_map do |document| + YAML.safe_load(document, + permitted_classes: [Symbol, *additional_permitted_classes], + permitted_symbols: [], + aliases: true + ) + end + rescue Psych::Exception => e + raise Loader::FormatError, e.message + end + + def all_hashes? + safe_config.all?(Hash) + end + + def too_many_documents? + safe_config.count > max_documents + end + + def too_big? + !deep_sizes.all?(&:valid?) + end + + def deep_sizes + safe_config.map do |config| + Gitlab::Utils::DeepSize.new(config, + max_size: Gitlab::CurrentSettings.current_application_settings.max_yaml_size_bytes, + max_depth: Gitlab::CurrentSettings.current_application_settings.max_yaml_depth) + end + end + end + end + end +end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 8b1298d0561..ceca206b084 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -26,7 +26,7 @@ module Gitlab 'manifest_src' => "'self'", 'media_src' => "'self' data: http: https:", 'script_src' => ContentSecurityPolicy::Directives.script_src, - 'style_src' => "'self' 'unsafe-inline'", + 'style_src' => ContentSecurityPolicy::Directives.style_src, 'worker_src' => "#{Gitlab::Utils.append_path(Gitlab.config.gitlab.url, 'assets/')} blob: data:", 'object_src' => "'none'", 'report_uri' => nil @@ -43,6 +43,7 @@ module Gitlab allow_websocket_connections(directives) allow_cdn(directives, Settings.gitlab.cdn_host) if Settings.gitlab.cdn_host.present? + allow_zuora(directives) if Gitlab.com? # Support for Sentry setup via configuration files will be removed in 16.0 # in favor of Gitlab::CurrentSettings. allow_legacy_sentry(directives) if Gitlab.config.sentry&.enabled && Gitlab.config.sentry&.clientside_dsn @@ -128,6 +129,14 @@ module Gitlab append_to_directive(directives, 'frame_src', cdn_host) end + def self.zuora_host + "https://*.zuora.com/apps/PublicHostedPageLite.do" + end + + def self.allow_zuora(directives) + append_to_directive(directives, 'frame_src', zuora_host) + end + def self.append_to_directive(directives, directive, text) directives[directive] = "#{directives[directive]} #{text}".strip end diff --git a/lib/gitlab/content_security_policy/directives.rb b/lib/gitlab/content_security_policy/directives.rb index 4ad420f9e2f..e293e5653c7 100644 --- a/lib/gitlab/content_security_policy/directives.rb +++ b/lib/gitlab/content_security_policy/directives.rb @@ -18,6 +18,10 @@ module Gitlab def self.script_src "'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.google.com/recaptcha/ https://www.recaptcha.net https://apis.google.com" end + + def self.style_src + "'self' 'unsafe-inline'" + end end end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 40e2e637114..756d0afa7e4 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -2,7 +2,7 @@ module Gitlab module Database - DATABASE_NAMES = %w[main ci].freeze + DATABASE_NAMES = %w[main ci main_clusterwide].freeze MAIN_DATABASE_NAME = 'main' CI_DATABASE_NAME = 'ci' @@ -34,10 +34,8 @@ module Gitlab # https://gitlab.com/gitlab-org/gitlab-foss/issues/61974 MAX_TEXT_SIZE_LIMIT = 1_000_000 - # Minimum schema version from which migrations are supported # Migrations before this version may have been removed - MIN_SCHEMA_VERSION = 20190506135400 - MIN_SCHEMA_GITLAB_VERSION = '11.11.0' + MIN_SCHEMA_GITLAB_VERSION = '15.0' # Schema we store dynamically managed partitions in (e.g. for time partitioning) DYNAMIC_PARTITIONS_SCHEMA = :gitlab_partitions_dynamic @@ -51,6 +49,8 @@ module Gitlab PRIMARY_DATABASE_NAME = ActiveRecord::Base.connection_db_config.name.to_sym # rubocop:disable Database/MultipleDatabases + FULLY_QUALIFIED_IDENTIFIER = /^\w+\.\w+$/ + def self.database_base_models @database_base_models ||= { # Note that we use ActiveRecord::Base here and not ApplicationRecord. @@ -59,6 +59,7 @@ module Gitlab # that inherit from ActiveRecord::Base; not just our own models that # inherit from ApplicationRecord. main: ::ActiveRecord::Base, + main_clusterwide: ::MainClusterwide::ApplicationRecord.connection_class? ? ::MainClusterwide::ApplicationRecord : nil, ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil }.compact.with_indifferent_access.freeze end @@ -74,6 +75,7 @@ module Gitlab # that inher from ActiveRecord::Base; not just our own models that # inherit from ApplicationRecord. main: ::ActiveRecord::Base, + main_clusterwide: ::MainClusterwide::ApplicationRecord.connection_class? ? ::MainClusterwide::ApplicationRecord : nil, ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil }.compact.with_indifferent_access.freeze end @@ -91,6 +93,7 @@ module Gitlab # that inher from ActiveRecord::Base; not just our own models that # inherit from ApplicationRecord. main: ::ActiveRecord::Base, + main_clusterwide: ::MainClusterwide::ApplicationRecord.connection_class? ? ::MainClusterwide::ApplicationRecord : nil, ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil }.compact.with_indifferent_access.freeze end @@ -102,7 +105,8 @@ module Gitlab gitlab_ci: [self.database_base_models[:ci] || self.database_base_models.fetch(:main)], # use CI or fallback to main gitlab_shared: database_base_models_with_gitlab_shared.values, # all models gitlab_internal: database_base_models.values, # all models - gitlab_pm: [self.database_base_models.fetch(:main)] # package metadata models + gitlab_pm: [self.database_base_models.fetch(:main)], # package metadata models + gitlab_main_clusterwide: [self.database_base_models[:main_clusterwide] || self.database_base_models.fetch(:main)] }.with_indifferent_access.freeze end @@ -125,7 +129,9 @@ module Gitlab end def self.has_config?(database_name) - Gitlab::Application.config.database_configuration[Rails.env].include?(database_name.to_s) + ActiveRecord::Base.configurations + .configs_for(env_name: Rails.env, name: database_name.to_s, include_replicas: true) + .present? end class PgUser < ApplicationRecord @@ -303,6 +309,14 @@ module Gitlab !read_only? end + # Determines minimum viable migration version, determined by the timestamp + # of the earliest migration file. + def self.read_minimum_migration_version + Dir.open( + Rails.root.join('db/migrate') + ).filter_map { |f| /\A\d{14}/.match(f)&.to_s }.map(&:to_i).min + end + # Monkeypatch rails with upgraded database observability def self.install_transaction_metrics_patches! ActiveRecord::Base.prepend(ActiveRecordBaseTransactionMetrics) diff --git a/lib/gitlab/database/indexing_exclusive_lease_guard.rb b/lib/gitlab/database/async_ddl_exclusive_lease_guard.rb index fb45de347e6..5742e96c9b3 100644 --- a/lib/gitlab/database/indexing_exclusive_lease_guard.rb +++ b/lib/gitlab/database/async_ddl_exclusive_lease_guard.rb @@ -2,16 +2,16 @@ module Gitlab module Database - module IndexingExclusiveLeaseGuard + module AsyncDdlExclusiveLeaseGuard extend ActiveSupport::Concern include ExclusiveLeaseGuard def lease_key - @lease_key ||= "gitlab/database/indexing/actions/#{database_config_name}" + @lease_key ||= "gitlab/database/asyncddl/actions/#{database_config_name}" end def database_config_name - Gitlab::Database.db_config_name(connection) + connection_db_config.name end end end diff --git a/lib/gitlab/database/async_foreign_keys.rb b/lib/gitlab/database/async_foreign_keys.rb new file mode 100644 index 00000000000..115ae9ba2e8 --- /dev/null +++ b/lib/gitlab/database/async_foreign_keys.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncForeignKeys + DEFAULT_ENTRIES_PER_INVOCATION = 2 + + def self.validate_pending_entries!(how_many: DEFAULT_ENTRIES_PER_INVOCATION) + PostgresAsyncForeignKeyValidation.ordered.limit(how_many).each do |record| + ForeignKeyValidator.new(record).perform + end + end + end + end +end diff --git a/lib/gitlab/database/async_foreign_keys/foreign_key_validator.rb b/lib/gitlab/database/async_foreign_keys/foreign_key_validator.rb new file mode 100644 index 00000000000..5958c56a45a --- /dev/null +++ b/lib/gitlab/database/async_foreign_keys/foreign_key_validator.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncForeignKeys + class ForeignKeyValidator + include AsyncDdlExclusiveLeaseGuard + + TIMEOUT_PER_ACTION = 1.day + STATEMENT_TIMEOUT = 12.hours + + def initialize(async_validation) + @async_validation = async_validation + end + + def perform + try_obtain_lease do + if foreign_key_exists? + log_index_info("Starting to validate foreign key") + validate_foreign_with_error_handling + log_index_info("Finished validating foreign key") + else + log_index_info(skip_log_message) + async_validation.destroy! + end + end + end + + private + + attr_reader :async_validation + + delegate :connection, :name, :table_name, :connection_db_config, to: :async_validation + + def foreign_key_exists? + relation = if table_name =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER + Gitlab::Database::PostgresForeignKey.by_constrained_table_identifier(table_name) + else + Gitlab::Database::PostgresForeignKey.by_constrained_table_name(table_name) + end + + relation.by_name(name).exists? + end + + def validate_foreign_with_error_handling + validate_foreign_key + async_validation.destroy! + rescue StandardError => error + async_validation.handle_exception!(error) + + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + Gitlab::AppLogger.error(message: error.message, **logging_options) + end + + def validate_foreign_key + set_statement_timeout do + connection.execute(<<~SQL.squish) + ALTER TABLE #{connection.quote_table_name(table_name)} + VALIDATE CONSTRAINT #{connection.quote_column_name(name)}; + SQL + end + end + + def set_statement_timeout + connection.execute(format("SET statement_timeout TO '%ds'", STATEMENT_TIMEOUT)) + yield + ensure + connection.execute('RESET statement_timeout') + end + + def lease_timeout + TIMEOUT_PER_ACTION + end + + def log_index_info(message) + Gitlab::AppLogger.info(message: message, **logging_options) + end + + def skip_log_message + "Skipping #{name} validation since it does not exist. " \ + "The queuing entry will be deleted" + end + + def logging_options + { + fk_name: name, + table_name: table_name, + class: self.class.name.to_s + } + end + end + end + end +end diff --git a/lib/gitlab/database/async_foreign_keys/migration_helpers.rb b/lib/gitlab/database/async_foreign_keys/migration_helpers.rb new file mode 100644 index 00000000000..b8b9fc6d156 --- /dev/null +++ b/lib/gitlab/database/async_foreign_keys/migration_helpers.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncForeignKeys + module MigrationHelpers + # Prepares a foreign key for asynchronous validation. + # + # Stores the FK information in the postgres_async_foreign_key_validations + # table to be executed later. + # + def prepare_async_foreign_key_validation(table_name, column_name = nil, name: nil) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + + return unless async_fk_validation_available? + + fk_name = name || concurrent_foreign_key_name(table_name, column_name) + + unless foreign_key_exists?(table_name, name: fk_name) + raise missing_schema_object_message(table_name, "foreign key", fk_name) + end + + async_validation = PostgresAsyncForeignKeyValidation + .find_or_create_by!(name: fk_name, table_name: table_name) + + Gitlab::AppLogger.info( + message: 'Prepared FK for async validation', + table_name: async_validation.table_name, + fk_name: async_validation.name) + + async_validation + end + + def unprepare_async_foreign_key_validation(table_name, column_name = nil, name: nil) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + + return unless async_fk_validation_available? + + fk_name = name || concurrent_foreign_key_name(table_name, column_name) + + PostgresAsyncForeignKeyValidation.find_by(name: fk_name).try(&:destroy) + end + + def prepare_partitioned_async_foreign_key_validation(table_name, column_name = nil, name: nil) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + + return unless async_fk_validation_available? + + Gitlab::Database::PostgresPartitionedTable.each_partition(table_name) do |partition| + prepare_async_foreign_key_validation(partition.identifier, column_name, name: name) + end + end + + def unprepare_partitioned_async_foreign_key_validation(table_name, column_name = nil, name: nil) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + + return unless async_fk_validation_available? + + Gitlab::Database::PostgresPartitionedTable.each_partition(table_name) do |partition| + unprepare_async_foreign_key_validation(partition.identifier, column_name, name: name) + end + end + + private + + def async_fk_validation_available? + connection.table_exists?(:postgres_async_foreign_key_validations) + end + end + end + end +end diff --git a/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb b/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb new file mode 100644 index 00000000000..de69a3d496f --- /dev/null +++ b/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncForeignKeys + class PostgresAsyncForeignKeyValidation < SharedModel + include QueueErrorHandlingConcern + + self.table_name = 'postgres_async_foreign_key_validations' + + MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH + MAX_LAST_ERROR_LENGTH = 10_000 + + validates :name, presence: true, uniqueness: true, length: { maximum: MAX_IDENTIFIER_LENGTH } + validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH } + + scope :ordered, -> { order(attempts: :asc, id: :asc) } + end + end + end +end diff --git a/lib/gitlab/database/async_indexes.rb b/lib/gitlab/database/async_indexes.rb index 6f301a66803..581c7e7ff94 100644 --- a/lib/gitlab/database/async_indexes.rb +++ b/lib/gitlab/database/async_indexes.rb @@ -16,6 +16,15 @@ module Gitlab IndexDestructor.new(async_index).perform end end + + def self.execute_pending_actions!(how_many: DEFAULT_INDEXES_PER_INVOCATION) + queue_ids = PostgresAsyncIndex.ordered.limit(how_many).pluck(:id) + removal_actions = PostgresAsyncIndex.where(id: queue_ids).to_drop.ordered + creation_actions = PostgresAsyncIndex.where(id: queue_ids).to_create.ordered + + removal_actions.each { |async_index| IndexDestructor.new(async_index).perform } + creation_actions.each { |async_index| IndexCreator.new(async_index).perform } + end end end end diff --git a/lib/gitlab/database/async_indexes/index_base.rb b/lib/gitlab/database/async_indexes/index_base.rb new file mode 100644 index 00000000000..bde75e12295 --- /dev/null +++ b/lib/gitlab/database/async_indexes/index_base.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncIndexes + class IndexBase + include AsyncDdlExclusiveLeaseGuard + extend ::Gitlab::Utils::Override + + TIMEOUT_PER_ACTION = 1.day + + def initialize(async_index) + @async_index = async_index + end + + def perform + try_obtain_lease do + if preconditions_met? + log_index_info("Starting async index #{action_type}") + execute_action_with_error_handling + log_index_info("Finished async index #{action_type}") + else + log_index_info(skip_log_message) + async_index.destroy! + end + end + end + + private + + attr_reader :async_index + + delegate :connection, :connection_db_config, to: :async_index + + def preconditions_met? + raise NotImplementedError, 'must implement preconditions_met?' + end + + def action_type + raise NotImplementedError, 'must implement action_type' + end + + def execute_action_with_error_handling + around_execution { execute_action } + rescue StandardError => error + async_index.handle_exception!(error) + + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + Gitlab::AppLogger.error(message: error.message, **logging_options) + end + + def around_execution + yield + end + + def execute_action + connection.execute(async_index.definition) + async_index.destroy! + end + + def index_exists? + connection.indexes(async_index.table_name).any? do |index| + index.name == async_index.name + end + end + + def lease_timeout + TIMEOUT_PER_ACTION + end + + def log_index_info(message) + Gitlab::AppLogger.info(message: message, **logging_options) + end + + def skip_log_message + "Skipping index #{action_type} since preconditions are not met. " \ + "The queuing entry will be deleted" + end + + def logging_options + { + table_name: async_index.table_name, + index_name: async_index.name, + class: self.class.name.to_s + } + end + end + end + end +end diff --git a/lib/gitlab/database/async_indexes/index_creator.rb b/lib/gitlab/database/async_indexes/index_creator.rb index 3ae2bb7b3e5..c5f4c5f30ad 100644 --- a/lib/gitlab/database/async_indexes/index_creator.rb +++ b/lib/gitlab/database/async_indexes/index_creator.rb @@ -3,48 +3,24 @@ module Gitlab module Database module AsyncIndexes - class IndexCreator - include IndexingExclusiveLeaseGuard - - TIMEOUT_PER_ACTION = 1.day + class IndexCreator < AsyncIndexes::IndexBase STATEMENT_TIMEOUT = 20.hours - def initialize(async_index) - @async_index = async_index - end - - def perform - try_obtain_lease do - if index_exists? - log_index_info('Skipping index creation as the index exists') - else - log_index_info('Creating async index') - - set_statement_timeout do - connection.execute(async_index.definition) - end - - log_index_info('Finished creating async index') - end - - async_index.destroy - end - end - private - attr_reader :async_index - - def index_exists? - connection.indexes(async_index.table_name).any? { |index| index.name == async_index.name } + override :preconditions_met? + def preconditions_met? + !index_exists? end - def connection - @connection ||= async_index.connection + override :action_type + def action_type + 'creation' end - def lease_timeout - TIMEOUT_PER_ACTION + override :around_execution + def around_execution(&block) + set_statement_timeout(&block) end def set_statement_timeout @@ -53,10 +29,6 @@ module Gitlab ensure connection.execute('RESET statement_timeout') end - - def log_index_info(message) - Gitlab::AppLogger.info(message: message, table_name: async_index.table_name, index_name: async_index.name) - end end end end diff --git a/lib/gitlab/database/async_indexes/index_destructor.rb b/lib/gitlab/database/async_indexes/index_destructor.rb index 66955df9d04..5596e099cb6 100644 --- a/lib/gitlab/database/async_indexes/index_destructor.rb +++ b/lib/gitlab/database/async_indexes/index_destructor.rb @@ -3,58 +3,29 @@ module Gitlab module Database module AsyncIndexes - class IndexDestructor - include IndexingExclusiveLeaseGuard - - TIMEOUT_PER_ACTION = 1.day - - def initialize(async_index) - @async_index = async_index - end - - def perform - try_obtain_lease do - if !index_exists? - log_index_info('Skipping dropping as the index does not exist') - else - log_index_info('Dropping async index') - - retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( - connection: connection, - timing_configuration: Gitlab::Database::Reindexing::REMOVE_INDEX_RETRY_CONFIG, - klass: self.class, - logger: Gitlab::AppLogger - ) - - retries.run(raise_on_exhaustion: false) do - connection.execute(async_index.definition) - end - - log_index_info('Finished dropping async index') - end - - async_index.destroy - end - end - + class IndexDestructor < AsyncIndexes::IndexBase private - attr_reader :async_index - - def index_exists? - connection.indexes(async_index.table_name).any? { |index| index.name == async_index.name } + override :preconditions_met? + def preconditions_met? + index_exists? end - def connection - @connection ||= async_index.connection + override :action_type + def action_type + 'removal' end - def lease_timeout - TIMEOUT_PER_ACTION - end + override :around_execution + def around_execution(&block) + retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( + connection: connection, + timing_configuration: Gitlab::Database::Reindexing::REMOVE_INDEX_RETRY_CONFIG, + klass: self.class, + logger: Gitlab::AppLogger + ) - def log_index_info(message) - Gitlab::AppLogger.info(message: message, table_name: async_index.table_name, index_name: async_index.name) + retries.run(raise_on_exhaustion: false, &block) end end end diff --git a/lib/gitlab/database/async_indexes/migration_helpers.rb b/lib/gitlab/database/async_indexes/migration_helpers.rb index c8f6761534c..f459c43e0ee 100644 --- a/lib/gitlab/database/async_indexes/migration_helpers.rb +++ b/lib/gitlab/database/async_indexes/migration_helpers.rb @@ -22,7 +22,7 @@ module Gitlab return unless async_index_creation_available? PostgresAsyncIndex.find_by(name: index_name).try do |async_index| - async_index.destroy + async_index.destroy! end end diff --git a/lib/gitlab/database/async_indexes/postgres_async_index.rb b/lib/gitlab/database/async_indexes/postgres_async_index.rb index dc932482d40..9f5f39613ed 100644 --- a/lib/gitlab/database/async_indexes/postgres_async_index.rb +++ b/lib/gitlab/database/async_indexes/postgres_async_index.rb @@ -4,6 +4,8 @@ module Gitlab module Database module AsyncIndexes class PostgresAsyncIndex < SharedModel + include QueueErrorHandlingConcern + self.table_name = 'postgres_async_indexes' MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH @@ -15,6 +17,7 @@ module Gitlab scope :to_create, -> { where("definition ILIKE 'CREATE%'") } scope :to_drop, -> { where("definition ILIKE 'DROP%'") } + scope :ordered, -> { order(attempts: :asc, id: :asc) } def to_s definition diff --git a/lib/gitlab/database/bulk_update.rb b/lib/gitlab/database/bulk_update.rb index 4b4a9b38fd8..36dbb157b0d 100644 --- a/lib/gitlab/database/bulk_update.rb +++ b/lib/gitlab/database/bulk_update.rb @@ -43,15 +43,7 @@ module Gitlab end def update! - if without_prepared_statement? - # A workaround for https://github.com/rails/rails/issues/24893 - # When prepared statements are prevented (such as when using the - # query counter or in omnibus by default), we cannot call - # `exec_update`, since that will discard the bindings. - connection.send(:exec_no_cache, sql, log_name, params) # rubocop: disable GitlabSecurity/PublicSend - else - connection.exec_update(sql, log_name, params) - end + connection.exec_update(sql, log_name, params) end def self.column_definitions(model, columns) @@ -93,14 +85,6 @@ module Gitlab end end - # A workaround for https://github.com/rails/rails/issues/24893 - # We need to detect if prepared statements have been disabled. - def without_prepared_statement? - strong_memoize(:without_prepared_statement) do - connection.send(:without_prepared_statement?, [1]) # rubocop: disable GitlabSecurity/PublicSend - end - end - def query_attribute(column, key, values) value = values[column.name] key[column.name] = value if key.try(:id) # optimistic update diff --git a/lib/gitlab/database/connection_timer.rb b/lib/gitlab/database/connection_timer.rb index f9b893ffd0f..4eb214e74f4 100644 --- a/lib/gitlab/database/connection_timer.rb +++ b/lib/gitlab/database/connection_timer.rb @@ -27,7 +27,7 @@ module Gitlab end def current_clock_value - Concurrent.monotonic_time + Process.clock_gettime(Process::CLOCK_MONOTONIC) end end diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb index f0343f9d8b5..622e310ead3 100644 --- a/lib/gitlab/database/load_balancing/connection_proxy.rb +++ b/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -97,11 +97,11 @@ module Gitlab if current_session.use_primary? && !current_session.use_replicas_for_read_queries? @load_balancer.read_write do |connection| - connection.send(...) + connection.public_send(...) end else @load_balancer.read do |connection| - connection.send(...) + connection.public_send(...) end end end @@ -117,7 +117,7 @@ module Gitlab end @load_balancer.read_write do |connection| - connection.send(...) + connection.public_send(...) end end diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index cb3a378ad64..23476e1f5e9 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -105,11 +105,9 @@ module Gitlab def read_write connection = nil transaction_open = nil - attempts = 3 - if prevent_load_balancer_retries_in_transaction? - attempts = 1 if pool.connection.transaction_open? - end + # Retry only once when in a transaction (see https://gitlab.com/gitlab-org/gitlab/-/issues/220242) + attempts = pool.connection.transaction_open? ? 1 : 3 # In the event of a failover the primary may be briefly unavailable. # Instead of immediately grinding to a halt we'll retry the operation @@ -348,10 +346,6 @@ module Gitlab row = ar_connection.select_all(sql).first row['location'] if row end - - def prevent_load_balancer_retries_in_transaction? - Gitlab::Utils.to_boolean(ENV['PREVENT_LOAD_BALANCER_RETRIES_IN_TRANSACTION'], default: false) - end end end end diff --git a/lib/gitlab/database/load_balancing/sticking.rb b/lib/gitlab/database/load_balancing/sticking.rb index 8e5dc98e96e..f5cb83e398a 100644 --- a/lib/gitlab/database/load_balancing/sticking.rb +++ b/lib/gitlab/database/load_balancing/sticking.rb @@ -121,19 +121,19 @@ module Gitlab end def unstick(namespace, id) - Gitlab::Redis::SharedState.with do |redis| + with_redis do |redis| redis.del(redis_key_for(namespace, id)) end end def set_write_location_for(namespace, id, location) - Gitlab::Redis::SharedState.with do |redis| + with_redis do |redis| redis.set(redis_key_for(namespace, id), location, ex: EXPIRATION) end end def last_write_location_for(namespace, id) - Gitlab::Redis::SharedState.with do |redis| + with_redis do |redis| redis.get(redis_key_for(namespace, id)) end end @@ -143,6 +143,12 @@ module Gitlab "database-load-balancing/write-location/#{name}/#{namespace}/#{id}" end + + private + + def with_redis(&block) + Gitlab::Redis::DbLoadBalancing.with(&block) + end end end end diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb index 2e08e1ffb42..83884e89d6e 100644 --- a/lib/gitlab/database/lock_writes_manager.rb +++ b/lib/gitlab/database/lock_writes_manager.rb @@ -10,18 +10,6 @@ module Gitlab # See https://www.postgresql.org/message-id/16934.1568989957%40sss.pgh.pa.us EXPECTED_TRIGGER_RECORD_COUNT = 3 - def self.tables_to_lock(connection) - Gitlab::Database::GitlabSchema.tables_to_schema.each do |table_name, schema_name| - yield table_name, schema_name - end - - Gitlab::Database::SharedModel.using_connection(connection) do - Postgresql::DetachedPartition.find_each do |detached_partition| - yield detached_partition.fully_qualified_table_name, detached_partition.table_schema - end - end - end - def initialize(table_name:, connection:, database_name:, with_retries: true, logger: nil, dry_run: false) @table_name = table_name @connection = connection diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index e41107370ec..9c1cb8e352c 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -14,6 +14,7 @@ module Gitlab include DynamicModelHelpers include RenameTableHelpers include AsyncIndexes::MigrationHelpers + include AsyncForeignKeys::MigrationHelpers def define_batchable_model(table_name, connection: self.connection) super(table_name, connection: connection) diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 25e75a10bb3..60df3370046 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -200,11 +200,14 @@ module Gitlab end end + # rubocop: disable Style/ArgumentsForwarding + # Reason: the default argument will not apply if we just forward via `...` def migrate_in(*args, coordinator: coordinator_for_tracking_database) with_migration_context do coordinator.perform_in(*args) end end + # rubocop: enable Style/ArgumentsForwarding def delete_queued_jobs(class_name) coordinator_for_tracking_database.steal(class_name) do |job| diff --git a/lib/gitlab/database/migrations/base_background_runner.rb b/lib/gitlab/database/migrations/base_background_runner.rb index 8975c04e33a..840add8783d 100644 --- a/lib/gitlab/database/migrations/base_background_runner.rb +++ b/lib/gitlab/database/migrations/base_background_runner.rb @@ -38,13 +38,15 @@ module Gitlab def run_jobs_for_migration(migration_name:, jobs:, run_until:) per_background_migration_result_dir = File.join(@result_dir, migration_name) - instrumentation = Instrumentation.new(result_dir: per_background_migration_result_dir) + instrumentation = Instrumentation.new(result_dir: per_background_migration_result_dir, + observer_classes: observers) + batch_names = (1..).each.lazy.map { |i| "batch_#{i}" } jobs.each do |j| break if run_until <= Time.current - meta = migration_meta(j) + meta = { job_meta: job_meta(j) } instrumentation.observe(version: nil, name: batch_names.next, @@ -55,9 +57,13 @@ module Gitlab end end - def migration_meta(_job) + def job_meta(_job) {} end + + def observers + ::Gitlab::Database::Migrations::Observers.all_observers + end end end end diff --git a/lib/gitlab/database/migrations/observation.rb b/lib/gitlab/database/migrations/observation.rb index 80388c4dbbb..cd048beac96 100644 --- a/lib/gitlab/database/migrations/observation.rb +++ b/lib/gitlab/database/migrations/observation.rb @@ -4,16 +4,12 @@ module Gitlab module Database module Migrations - Observation = Struct.new( - :version, - :name, - :walltime, - :success, - :total_database_size_change, - :meta, - :query_statistics, - keyword_init: true - ) + Observation = Struct.new(:version, :name, :walltime, :success, :total_database_size_change, + :meta, :query_statistics, keyword_init: true) do + def to_json(...) + as_json.except('meta').to_json(...) + end + end end end end diff --git a/lib/gitlab/database/migrations/observers/batch_details.rb b/lib/gitlab/database/migrations/observers/batch_details.rb new file mode 100644 index 00000000000..0f8cdcf3cd6 --- /dev/null +++ b/lib/gitlab/database/migrations/observers/batch_details.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module Observers + class BatchDetails < MigrationObserver + FILE_NAME = 'batch-details.json' + + def before + @started_at = get_time + end + + def after + @finished_at = get_time + end + + def record + File.open(path, 'wb') { |file| file.write(file_contents.to_json) } + end + + private + + attr_reader :started_at, :finished_at + + def file_contents + { time_spent: time_spent }.merge(job_meta) + end + + def get_time + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + + def time_spent + @time_spent ||= (finished_at - started_at).round(2) + end + + def path + File.join(output_dir, FILE_NAME) + end + + def job_meta + meta = observation.meta + + return {} unless meta + + meta[:job_meta].to_h + end + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index c123d01f327..01fdba22c19 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -6,6 +6,8 @@ module Gitlab class TestBatchedBackgroundRunner < BaseBackgroundRunner include Gitlab::Database::DynamicModelHelpers + MIGRATION_DETAILS_FILE_NAME = 'details.json' + def initialize(result_dir:, connection:, from_id:) super(result_dir: result_dir, connection: connection) @connection = connection @@ -51,6 +53,7 @@ module Gitlab migration.column_name, batch_min_value: batch_start, batch_size: migration.batch_size, + job_class: migration.job_class, job_arguments: migration.job_arguments ) @@ -64,7 +67,11 @@ module Gitlab end end - [migration.job_class_name, jobs_to_sample] + job_class_name = migration.job_class_name + + export_migration_details(job_class_name, migration.slice(:interval, :total_tuple_count, :max_batch_size)) + + [job_class_name, jobs_to_sample] end end end @@ -112,11 +119,25 @@ module Gitlab Gitlab::Database::SharedModel.using_connection(connection, &block) end - def migration_meta(job) + def job_meta(job) set_shared_model_connection do - job.batched_migration.slice(:max_batch_size, :total_tuple_count, :interval) + job.slice(:min_value, :max_value, :batch_size, :sub_batch_size, :pause_ms) end end + + def export_migration_details(migration_name, attributes) + directory = result_dir.join(migration_name) + + FileUtils.mkdir_p(directory) unless Dir.exist?(directory) + + File.write(directory.join(MIGRATION_DETAILS_FILE_NAME), attributes.to_json) + end + + def observers + ::Gitlab::Database::Migrations::Observers.all_observers + [ + ::Gitlab::Database::Migrations::Observers::BatchDetails + ] + end end end end diff --git a/lib/gitlab/database/postgres_constraint.rb b/lib/gitlab/database/postgres_constraint.rb index fa590914332..fa3870cb9c7 100644 --- a/lib/gitlab/database/postgres_constraint.rb +++ b/lib/gitlab/database/postgres_constraint.rb @@ -4,7 +4,6 @@ module Gitlab module Database # Backed by the postgres_constraints view class PostgresConstraint < SharedModel - IDENTIFIER_REGEX = /^\w+\.\w+$/.freeze self.primary_key = :oid scope :check_constraints, -> { where(constraint_type: 'c') } @@ -18,7 +17,7 @@ module Gitlab scope :valid, -> { where(constraint_valid: true) } scope :by_table_identifier, ->(identifier) do - unless identifier =~ IDENTIFIER_REGEX + unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}" end diff --git a/lib/gitlab/database/postgres_foreign_key.rb b/lib/gitlab/database/postgres_foreign_key.rb index d3ede45fe86..04ef574a451 100644 --- a/lib/gitlab/database/postgres_foreign_key.rb +++ b/lib/gitlab/database/postgres_foreign_key.rb @@ -5,17 +5,23 @@ module Gitlab class PostgresForeignKey < SharedModel self.primary_key = :oid - # These values come from the possible confdeltype values in pg_constraint - enum on_delete_action: { + # These values come from the possible confdeltype / confupdtype values in pg_constraint + ACTION_TYPES = { restrict: 'r', cascade: 'c', nullify: 'n', set_default: 'd', no_action: 'a' - } + }.freeze + + enum on_delete_action: ACTION_TYPES, _prefix: :on_delete + + enum on_update_action: ACTION_TYPES, _prefix: :on_update scope :by_referenced_table_identifier, ->(identifier) do - raise ArgumentError, "Referenced table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + unless identifier =~ Database::FULLY_QUALIFIED_IDENTIFIER + raise ArgumentError, "Referenced table name is not fully qualified with a schema: #{identifier}" + end where(referenced_table_identifier: identifier) end @@ -23,7 +29,9 @@ module Gitlab scope :by_referenced_table_name, ->(name) { where(referenced_table_name: name) } scope :by_constrained_table_identifier, ->(identifier) do - raise ArgumentError, "Constrained table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + unless identifier =~ Database::FULLY_QUALIFIED_IDENTIFIER + raise ArgumentError, "Constrained table name is not fully qualified with a schema: #{identifier}" + end where(constrained_table_identifier: identifier) end @@ -43,6 +51,12 @@ module Gitlab where(on_delete_action: on_delete) end + + scope :by_on_update_action, ->(on_update) do + raise ArgumentError, "Invalid on_update action #{on_update}" unless on_update_actions.key?(on_update) + + where(on_update_action: on_update) + end end end end diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb index 4a9d8728c83..50009cadf5d 100644 --- a/lib/gitlab/database/postgres_index.rb +++ b/lib/gitlab/database/postgres_index.rb @@ -14,7 +14,9 @@ module Gitlab has_many :queued_reindexing_actions, class_name: 'Gitlab::Database::Reindexing::QueuedAction', foreign_key: :index_identifier scope :by_identifier, ->(identifier) do - raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER + raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" + end find(identifier) end diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb index e4f70ee1745..36dc6818157 100644 --- a/lib/gitlab/database/postgres_partition.rb +++ b/lib/gitlab/database/postgres_partition.rb @@ -8,7 +8,9 @@ module Gitlab belongs_to :postgres_partitioned_table, foreign_key: 'parent_identifier', primary_key: 'identifier' scope :for_identifier, ->(identifier) do - raise ArgumentError, "Partition name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER + raise ArgumentError, "Partition name is not fully qualified with a schema: #{identifier}" + end where(primary_key => identifier) end diff --git a/lib/gitlab/database/postgres_partitioned_table.rb b/lib/gitlab/database/postgres_partitioned_table.rb index 3bd342f940f..fead7379e43 100644 --- a/lib/gitlab/database/postgres_partitioned_table.rb +++ b/lib/gitlab/database/postgres_partitioned_table.rb @@ -10,7 +10,9 @@ module Gitlab has_many :postgres_partitions, foreign_key: 'parent_identifier', primary_key: 'identifier' scope :by_identifier, ->(identifier) do - raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER + raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}" + end find(identifier) end @@ -19,6 +21,13 @@ module Gitlab find_by("identifier = concat(current_schema(), '.', ?)", name) end + def self.each_partition(table_name, &block) + find_by_name_in_current_schema(table_name) + .postgres_partitions + .order(:name) + .each(&block) + end + def dynamic? DYNAMIC_PARTITION_STRATEGIES.include?(strategy) end diff --git a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb index c51282c9a55..4ae3622479f 100644 --- a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb +++ b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb @@ -9,7 +9,18 @@ module Gitlab DMLNotAllowedError = Class.new(UnsupportedSchemaError) DMLAccessDeniedError = Class.new(UnsupportedSchemaError) - IGNORED_SCHEMAS = %i[gitlab_shared gitlab_internal].freeze + # Re-map schemas observed schemas to a single cluster mode + # - symbol: + # The mapped schema indicates that it contains all data in a single-cluster mode + # - nil: + # Inidicates that changes made to this schema are ignored and always allowed + SCHEMA_MAPPING = { + gitlab_shared: nil, + gitlab_internal: nil, + + # Pods specific changes + gitlab_main_clusterwide: :gitlab_main + }.freeze class << self def enabled? @@ -90,7 +101,13 @@ module Gitlab def dml_schemas(tables) extra_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(tables) - extra_schemas.subtract(IGNORED_SCHEMAS) + + SCHEMA_MAPPING.each do |schema, mapped_schema| + next unless extra_schemas.delete?(schema) + + extra_schemas.add(mapped_schema) if mapped_schema + end + extra_schemas end diff --git a/lib/gitlab/database/queue_error_handling_concern.rb b/lib/gitlab/database/queue_error_handling_concern.rb new file mode 100644 index 00000000000..7d35426d029 --- /dev/null +++ b/lib/gitlab/database/queue_error_handling_concern.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueueErrorHandlingConcern + extend ActiveSupport::Concern + + MAX_LAST_ERROR_LENGTH = 10_000 + + included do + validates :last_error, length: { maximum: MAX_LAST_ERROR_LENGTH }, + if: ->(record) { record.respond_to?(:last_error) } + end + + def handle_exception!(error) + transaction do + increment!(:attempts) + update!(last_error: format_last_error(error)) + end + end + + private + + def format_last_error(error) + [error.message] + .concat(error.backtrace) + .join("\n") + .truncate(MAX_LAST_ERROR_LENGTH) + end + end + end +end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index aba45fcc57b..78de7161a0f 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -28,6 +28,7 @@ module Gitlab # Hack: Before we do actual reindexing work, create async indexes Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops) Gitlab::Database::AsyncIndexes.drop_pending_indexes! + Gitlab::Database::AsyncForeignKeys.validate_pending_entries! if Feature.enabled?(:database_async_foreign_key_validation, type: :ops) automatic_reindexing end diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb index eca118a4ff2..57e2e0c1beb 100644 --- a/lib/gitlab/database/reindexing/coordinator.rb +++ b/lib/gitlab/database/reindexing/coordinator.rb @@ -4,7 +4,7 @@ module Gitlab module Database module Reindexing class Coordinator - include IndexingExclusiveLeaseGuard + include AsyncDdlExclusiveLeaseGuard # Maximum lease time for the global Redis lease # This should be higher than the maximum time for any @@ -54,7 +54,7 @@ module Gitlab private - delegate :connection, to: :index + delegate :connection, :connection_db_config, to: :index def with_notifications(action) notifier.notify_start(action) diff --git a/lib/gitlab/database/schema_validation/database.rb b/lib/gitlab/database/schema_validation/database.rb new file mode 100644 index 00000000000..dfc845f0b44 --- /dev/null +++ b/lib/gitlab/database/schema_validation/database.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + class Database + def initialize(connection) + @connection = connection + end + + def fetch_index_by_name(index_name) + index_map[index_name] + end + + def indexes + index_map.values + end + + private + + def index_map + @index_map ||= + fetch_indexes.transform_values! do |index_stmt| + Index.new(PgQuery.parse(index_stmt).tree.stmts.first.stmt.index_stmt) + end + end + + attr_reader :connection + + def fetch_indexes + sql = <<~SQL + SELECT indexname, indexdef + FROM pg_indexes + WHERE indexname NOT LIKE '%_pkey' AND schemaname IN ('public', 'gitlab_partitions_static'); + SQL + + @fetch_indexes ||= connection.exec_query(sql).rows.to_h + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/index.rb b/lib/gitlab/database/schema_validation/index.rb new file mode 100644 index 00000000000..af0d5f31f4e --- /dev/null +++ b/lib/gitlab/database/schema_validation/index.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + class Index + def initialize(parsed_stmt) + @parsed_stmt = parsed_stmt + end + + def name + parsed_stmt.idxname + end + + def statement + @statement ||= PgQuery.deparse_stmt(parsed_stmt) + end + + private + + attr_reader :parsed_stmt + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/indexes.rb b/lib/gitlab/database/schema_validation/indexes.rb new file mode 100644 index 00000000000..b7c3705bde9 --- /dev/null +++ b/lib/gitlab/database/schema_validation/indexes.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + class Indexes + def initialize(structure_sql, database) + @structure_sql = structure_sql + @database = database + end + + def missing_indexes + structure_sql.indexes.map(&:name) - database.indexes.map(&:name) + end + + def extra_indexes + database.indexes.map(&:name) - structure_sql.indexes.map(&:name) + end + + def wrong_indexes + structure_sql.indexes.filter_map do |structure_sql_index| + database_index = database.fetch_index_by_name(structure_sql_index.name) + + next if database_index.nil? + next if database_index.statement == structure_sql_index.statement + + structure_sql_index.name + end + end + + private + + attr_reader :structure_sql, :database + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/structure_sql.rb b/lib/gitlab/database/schema_validation/structure_sql.rb new file mode 100644 index 00000000000..32c69a0e5e7 --- /dev/null +++ b/lib/gitlab/database/schema_validation/structure_sql.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + class StructureSql + def initialize(structure_file_path) + @structure_file_path = structure_file_path + end + + def indexes + @indexes ||= index_statements.map do |index_statement| + index_statement.relation.schemaname = "public" if index_statement.relation.schemaname == '' + + Index.new(index_statement) + end + end + + private + + attr_reader :structure_file_path + + def index_statements + parsed_structure_file.tree.stmts.filter_map { |s| s.stmt.index_stmt } + end + + def parsed_structure_file + PgQuery.parse(File.read(structure_file_path)) + end + end + end + end +end diff --git a/lib/gitlab/database/shared_model.rb b/lib/gitlab/database/shared_model.rb index 877866b9b23..41c3a27bc5b 100644 --- a/lib/gitlab/database/shared_model.rb +++ b/lib/gitlab/database/shared_model.rb @@ -44,6 +44,11 @@ module Gitlab end end + # in case the connection has been switched with using_connection + def connection_pool + connection.pool + end + private def overriding_connection diff --git a/lib/gitlab/database/tables_locker.rb b/lib/gitlab/database/tables_locker.rb new file mode 100644 index 00000000000..c417ce716e8 --- /dev/null +++ b/lib/gitlab/database/tables_locker.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class TablesLocker + GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo].freeze + + def initialize(logger: nil, dry_run: false) + @logger = logger + @dry_run = dry_run + end + + def unlock_writes + Gitlab::Database::EachDatabase.each_database_connection do |connection, database_name| + tables_to_lock(connection) do |table_name, schema_name| + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/366834 + next if schema_name.in? GITLAB_SCHEMAS_TO_IGNORE + + lock_writes_manager(table_name, connection, database_name).unlock_writes + end + end + end + + def lock_writes + Gitlab::Database::EachDatabase.each_database_connection(include_shared: false) do |connection, database_name| + schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) + + tables_to_lock(connection) do |table_name, schema_name| + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/366834 + next if schema_name.in? GITLAB_SCHEMAS_TO_IGNORE + + if schemas_for_connection.include?(schema_name) + lock_writes_manager(table_name, connection, database_name).unlock_writes + else + lock_writes_manager(table_name, connection, database_name).lock_writes + end + end + end + end + + private + + def tables_to_lock(connection, &block) + Gitlab::Database::GitlabSchema.tables_to_schema.each(&block) + + Gitlab::Database::SharedModel.using_connection(connection) do + Postgresql::DetachedPartition.find_each do |detached_partition| + yield detached_partition.fully_qualified_table_name, detached_partition.table_schema + end + end + end + + def lock_writes_manager(table_name, connection, database_name) + Gitlab::Database::LockWritesManager.new( + table_name: table_name, + connection: connection, + database_name: database_name, + with_retries: true, + logger: @logger, + dry_run: @dry_run + ) + end + end + end +end diff --git a/lib/gitlab/database/tables_truncate.rb b/lib/gitlab/database/tables_truncate.rb index daef0402742..a6430d1758b 100644 --- a/lib/gitlab/database/tables_truncate.rb +++ b/lib/gitlab/database/tables_truncate.rb @@ -71,17 +71,25 @@ module Gitlab @connection ||= Gitlab::Database.database_base_models[database_name].connection end + def remove_schema_name(table_with_schema) + ActiveRecord::ConnectionAdapters::PostgreSQL::Utils + .extract_schema_qualified_name(table_with_schema) + .identifier + end + + def disable_locks_on_table(table) + sql_statement = "SELECT set_config('lock_writes.#{table}', 'false', false)" + logger&.info(sql_statement) + connection.execute(sql_statement) unless dry_run + end + def truncate_tables_in_batches(tables_sorted) truncated_tables = [] tables_sorted.flatten.each do |table| - table_name_without_schema = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils - .extract_schema_qualified_name(table) - .identifier + table_name_without_schema = remove_schema_name(table) - sql_statement = "SELECT set_config('lock_writes.#{table_name_without_schema}', 'false', false)" - logger&.info(sql_statement) - connection.execute(sql_statement) unless dry_run + disable_locks_on_table(table_name_without_schema) # Temporarily unlocking writes on the attached partitions of the table. # Because in some cases they might have been locked for writes as well, when they used to be @@ -89,13 +97,7 @@ module Gitlab Gitlab::Database::SharedModel.using_connection(connection) do table_partitions = Gitlab::Database::PostgresPartition.for_parent_table(table_name_without_schema) table_partitions.each do |table_partition| - partition_name_without_schema = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils - .extract_schema_qualified_name(table_partition.identifier) - .identifier - - sql_statement = "SELECT set_config('lock_writes.#{partition_name_without_schema}', 'false', false)" - logger&.info(sql_statement) - connection.execute(sql_statement) unless dry_run + disable_locks_on_table(remove_schema_name(table_partition.identifier)) end end end diff --git a/lib/gitlab/database/transaction_timeout_settings.rb b/lib/gitlab/database/transaction_timeout_settings.rb new file mode 100644 index 00000000000..9485b8d4cbc --- /dev/null +++ b/lib/gitlab/database/transaction_timeout_settings.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class TransactionTimeoutSettings + SETTING = 'idle_in_transaction_session_timeout' + + def initialize(connection) + @connection = connection + end + + def disable_timeouts + @connection.execute("SET #{SETTING} = 0") + end + + def restore_timeouts + @connection.execute("RESET #{SETTING}") + end + end + end +end diff --git a/lib/gitlab/database_importers/work_items/base_type_importer.rb b/lib/gitlab/database_importers/work_items/base_type_importer.rb index 1e29ae7761b..9796a5905e3 100644 --- a/lib/gitlab/database_importers/work_items/base_type_importer.rb +++ b/lib/gitlab/database_importers/work_items/base_type_importer.rb @@ -4,6 +4,85 @@ module Gitlab module DatabaseImporters module WorkItems module BaseTypeImporter + WIDGET_NAMES = { + assignees: 'Assignees', + labels: 'Labels', + description: 'Description', + hierarchy: 'Hierarchy', + start_and_due_date: 'Start and due date', + milestone: 'Milestone', + notes: 'Notes', + iteration: 'Iteration', + weight: 'Weight', + health_status: 'Health status', + progress: 'Progress', + status: 'Status', + requirement_legacy: 'Requirement legacy', + test_reports: 'Test reports' + }.freeze + + WIDGETS_FOR_TYPE = { + issue: [ + :assignees, + :labels, + :description, + :hierarchy, + :start_and_due_date, + :milestone, + :notes, + :iteration, + :weight, + :health_status + ], + incident: [ + :description, + :hierarchy, + :notes + ], + test_case: [ + :description, + :notes + ], + requirement: [ + :description, + :notes, + :status, + :requirement_legacy, + :test_reports + ], + task: [ + :assignees, + :labels, + :description, + :hierarchy, + :start_and_due_date, + :milestone, + :notes, + :iteration, + :weight + ], + objective: [ + :assignees, + :labels, + :description, + :hierarchy, + :milestone, + :notes, + :health_status, + :progress + ], + key_result: [ + :assignees, + :labels, + :description, + :hierarchy, + :start_and_due_date, + :notes, + :health_status, + :progress + ] + }.freeze + def self.upsert_types current_time = Time.current @@ -16,6 +95,29 @@ module Gitlab base_types, unique_by: :idx_work_item_types_on_namespace_id_and_name_null_namespace ) + + upsert_widgets + end + + def self.upsert_widgets + type_ids_by_name = ::WorkItems::Type.default.pluck(:name, :id).to_h # rubocop: disable CodeReuse/ActiveRecord + + widgets = WIDGETS_FOR_TYPE.flat_map do |type_sym, widget_syms| + type_name = ::WorkItems::Type::TYPE_NAMES[type_sym] + + widget_syms.map do |widget_sym| + { + work_item_type_id: type_ids_by_name[type_name], + name: WIDGET_NAMES[widget_sym], + widget_type: ::WorkItems::WidgetDefinition.widget_types[widget_sym] + } + end + end + + ::WorkItems::WidgetDefinition.upsert_all( + widgets, + unique_by: :index_work_item_widget_definitions_on_default_witype_and_name + ) end end end diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index 6d114de8ae8..203cee1fd5e 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -13,8 +13,8 @@ module Gitlab Gitlab::FileDetector.type_of(blob_name) == file_type end - def self.link(*args) - new(*args).link + def self.link(...) + new(...).link end attr_accessor :plain_text, :highlighted_text diff --git a/lib/gitlab/deploy_key_access.rb b/lib/gitlab/deploy_key_access.rb index ca16582d2b4..a582c978be7 100644 --- a/lib/gitlab/deploy_key_access.rb +++ b/lib/gitlab/deploy_key_access.rb @@ -17,11 +17,14 @@ module Gitlab attr_reader :deploy_key def protected_tag_accessible_to?(ref, action:) - assert_project! - - # a deploy key can always push a protected tag - # (which is not always the case when pushing to a protected branch) - true + if Feature.enabled?(:deploy_key_for_protected_tags, project) + super + else + assert_project! + # a deploy key can always push a protected tag + # (which is not always the case when pushing to a protected branch) + true + end end def can_collaborate?(_ref) diff --git a/lib/gitlab/doctor/secrets.rb b/lib/gitlab/doctor/secrets.rb index cd075569d10..03d3f062372 100644 --- a/lib/gitlab/doctor/secrets.rb +++ b/lib/gitlab/doctor/secrets.rb @@ -16,7 +16,7 @@ module Gitlab models_with_attributes = Hash.new { |h, k| h[k] = [] } models_with_encrypted_attributes.each do |model| - models_with_attributes[model] += model.encrypted_attributes.keys + models_with_attributes[model] += model.attr_encrypted_attributes.keys end models_with_encrypted_tokens.each do |model| @@ -79,7 +79,7 @@ module Gitlab end def models_with_encrypted_attributes - all_models.select { |d| d.encrypted_attributes.present? } + all_models.select { |d| d.attr_encrypted_attributes.present? } end def models_with_encrypted_tokens diff --git a/lib/gitlab/email/common.rb b/lib/gitlab/email/common.rb index afee8d9cd3d..01316995c4d 100644 --- a/lib/gitlab/email/common.rb +++ b/lib/gitlab/email/common.rb @@ -54,6 +54,10 @@ module Gitlab # It's looking for each <...> references.scan(/(?!<)[^<>]+(?=>)/) end + + def encrypted_secrets + Settings.encrypted(config.encrypted_secret_file) + end end end end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index e21a88c4e0d..c325112b673 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -62,7 +62,7 @@ module Gitlab def create_issue ::Issues::CreateService.new( - project: project, + container: project, current_user: author, params: { title: mail.subject, diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 06365296a76..076ba42daac 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -92,7 +92,7 @@ module Gitlab def create_issue! result = ::Issues::CreateService.new( - project: project, + container: project, current_user: User.support_bot, params: { title: mail.subject, diff --git a/lib/gitlab/email/html_parser.rb b/lib/gitlab/email/html_parser.rb index 27ba5d2a314..10dbedbb464 100644 --- a/lib/gitlab/email/html_parser.rb +++ b/lib/gitlab/email/html_parser.rb @@ -18,9 +18,6 @@ module Gitlab end def filter_replies! - document.xpath('//blockquote').each(&:remove) - document.xpath('//table').each(&:remove) - # bogus links with no href are sometimes added by outlook, # and can result in Html2Text adding extra square brackets # to the text, so we unwrap them here. @@ -37,7 +34,11 @@ module Gitlab end def filtered_text - @filtered_text ||= Html2Text.convert(filtered_html) + @filtered_text ||= if Feature.enabled?(:service_desk_html_to_text_email_handler) + ::Gitlab::Email::HtmlToMarkdownParser.convert(filtered_html) + else + Html2Text.convert(filtered_html) + end end end end diff --git a/lib/gitlab/email/html_to_markdown_parser.rb b/lib/gitlab/email/html_to_markdown_parser.rb new file mode 100644 index 00000000000..42dd012308b --- /dev/null +++ b/lib/gitlab/email/html_to_markdown_parser.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'nokogiri' + +module Gitlab + module Email + class HtmlToMarkdownParser < Html2Text + ADDITIONAL_TAGS = %w[em strong img details].freeze + IMG_ATTRS = %w[alt src].freeze + + def self.convert(html) + html = fix_newlines(replace_entities(html)) + doc = Nokogiri::HTML(html) + + HtmlToMarkdownParser.new(doc).convert + end + + def iterate_over(node) + return super unless ADDITIONAL_TAGS.include?(node.name) + + if node.name == 'img' + node.keys.each { |key| node.remove_attribute(key) unless IMG_ATTRS.include?(key) } # rubocop:disable Style/HashEachMethods + end + + Kramdown::Document.new(node.to_html, input: 'html').to_commonmark + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/create.rb b/lib/gitlab/email/message/in_product_marketing/create.rb index 6b01c83b8e7..68f9a9a21c9 100644 --- a/lib/gitlab/email/message/in_product_marketing/create.rb +++ b/lib/gitlab/email/message/in_product_marketing/create.rb @@ -68,7 +68,7 @@ module Gitlab private def project_link - link(s_('InProductMarketing|create a project'), help_page_url('gitlab-basics/create-project')) + link(s_('InProductMarketing|create a project'), help_page_url('user/project/index')) end def repo_link @@ -76,7 +76,7 @@ module Gitlab end def github_link - link(s_('InProductMarketing|GitHub Enterprise projects to GitLab'), help_page_url('integration/github')) + link(s_('InProductMarketing|GitHub Enterprise projects to GitLab'), help_page_url('user/project/import/github')) end def bitbucket_link @@ -84,11 +84,11 @@ module Gitlab end def mirroring_link - link(s_('InProductMarketing|repository mirroring'), help_page_url('user/project/repository/repository_mirroring')) + link(s_('InProductMarketing|repository mirroring'), help_page_url('user/project/repository/mirror/index')) end def basics_link - link(s_('InProductMarketing|Git basics'), help_page_url('gitlab-basics/index')) + link(s_('InProductMarketing|Git basics'), help_page_url('topics/git/index')) end def import_link diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index b1fd35184ac..99240f2ad48 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -16,6 +16,7 @@ module Gitlab ENCODING_CONFIDENCE_THRESHOLD = 50 UNICODE_REPLACEMENT_CHARACTER = "�" + BOM_UTF8 = "\xEF\xBB\xBF" def encode!(message) message = force_encode_utf8(message) @@ -147,6 +148,10 @@ module Gitlab filename.force_encoding("UTF-8") end + def strip_bom(message) + message.delete_prefix(BOM_UTF8) + end + private def force_encode_utf8(message) diff --git a/lib/gitlab/encrypted_incoming_email_command.rb b/lib/gitlab/encrypted_incoming_email_command.rb new file mode 100644 index 00000000000..a18382439d6 --- /dev/null +++ b/lib/gitlab/encrypted_incoming_email_command.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# rubocop:disable Rails/Output +module Gitlab + class EncryptedIncomingEmailCommand < EncryptedCommandBase + DISPLAY_NAME = "INCOMING_EMAIL" + EDIT_COMMAND_NAME = "gitlab:incoming_email:secret:edit" + + class << self + def encrypted_secrets + Gitlab::IncomingEmail.encrypted_secrets + end + + def encrypted_file_template + <<~YAML + # password: '123' + # user: 'gitlab-incoming@gmail.com' + YAML + end + end + end +end +# rubocop:enable Rails/Output diff --git a/lib/gitlab/encrypted_service_desk_email_command.rb b/lib/gitlab/encrypted_service_desk_email_command.rb new file mode 100644 index 00000000000..ece6da7c1b3 --- /dev/null +++ b/lib/gitlab/encrypted_service_desk_email_command.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# rubocop:disable Rails/Output +module Gitlab + class EncryptedServiceDeskEmailCommand < EncryptedCommandBase + DISPLAY_NAME = "SERVICE_DESK_EMAIL" + EDIT_COMMAND_NAME = "gitlab:service_desk_email:secret:edit" + + class << self + def encrypted_secrets + Gitlab::ServiceDeskEmail.encrypted_secrets + end + + def encrypted_file_template + <<~YAML + # password: '123' + # user: 'gitlab-incoming@gmail.com' + YAML + end + end + end +end +# rubocop:enable Rails/Output diff --git a/lib/gitlab/environment.rb b/lib/gitlab/environment.rb index b1a9603d3a5..86094727df5 100644 --- a/lib/gitlab/environment.rb +++ b/lib/gitlab/environment.rb @@ -5,5 +5,18 @@ module Gitlab def self.hostname @hostname ||= ENV['HOSTNAME'] || Socket.gethostname end + + # Check whether codebase is going through static verification + # in order to skip executing parts of the codebase + # + # @return [Boolean] Is the code going through static verification? + def self.static_verification? + static_verification = Gitlab::Utils.to_boolean(ENV['STATIC_VERIFICATION'], default: false) + env_production = ENV['RAILS_ENV'] == 'production' + + warn '[WARNING] Static Verification bypass is enabled in Production.' if static_verification && env_production + + static_verification + end end end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 876a1cbb183..2b00fe48951 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -57,6 +57,7 @@ module Gitlab config.before_send = method(:before_send_sentry) config.background_worker_threads = 0 config.send_default_pii = true + config.traces_sample_rate = 0.2 if Gitlab::Utils.to_boolean(ENV['ENABLE_SENTRY_PERFORMANCE_MONITORING']) yield config if block_given? end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index f6431483a15..7aabf699a59 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -65,13 +65,15 @@ module Gitlab status_code = Gitlab::PollingInterval.polling_enabled? ? 304 : 429 - add_instrument_for_cache_hit(status_code, route, request) - Gitlab::ApplicationContext.push( feature_category: route.feature_category, - caller_id: route.caller_id + caller_id: route.caller_id, + remote_ip: request.remote_ip ) + request.env[Gitlab::Metrics::RequestsRackMiddleware::REQUEST_URGENCY_KEY] = route.urgency + add_instrument_for_cache_hit(status_code, route, request) + new_headers = { 'ETag' => etag, 'X-Gitlab-From-Cache' => 'true' @@ -102,7 +104,10 @@ module Gitlab format: request.format.ref, method: request.request_method, path: request.filtered_path, - status: status + status: status, + metadata: Gitlab::ApplicationContext.current, + request_urgency: route.urgency.name, + target_duration_s: route.urgency.duration } ActiveSupport::Notifications.instrument( diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 684afc6762a..754b54fda81 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -3,24 +3,34 @@ module Gitlab module EtagCaching module Router - Route = Struct.new(:router, :regexp, :name, :feature_category, :caller_id) do + Route = Struct.new(:router, :regexp, :name, :feature_category, :caller_id, :urgency, keyword_init: true) do delegate :match, to: :regexp delegate :cache_key, to: :router end module Helpers - def build_route(attrs) - EtagCaching::Router::Route.new(self, *attrs) + def build_graphql_route(regexp, name, feature_category) + EtagCaching::Router::Route.new( + router: self, + regexp: regexp, + name: name, + feature_category: feature_category, + # This information can be loaded from the graphql query, but is not + # included yet + # https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/665 + caller_id: nil, + urgency: Gitlab::EndpointAttributes::DEFAULT_URGENCY + ) end - def build_rails_route(attrs) - regexp, name, controller, action_name = *attrs + def build_rails_route(regexp, name, controller, action_name) EtagCaching::Router::Route.new( - self, - regexp, - name, - controller.feature_category_for_action(action_name).to_s, - controller.endpoint_id_for_action(action_name).to_s + router: self, + regexp: regexp, + name: name, + feature_category: controller.feature_category_for_action(action_name).to_s, + caller_id: controller.endpoint_id_for_action(action_name).to_s, + urgency: controller.urgency_for_action(action_name) ) end end diff --git a/lib/gitlab/etag_caching/router/graphql.rb b/lib/gitlab/etag_caching/router/graphql.rb index 1f56670ee7f..7a0fb2ac269 100644 --- a/lib/gitlab/etag_caching/router/graphql.rb +++ b/lib/gitlab/etag_caching/router/graphql.rb @@ -23,7 +23,7 @@ module Gitlab 'on_demand_scans', 'dynamic_application_security_testing' ] - ].map(&method(:build_route)).freeze + ].map { |attrs| build_graphql_route(*attrs) }.freeze def self.match(request) return unless request.path_info == graphql_api_path diff --git a/lib/gitlab/etag_caching/router/rails.rb b/lib/gitlab/etag_caching/router/rails.rb index d80c003fe53..2924370f494 100644 --- a/lib/gitlab/etag_caching/router/rails.rb +++ b/lib/gitlab/etag_caching/router/rails.rb @@ -104,7 +104,7 @@ module Gitlab ::Projects::MergeRequests::ContentController, :cached_widget ] - ].map(&method(:build_rails_route)).freeze + ].map { |attrs| build_rails_route(*attrs) }.freeze # Overridden in EE to add more routes def self.all_routes diff --git a/lib/gitlab/external_authorization/config.rb b/lib/gitlab/external_authorization/config.rb index 8654a8c1e2e..e77318144ef 100644 --- a/lib/gitlab/external_authorization/config.rb +++ b/lib/gitlab/external_authorization/config.rb @@ -37,6 +37,12 @@ module Gitlab client_cert.present? && client_key.present? end + def allow_deploy_tokens_and_deploy_keys? + return true unless enabled? + + service_url.blank? && application_settings.allow_deploy_tokens_and_keys_with_external_authn? + end + private def application_settings diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index 6225955a930..95f896a74e9 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -44,7 +44,15 @@ module Gitlab # Overridden in Gitlab::WikiFileFinder def search_paths(query) - repository.search_files_by_name(query, ref) + if Feature.enabled?(:code_basic_search_files_by_regexp, project) + return [] if query.blank? || ref.blank? + + escaped_query = RE2::Regexp.escape(query) + query_regexp = Gitlab::EncodingHelper.encode_utf8_no_detect("(?i)#{escaped_query}") + repository.search_files_by_regexp(query_regexp, ref) + else + repository.search_files_by_name(query, ref) + end end end end diff --git a/lib/gitlab/git/raw_diff_change.rb b/lib/gitlab/git/raw_diff_change.rb index 9a41f04a4db..c0121311605 100644 --- a/lib/gitlab/git/raw_diff_change.rb +++ b/lib/gitlab/git/raw_diff_change.rb @@ -38,7 +38,7 @@ module Gitlab def extract_paths(file_path) case operation when :copied, :renamed - file_path.split(/\t/) + file_path.split("\t") when :deleted [file_path, nil] when :added diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 2f9cfe3e764..e054b6df98f 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -465,9 +465,9 @@ module Gitlab end # Returns the SHA of the most recent common ancestor of +from+ and +to+ - def merge_base(*commits) + def merge_base(...) wrapped_gitaly_errors do - gitaly_repository_client.find_merge_base(*commits) + gitaly_repository_client.find_merge_base(...) end end @@ -652,15 +652,23 @@ module Gitlab end end - def merge(user, source_sha, target_branch, message, &block) + def merge(user, source_sha:, target_branch:, message:, target_sha: nil, &block) wrapped_gitaly_errors do - gitaly_operation_client.user_merge_branch(user, source_sha, target_branch, message, &block) + gitaly_operation_client.user_merge_branch(user, + source_sha: source_sha, + target_branch: target_branch, + message: message, + target_sha: target_sha, + &block) end end - def ff_merge(user, source_sha, target_branch) + def ff_merge(user, source_sha:, target_branch:, target_sha: nil) wrapped_gitaly_errors do - gitaly_operation_client.user_ff_branch(user, source_sha, target_branch) + gitaly_operation_client.user_ff_branch(user, + source_sha: source_sha, + target_branch: target_branch, + target_sha: target_sha) end end @@ -720,9 +728,9 @@ module Gitlab raise DeleteBranchError, e end - def delete_refs(*ref_names) + def delete_refs(...) wrapped_gitaly_errors do - gitaly_delete_refs(*ref_names) + gitaly_delete_refs(...) end end @@ -847,9 +855,10 @@ module Gitlab end end - def list_refs(patterns = [Gitlab::Git::BRANCH_REF_PREFIX]) + # peel_tags slows down the request by a factor of 3-4 + def list_refs(patterns = [Gitlab::Git::BRANCH_REF_PREFIX], pointing_at_oids: [], peel_tags: false) wrapped_gitaly_errors do - gitaly_ref_client.list_refs(patterns) + gitaly_ref_client.list_refs(patterns, pointing_at_oids: pointing_at_oids, peel_tags: peel_tags) end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 35b330fa089..9d19695363a 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -367,7 +367,7 @@ module Gitlab end def deploy_key? - actor.is_a?(DeployKey) && !Gitlab::ExternalAuthorization.enabled? + actor.is_a?(DeployKey) && Gitlab::ExternalAuthorization.allow_deploy_tokens_and_deploy_keys? end def deploy_token @@ -375,7 +375,7 @@ module Gitlab end def deploy_token? - actor.is_a?(DeployToken) && !Gitlab::ExternalAuthorization.enabled? + actor.is_a?(DeployToken) && Gitlab::ExternalAuthorization.allow_deploy_tokens_and_deploy_keys? end def ci? diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 199257f767d..77d2ba315a8 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -128,8 +128,8 @@ module Gitlab raise "storage #{storage.inspect} is missing a gitaly_address" end - unless %w(tcp unix tls).include?(URI(address).scheme) - raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'" + unless %w(tcp unix tls dns).include?(URI(address).scheme) + raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls' or 'dns'" end address diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index e5f8a255f7d..4df9d800ea6 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -6,6 +6,12 @@ module Gitlab include Gitlab::EncodingHelper include WithFeatureFlagActors + WHITESPACE_CHANGES = { + 'ignore_all_spaces' => Gitaly::CommitDiffRequest::WhitespaceChanges::WHITESPACE_CHANGES_IGNORE_ALL, + 'ignore_spaces' => Gitaly::CommitDiffRequest::WhitespaceChanges::WHITESPACE_CHANGES_IGNORE, + 'unspecified' => Gitaly::CommitDiffRequest::WhitespaceChanges::WHITESPACE_CHANGES_UNSPECIFIED + }.freeze + TREE_ENTRIES_DEFAULT_LIMIT = 100_000 def initialize(repository) @@ -538,6 +544,11 @@ module Gitlab def call_commit_diff(request_params, options = {}) request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) + + if Feature.enabled?(:add_ignore_all_white_spaces) && (request_params[:ignore_whitespace_change]) + request_params[:whitespace_changes] = WHITESPACE_CHANGES['ignore_all_spaces'] + end + request_params[:enforce_limits] = options.fetch(:limits, true) request_params[:collapse_diffs] = !options.fetch(:expanded, true) request_params.merge!(Gitlab::Git::DiffCollection.limits(options)) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 66f70ed9dc6..313334737c0 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -40,11 +40,6 @@ module Gitlab ) response = gitaly_client_call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.long_timeout) - if pre_receive_error = response.pre_receive_error.presence - raise Gitlab::Git::PreReceiveError, pre_receive_error - elsif response.exists - raise Gitlab::Git::Repository::TagExistsError - end Gitlab::Git::Tag.new(@repository, response.tag) rescue GRPC::BadStatus => e @@ -79,10 +74,6 @@ module Gitlab response = gitaly_client_call(@repository.storage, :operation_service, :user_create_branch, request, timeout: GitalyClient.long_timeout) - if response.pre_receive_error.present? - raise Gitlab::Git::PreReceiveError, response.pre_receive_error - end - branch = response.branch return unless branch @@ -128,12 +119,8 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) - response = gitaly_client_call(@repository.storage, :operation_service, - :user_delete_branch, request, timeout: GitalyClient.long_timeout) - - if pre_receive_error = response.pre_receive_error.presence - raise Gitlab::Git::PreReceiveError, pre_receive_error - end + gitaly_client_call(@repository.storage, :operation_service, + :user_delete_branch, request, timeout: GitalyClient.long_timeout) rescue GRPC::BadStatus => e detailed_error = GitalyClient.decode_detailed_error(e) @@ -165,7 +152,7 @@ module Gitlab response.commit_id end - def user_merge_branch(user, source_sha, target_branch, message) + def user_merge_branch(user, source_sha:, target_branch:, message:, target_sha: nil) request_enum = QueueEnumerator.new response_enum = gitaly_client_call( @repository.storage, @@ -181,6 +168,7 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit_id: source_sha, branch: encode_binary(target_branch), + expected_old_oid: target_sha, message: encode_binary(message), timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) @@ -197,7 +185,6 @@ module Gitlab raise Gitlab::Git::CommitError, 'failed to apply merge to branch' unless branch_update.commit_id.present? Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) - rescue GRPC::BadStatus => e detailed_error = GitalyClient.decode_detailed_error(e) @@ -220,12 +207,13 @@ module Gitlab request_enum.close end - def user_ff_branch(user, source_sha, target_branch) + def user_ff_branch(user, source_sha:, target_branch:, target_sha: nil) request = Gitaly::UserFFBranchRequest.new( repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit_id: source_sha, - branch: encode_binary(target_branch) + branch: encode_binary(target_branch), + expected_old_oid: target_sha ) response = gitaly_client_call( @@ -246,25 +234,54 @@ module Gitlab end def user_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:, dry_run: false) - call_cherry_pick_or_revert(:cherry_pick, - user: user, - commit: commit, - branch_name: branch_name, - message: message, - start_branch_name: start_branch_name, - start_repository: start_repository, - dry_run: dry_run) + response = call_cherry_pick_or_revert(:cherry_pick, + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository, + dry_run: dry_run) + + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) + rescue GRPC::BadStatus => e + detailed_error = GitalyClient.decode_detailed_error(e) + + case detailed_error&.error + when :access_check + access_check_error = detailed_error.access_check + # These messages were returned from internal/allowed API calls + raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) + when :cherry_pick_conflict + raise Gitlab::Git::Repository::CreateTreeError, 'CONFLICT' + when :changes_already_applied + raise Gitlab::Git::Repository::CreateTreeError, 'EMPTY' + when :target_branch_diverged + raise Gitlab::Git::CommitError, 'branch diverged' + else + raise e + end end def user_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:, dry_run: false) - call_cherry_pick_or_revert(:revert, - user: user, - commit: commit, - branch_name: branch_name, - message: message, - start_branch_name: start_branch_name, - start_repository: start_repository, - dry_run: dry_run) + response = call_cherry_pick_or_revert(:revert, + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository, + dry_run: dry_run) + + if response.pre_receive_error.presence + raise Gitlab::Git::PreReceiveError, response.pre_receive_error + elsif response.commit_error.presence + raise Gitlab::Git::CommitError, response.commit_error + elsif response.create_tree_error.presence + raise Gitlab::Git::Repository::CreateTreeError, response.create_tree_error_code + end + + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:, push_options: []) @@ -520,7 +537,7 @@ module Gitlab dry_run: dry_run ) - response = gitaly_client_call( + gitaly_client_call( @repository.storage, :operation_service, :"user_#{rpc}", @@ -528,37 +545,6 @@ module Gitlab remote_storage: start_repository.storage, timeout: GitalyClient.long_timeout ) - - handle_cherry_pick_or_revert_response(response) - rescue GRPC::BadStatus => e - detailed_error = GitalyClient.decode_detailed_error(e) - - case detailed_error&.error - when :access_check - access_check_error = detailed_error.access_check - # These messages were returned from internal/allowed API calls - raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) - when :cherry_pick_conflict - raise Gitlab::Git::Repository::CreateTreeError, 'CONFLICT' - when :changes_already_applied - raise Gitlab::Git::Repository::CreateTreeError, 'EMPTY' - when :target_branch_diverged - raise Gitlab::Git::CommitError, 'branch diverged' - else - raise e - end - end - - def handle_cherry_pick_or_revert_response(response) - if response.pre_receive_error.presence - raise Gitlab::Git::PreReceiveError, response.pre_receive_error - elsif response.commit_error.presence - raise Gitlab::Git::CommitError, response.commit_error - elsif response.create_tree_error.presence - raise Gitlab::Git::Repository::CreateTreeError, response.create_tree_error_code - end - - Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end # rubocop:disable Metrics/ParameterLists diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 74034c4e717..ac6491e8770 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -205,10 +205,13 @@ module Gitlab raise ArgumentError, ex end - def list_refs(patterns = [Gitlab::Git::BRANCH_REF_PREFIX]) + # peel_tags slows down the request by a factor of 3-4 + def list_refs(patterns = [Gitlab::Git::BRANCH_REF_PREFIX], pointing_at_oids: [], peel_tags: false) request = Gitaly::ListRefsRequest.new( repository: @gitaly_repo, - patterns: patterns + patterns: patterns, + pointing_at_oids: pointing_at_oids, + peel_tags: peel_tags ) response = gitaly_client_call(@storage, :ref_service, :list_refs, request, timeout: GitalyClient.fast_timeout) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 203854264ce..bcc03ca08c9 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -24,8 +24,20 @@ module Gitlab response.exists end - def optimize_repository - request = Gitaly::OptimizeRepositoryRequest.new(repository: @gitaly_repo) + # Optimize the repository. By default, this will perform heuristical housekeeping in the repository, which + # is the recommended approach and will only optimize what needs to be optimized. If `eager = true`, then + # Gitaly will instead be asked to perform eager housekeeping. As a consequence the housekeeping run will take a + # _lot_ longer. It is not recommended to use eager housekeeping in general, but only in situations where it is + # explicitly required. + def optimize_repository(eager: false) + strategy = if eager + Gitaly::OptimizeRepositoryRequest::Strategy::STRATEGY_EAGER + else + Gitaly::OptimizeRepositoryRequest::Strategy::STRATEGY_HEURISTICAL + end + + request = Gitaly::OptimizeRepositoryRequest.new(repository: @gitaly_repo, + strategy: strategy) gitaly_client_call(@storage, :repository_service, :optimize_repository, request, timeout: GitalyClient.long_timeout) end diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb index de66f310edf..b1e259fe940 100644 --- a/lib/gitlab/github_import/importer/pull_request_review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_review_importer.rb @@ -113,6 +113,9 @@ module Gitlab state: ::MergeRequestReviewer.states['reviewed'], created_at: submitted_at ) + rescue ActiveRecord::RecordNotUnique + # multiple reviews from single person could make a SQL concurrency issue here + nil end # rubocop:disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb index 4090555c85e..e0a7e6479f5 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb @@ -16,6 +16,10 @@ module Gitlab @parallel = parallel @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY % { project: project.id, collection: collection_method } + @job_waiter_cache_key = JOB_WAITER_CACHE_KEY % + { project: project.id, collection: collection_method } + @job_waiter_remaining_cache_key = JOB_WAITER_REMAINING_CACHE_KEY % + { project: project.id, collection: collection_method } end # In single endpoint there is no issue info to which associated related diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb index 2424b3e8c1f..8e9d6d8dd50 100644 --- a/lib/gitlab/github_import/markdown_text.rb +++ b/lib/gitlab/github_import/markdown_text.rb @@ -16,8 +16,8 @@ module Gitlab PULL_REF_MATCHER = '%{github_url}/%{import_source}/pull' class << self - def format(*args) - new(*args).to_s + def format(...) + new(...).to_s end # Links like `https://domain.github.com/<namespace>/<project>/pull/<iid>` needs to be converted diff --git a/lib/gitlab/github_import/object_counter.rb b/lib/gitlab/github_import/object_counter.rb index 8873db24118..7ee64b2abac 100644 --- a/lib/gitlab/github_import/object_counter.rb +++ b/lib/gitlab/github_import/object_counter.rb @@ -7,6 +7,7 @@ module Gitlab OPERATIONS = %w[fetched imported].freeze PROJECT_COUNTER_LIST_KEY = 'github-importer/object-counters-list/%{project}/%{operation}' PROJECT_COUNTER_KEY = 'github-importer/object-counter/%{project}/%{operation}/%{object_type}' + EMPTY_SUMMARY = OPERATIONS.index_with { |operation| {} } GLOBAL_COUNTER_KEY = 'github_importer_%{operation}_%{object_type}' GLOBAL_COUNTER_DESCRIPTION = 'The number of %{operation} Github %{object_type}' @@ -29,6 +30,18 @@ module Gitlab end def summary(project) + cached_summary = cashed_summary(project) + # Actual information about objects that have already been imported is stored + # in the Redis Cache until Redis key is expired. + # After import is completed we store this information in project's import_checksums + return cached_summary if cached_summary != EMPTY_SUMMARY || project.import_state.blank? + + project.import_state.in_progress? ? cached_summary : project.import_checksums + end + + private + + def cashed_summary(project) OPERATIONS.each_with_object({}) do |operation, result| result[operation] = {} @@ -42,8 +55,6 @@ module Gitlab end end - private - # Global counters are long lived, in Prometheus, # and it's used to report the health of the Github Importer # in the Grafana Dashboard diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 03aa02fb659..4b54a77983d 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -3,11 +3,18 @@ module Gitlab module GithubImport module ParallelScheduling - attr_reader :project, :client, :page_counter, :already_imported_cache_key + attr_reader :project, :client, :page_counter, :already_imported_cache_key, + :job_waiter_cache_key, :job_waiter_remaining_cache_key # The base cache key to use for tracking already imported objects. ALREADY_IMPORTED_CACHE_KEY = 'github-importer/already-imported/%{project}/%{collection}' + # The base cache key to use for storing job waiter key + JOB_WAITER_CACHE_KEY = + 'github-importer/job-waiter/%{project}/%{collection}' + # The base cache key to use for storing job waiter remaining jobs + JOB_WAITER_REMAINING_CACHE_KEY = + 'github-importer/job-waiter-remaining/%{project}/%{collection}' # project - An instance of `Project`. # client - An instance of `Gitlab::GithubImport::Client`. @@ -19,6 +26,10 @@ module Gitlab @page_counter = PageCounter.new(project, collection_method) @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY % { project: project.id, collection: collection_method } + @job_waiter_cache_key = JOB_WAITER_CACHE_KEY % + { project: project.id, collection: collection_method } + @job_waiter_remaining_cache_key = JOB_WAITER_REMAINING_CACHE_KEY % + { project: project.id, collection: collection_method } end def parallel? @@ -74,7 +85,27 @@ module Gitlab def parallel_import raise 'Batch settings must be defined for parallel import' if parallel_import_batch.blank? - spread_parallel_import + if Feature.enabled?(:improved_spread_parallel_import) + improved_spread_parallel_import + else + spread_parallel_import + end + end + + def improved_spread_parallel_import + enqueued_job_counter = 0 + + each_object_to_import do |object| + repr = object_representation(object) + + job_delay = calculate_job_delay(enqueued_job_counter) + sidekiq_worker_class.perform_in(job_delay, project.id, repr.to_hash, job_waiter.key) + enqueued_job_counter += 1 + + job_waiter.jobs_remaining = Gitlab::Cache::Import::Caching.increment(job_waiter_remaining_cache_key) + end + + job_waiter end def spread_parallel_import @@ -233,6 +264,22 @@ module Gitlab parallel: parallel? ) end + + def job_waiter + @job_waiter ||= begin + key = Gitlab::Cache::Import::Caching.read(job_waiter_cache_key) + key ||= Gitlab::Cache::Import::Caching.write(job_waiter_cache_key, JobWaiter.generate_key) + jobs_remaining = Gitlab::Cache::Import::Caching.read(job_waiter_remaining_cache_key).to_i || 0 + + JobWaiter.new(jobs_remaining, key) + end + end + + def calculate_job_delay(job_index) + multiplier = (job_index / parallel_import_batch[:size]) + + (multiplier * parallel_import_batch[:delay]) + 1.second + end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index ceef072a710..c9766ee095a 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -40,7 +40,6 @@ module Gitlab gon.sprite_icons = IconsHelper.sprite_icon_path gon.sprite_file_icons = IconsHelper.sprite_file_icons_path gon.emoji_sprites_css_path = ActionController::Base.helpers.stylesheet_path('emoji_sprites') - gon.select2_css_path = ActionController::Base.helpers.stylesheet_path('lazy_bundles/select2.css') gon.gridstack_css_path = ActionController::Base.helpers.stylesheet_path('lazy_bundles/gridstack.css') gon.test_env = Rails.env.test? gon.disable_animations = Gitlab.config.gitlab['disable_animations'] @@ -68,7 +67,7 @@ module Gitlab push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:vscode_web_ide, current_user) push_frontend_feature_flag(:integration_slack_app_notifications) - push_frontend_feature_flag(:new_fonts, current_user) + push_frontend_feature_flag(:full_path_project_search, current_user) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb index 884fc85c4ec..983bdb9c0a2 100644 --- a/lib/gitlab/graphql/authorize/authorize_resource.rb +++ b/lib/gitlab/graphql/authorize/authorize_resource.rb @@ -67,8 +67,8 @@ module Gitlab self.class.authorization.ok?(object, current_user) end - def raise_resource_not_available_error!(*args) - self.class.raise_resource_not_available_error!(*args) + def raise_resource_not_available_error!(...) + self.class.raise_resource_not_available_error!(...) end end end diff --git a/lib/gitlab/graphql/deprecation.rb b/lib/gitlab/graphql/deprecation.rb deleted file mode 100644 index 9b17962f9ec..00000000000 --- a/lib/gitlab/graphql/deprecation.rb +++ /dev/null @@ -1,149 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - class Deprecation - REASON_RENAMED = :renamed - REASON_ALPHA = :alpha - - REASONS = { - REASON_RENAMED => 'This was renamed.', - REASON_ALPHA => 'This feature is in Alpha. It can be changed or removed at any time.' - }.freeze - - include ActiveModel::Validations - - validates :milestone, presence: true, format: { with: /\A\d+\.\d+\z/, message: 'must be milestone-ish' } - validates :reason, presence: true - validates :reason, - format: { with: /.*[^.]\z/, message: 'must not end with a period' }, - if: :reason_is_string? - validate :milestone_is_string - validate :reason_known_or_string - - def self.parse(alpha: nil, deprecated: nil) - options = alpha || deprecated - return unless options - - if alpha - raise ArgumentError, '`alpha` and `deprecated` arguments cannot be passed at the same time' \ - if deprecated - - options[:reason] = :alpha - end - - new(**options) - end - - def initialize(reason: nil, milestone: nil, replacement: nil) - @reason = reason.presence - @milestone = milestone.presence - @replacement = replacement.presence - end - - def ==(other) - return false unless other.is_a?(self.class) - - [reason_text, milestone, replacement] == [:reason_text, :milestone, :replacement].map do |attr| - other.send(attr) # rubocop: disable GitlabSecurity/PublicSend - end - end - alias_method :eql, :== - - def markdown(context: :inline) - parts = [ - "#{changed_in_milestone(format: :markdown)}.", - reason_text, - replacement_markdown.then { |r| "Use: #{r}." if r } - ].compact - - case context - when :block - ['WARNING:', *parts].join("\n") - when :inline - parts.join(' ') - end - end - - def replacement_markdown - return unless replacement.present? - return "`#{replacement}`" unless replacement.include?('.') # only fully qualified references can be linked - - "[`#{replacement}`](##{replacement.downcase.tr('.', '')})" - end - - def edit_description(original_description) - @original_description = original_description - return unless original_description - - original_description + description_suffix - end - - def original_description - return unless @original_description - return @original_description if @original_description.ends_with?('.') - - "#{@original_description}." - end - - def deprecation_reason - [ - reason_text, - replacement && "Please use `#{replacement}`.", - "#{changed_in_milestone}." - ].compact.join(' ') - end - - def alpha? - reason == REASON_ALPHA - end - - private - - attr_reader :reason, :milestone, :replacement - - def milestone_is_string - return if milestone.is_a?(String) - - errors.add(:milestone, 'must be a string') - end - - def reason_known_or_string - return if REASONS.key?(reason) - return if reason_is_string? - - errors.add(:reason, 'must be a known reason or a string') - end - - def reason_is_string? - reason.is_a?(String) - end - - def reason_text - @reason_text ||= REASONS[reason] || "#{reason.to_s.strip}." - end - - def description_suffix - " #{changed_in_milestone}: #{reason_text}" - end - - # Returns 'Deprecated in <milestone>' for proper deprecations. - # Retruns 'Introduced in <milestone>' for :alpha deprecations. - # Formatted to markdown or plain format. - def changed_in_milestone(format: :plain) - verb = if alpha? - 'Introduced' - else - 'Deprecated' - end - - case format - when :plain - "#{verb} in #{milestone}" - when :markdown - "**#{verb}** in #{milestone}" - end - end - end - end -end diff --git a/lib/gitlab/graphql/deprecations.rb b/lib/gitlab/graphql/deprecations.rb new file mode 100644 index 00000000000..9cd8462f2e8 --- /dev/null +++ b/lib/gitlab/graphql/deprecations.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +# Concern for handling GraphQL deprecations. +# https://docs.gitlab.com/ee/development/api_graphql_styleguide.html#deprecating-schema-items +module Gitlab + module Graphql + module Deprecations + extend ActiveSupport::Concern + + included do + attr_accessor :deprecation + end + + def visible?(ctx) + super && ctx[:remove_deprecated] == true ? deprecation.nil? : true + end + + private + + # Set deprecation, mutate the arguments + def init_gitlab_deprecation(kwargs) + if kwargs[:deprecation_reason].present? + raise ArgumentError, <<~ERROR + Use `deprecated` property instead of `deprecation_reason`. See + #{Rails.application.routes.url_helpers.help_page_url('development/api_graphql_styleguide', anchor: 'deprecating-schema-items')} + ERROR + end + + # GitLab allows items to be marked as "alpha", which leverages GraphQL deprecations. + # TODO remove + deprecation_args = kwargs.extract!(:alpha, :deprecated) + + self.deprecation = Deprecation.parse(**deprecation_args) + return unless deprecation + + unless deprecation.valid? + raise ArgumentError, "Bad deprecation. #{deprecation.errors.full_messages.to_sentence}" + end + + kwargs[:deprecation_reason] = deprecation.deprecation_reason + kwargs[:description] = deprecation.edit_description(kwargs[:description]) + end + end + end +end diff --git a/lib/gitlab/graphql/deprecations/deprecation.rb b/lib/gitlab/graphql/deprecations/deprecation.rb new file mode 100644 index 00000000000..7f4cea7c635 --- /dev/null +++ b/lib/gitlab/graphql/deprecations/deprecation.rb @@ -0,0 +1,151 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Deprecations + class Deprecation + REASON_RENAMED = :renamed + REASON_ALPHA = :alpha # TODO remove support in this class + + REASONS = { + REASON_RENAMED => 'This was renamed.', + REASON_ALPHA => 'This feature is in Alpha. It can be changed or removed at any time.' + }.freeze + + include ActiveModel::Validations + + validates :milestone, presence: true, format: { with: /\A\d+\.\d+\z/, message: 'must be milestone-ish' } + validates :reason, presence: true + validates :reason, + format: { with: /.*[^.]\z/, message: 'must not end with a period' }, + if: :reason_is_string? + validate :milestone_is_string + validate :reason_known_or_string + + def self.parse(alpha: nil, deprecated: nil) + options = alpha || deprecated + return unless options + + if alpha + raise ArgumentError, '`alpha` and `deprecated` arguments cannot be passed at the same time' \ + if deprecated + + options[:reason] = :alpha + end + + new(**options) + end + + def initialize(reason: nil, milestone: nil, replacement: nil) + @reason = reason.presence + @milestone = milestone.presence + @replacement = replacement.presence + end + + def ==(other) + return false unless other.is_a?(self.class) + + [reason_text, milestone, replacement] == [:reason_text, :milestone, :replacement].map do |attr| + other.send(attr) # rubocop: disable GitlabSecurity/PublicSend + end + end + alias_method :eql, :== + + def markdown(context: :inline) + parts = [ + "#{changed_in_milestone(format: :markdown)}.", + reason_text, + replacement_markdown.then { |r| "Use: #{r}." if r } + ].compact + + case context + when :block + ['WARNING:', *parts].join("\n") + when :inline + parts.join(' ') + end + end + + def replacement_markdown + return unless replacement.present? + return "`#{replacement}`" unless replacement.include?('.') # only fully qualified references can be linked + + "[`#{replacement}`](##{replacement.downcase.tr('.', '')})" + end + + def edit_description(original_description) + @original_description = original_description + return unless original_description + + original_description + description_suffix + end + + def original_description + return unless @original_description + return @original_description if @original_description.ends_with?('.') + + "#{@original_description}." + end + + def deprecation_reason + [ + reason_text, + replacement && "Please use `#{replacement}`.", + "#{changed_in_milestone}." + ].compact.join(' ') + end + + def alpha? + reason == REASON_ALPHA + end + + private + + attr_reader :reason, :milestone, :replacement + + def milestone_is_string + return if milestone.is_a?(String) + + errors.add(:milestone, 'must be a string') + end + + def reason_known_or_string + return if REASONS.key?(reason) + return if reason_is_string? + + errors.add(:reason, 'must be a known reason or a string') + end + + def reason_is_string? + reason.is_a?(String) + end + + def reason_text + @reason_text ||= REASONS[reason] || "#{reason.to_s.strip}." + end + + def description_suffix + " #{changed_in_milestone}: #{reason_text}" + end + + # Returns 'Deprecated in <milestone>' for proper deprecations. + # Retruns 'Introduced in <milestone>' for :alpha deprecations. + # Formatted to markdown or plain format. + def changed_in_milestone(format: :plain) + verb = if alpha? + 'Introduced' + else + 'Deprecated' + end + + case format + when :plain + "#{verb} in #{milestone}" + when :markdown + "**#{verb}** in #{milestone}" + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/markdown_field.rb b/lib/gitlab/graphql/markdown_field.rb index 43dddf4c4bc..f7bd1d43995 100644 --- a/lib/gitlab/graphql/markdown_field.rb +++ b/lib/gitlab/graphql/markdown_field.rb @@ -15,7 +15,7 @@ module Gitlab resolver_method = "#{name}_resolver".to_sym kwargs[:resolver_method] = resolver_method - kwargs[:description] ||= "The GitLab Flavored Markdown rendering of `#{method_name}`" + kwargs[:description] ||= "GitLab Flavored Markdown rendering of `#{method_name}`" # Adding complexity to rendered notes since that could cause queries. kwargs[:complexity] ||= 5 diff --git a/lib/gitlab/graphql/queries.rb b/lib/gitlab/graphql/queries.rb index cf06a2729d9..9cdc84ffaa3 100644 --- a/lib/gitlab/graphql/queries.rb +++ b/lib/gitlab/graphql/queries.rb @@ -91,7 +91,8 @@ module Gitlab end def print_field(field, indent: '') - if skips? && field.directives.any? { |d| d.name == 'client' } + if skips? && + (field.directives.any? { |d| d.name == 'client' || d.name == 'persist' } || field.name == '__persist') skipped = self.class.new(false) skipped.print_node(field) @@ -136,7 +137,7 @@ module Gitlab qs = [query] + all_imports(mode: mode).uniq.sort.map { |p| fragment(p).query } t = qs.join("\n\n").gsub(/\n\n+/, "\n\n") - return t unless /@client/.match?(t) + return t unless /(@client)|(persist)/.match?(t) doc = ::GraphQL.parse(t) printer = ClientFieldRedactor.new diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index 3ef60be67a9..aec430f2686 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -59,8 +59,6 @@ module Gitlab end def dns_rebind_protection? - return false if Gitlab.http_proxy_env? - Gitlab::CurrentSettings.dns_rebinding_protection_enabled? end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 31952f75006..8fe5868ca57 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,30 +44,30 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 35, + 'da_DK' => 34, 'de' => 16, 'en' => 100, 'eo' => 0, - 'es' => 34, + 'es' => 33, 'fil_PH' => 0, - 'fr' => 98, + 'fr' => 99, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, - 'ja' => 29, + 'ja' => 31, 'ko' => 20, - 'nb_NO' => 24, + 'nb_NO' => 23, 'nl_NL' => 0, 'pl_PL' => 3, 'pt_BR' => 57, - 'ro_RO' => 94, + 'ro_RO' => 91, 'ru' => 26, 'si_LK' => 11, 'tr_TR' => 10, - 'uk' => 54, + 'uk' => 55, 'zh_CN' => 98, 'zh_HK' => 1, - 'zh_TW' => 99 + 'zh_TW' => 98 }.freeze private_constant :TRANSLATION_LEVELS @@ -116,5 +116,43 @@ module Gitlab def with_default_locale(&block) with_locale(::I18n.default_locale, &block) end + + def setup(domain:, default_locale:) + setup_repositories(domain) + setup_default_locale(default_locale) + end + + private + + def setup_repositories(domain) + translation_repositories = [ + (po_repository(domain, 'jh/locale') if Gitlab.jh?), + po_repository(domain, 'locale') + ].compact + + FastGettext.add_text_domain( + domain, + type: :chain, + chain: translation_repositories, + ignore_fuzzy: true + ) + + FastGettext.default_text_domain = domain + end + + def po_repository(domain, path) + FastGettext::TranslationRepository.build( + domain, + path: Rails.root.join(path), + type: :po, + ignore_fuzzy: true + ) + end + + def setup_default_locale(locale) + FastGettext.default_locale = locale + FastGettext.default_available_locales = available_locales + ::I18n.available_locales = available_locales + end end end diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb index f3cca97950d..6623d42f526 100644 --- a/lib/gitlab/i18n/translation_entry.rb +++ b/lib/gitlab/i18n/translation_entry.rb @@ -65,7 +65,7 @@ module Gitlab end def translations_have_multiple_lines? - translation_entries.any? { |translation| translation.is_a?(Array) } + translation_entries.any?(Array) end def msgid_contains_unescaped_chars? diff --git a/lib/gitlab/import_export/base/object_builder.rb b/lib/gitlab/import_export/base/object_builder.rb index 7dee0f783cc..0f24492ed3c 100644 --- a/lib/gitlab/import_export/base/object_builder.rb +++ b/lib/gitlab/import_export/base/object_builder.rb @@ -15,8 +15,8 @@ module Gitlab LRU_CACHE_SIZE = 1000 class ObjectBuilder - def self.build(*args) - new(*args).find + def self.build(...) + new(...).find end def initialize(klass, attributes) diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index d1fd45882d3..e3813070aa4 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -158,9 +158,9 @@ module Gitlab end def remove_encrypted_attributes! - return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any? + return unless relation_class.respond_to?(:attr_encrypted_attributes) && relation_class.attr_encrypted_attributes.any? - relation_class.encrypted_attributes.each_key do |key| + relation_class.attr_encrypted_attributes.each_key do |key| @relation_hash[key.to_s] = nil end end diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index 7f3254be3e8..e30414265be 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -98,6 +98,9 @@ methods: epics: - :state +# Add in this list the nested associations that are used to export the parent +# association, but are not present in the tree list. In other words, the associations +# that needs to be preloaded but do not need to be exported. preloads: export_reorders: diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index c2510bbe938..628c7c71539 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -176,3 +176,5 @@ module Gitlab end end end + +Gitlab::ImportExport::Importer.prepend_mod_with('Gitlab::ImportExport::Importer') diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index cf62f181366..389ab8b4c97 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -85,17 +85,30 @@ module Gitlab end def exportable_json_record(record, options, key) - associations = relations_schema[:include_if_exportable]&.dig(key) - return Raw.new(record.to_json(options)) unless associations && options[:include] + return Raw.new(record.to_json(options)) unless options[:include].any? + conditional_associations = relations_schema[:include_if_exportable]&.dig(key) + + filtered_options = + if conditional_associations.present? + filter_conditional_include(record, options, conditional_associations) + else + options + end + + Raw.new(authorized_record_json(record, filtered_options)) + end + + def filter_conditional_include(record, options, conditional_associations) filtered_options = options.deep_dup - associations.each do |association| + + conditional_associations.each do |association| filtered_options[:include].delete_if do |option| !exportable_json_association?(option, record, association.to_sym) end end - Raw.new(record.to_json(filtered_options)) + filtered_options end def exportable_json_association?(option, record, association) @@ -105,6 +118,34 @@ module Gitlab record.exportable_association?(association, current_user: current_user) end + def authorized_record_json(record, options) + include_keys = options[:include].flat_map(&:keys) + keys_to_authorize = record.try(:restricted_associations, include_keys) + return record.to_json(options) if keys_to_authorize.blank? + + record_hash = record.as_json(options).with_indifferent_access + filtered_record_hash(record, keys_to_authorize, record_hash).to_json(options) + end + + def filtered_record_hash(record, keys_to_authorize, record_hash) + keys_to_authorize.each do |key| + next unless record_hash[key].present? + + readable = record.try(:readable_records, key, current_user: current_user) + if record.has_many_association?(key) + readable_ids = readable.pluck(:id) + + record_hash[key].keep_if do |association_record| + readable_ids.include?(association_record[:id]) + end + else + record_hash[key] = nil unless readable.present? + end + end + + record_hash + end + def batch(relation, key) opts = { of: BATCH_SIZE } order_by = reorders(relation, key) diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 99364996864..d97ffee8698 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -1005,6 +1005,7 @@ excluded_attributes: - :protected_branch_id create_access_levels: - :protected_tag_id + - :deploy_key_id deploy_access_levels: - :protected_environment_id boards: @@ -1104,6 +1105,9 @@ methods: issues: - :state +# Add in this list the nested associations that are used to export the parent +# association, but are not present in the tree list. In other words, the associations +# that needs to be preloaded but do not need to be exported. preloads: issues: project: :route @@ -1112,8 +1116,8 @@ preloads: # tags: # needed by tag_list project: # deprecated: needed by coverage_regex of Ci::Build merge_requests: - source_project: # needed by source_branch_sha and diff_head_sha - target_project: # needed by target_branch_sha + source_project: :route # needed by source_branch_sha and diff_head_sha + target_project: :route # needed by target_branch_sha assignees: # needed by assigne_id that is implemented by DeprecatedAssignee # Specify a custom export reordering for a given relationship diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 568315930d8..4134c428500 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -165,9 +165,21 @@ module Gitlab end def setup_protected_branch_access_level + return if root_group_owner? + return if @relation_hash['access_level'] == Gitlab::Access::NO_ACCESS + return if @relation_hash['access_level'] == Gitlab::Access::MAINTAINER + @relation_hash['access_level'] = Gitlab::Access::MAINTAINER end + def root_group_owner? + root_ancestor = @importable.root_ancestor + + return false unless root_ancestor.is_a?(::Group) + + root_ancestor.max_member_access_for_user(@user) == Gitlab::Access::OWNER + end + def compute_relative_position return unless max_relative_position diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index 52234b50a1f..880b112d815 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -38,6 +38,10 @@ module Gitlab key.is_a?(String) && key =~ /\A#{KEY_PREFIX}:\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/o end + def self.generate_key + "#{KEY_PREFIX}:#{SecureRandom.uuid}" + end + attr_reader :key, :finished, :worker_label attr_accessor :jobs_remaining diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 8332e4f6d56..bdfbe2041cd 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -103,7 +103,7 @@ module Gitlab opts = standardize_opts(opts) Oj.load(string, opts) - rescue Oj::ParseError, EncodingError, Encoding::UndefinedConversionError => ex + rescue Oj::ParseError, EncodingError, Encoding::UndefinedConversionError, JSON::GeneratorError => ex raise parser_error, ex end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index f7cd28df5c9..bad2e265f73 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -3,6 +3,9 @@ require 'yaml' require 'json' require 'pathname' +require 'active_support' +require "active_support/core_ext/module/delegation" +require_relative 'encrypted_configuration' unless defined?(Gitlab::EncryptedConfiguration) require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) # This service is run independently of the main Rails process, @@ -42,6 +45,20 @@ module Gitlab } }.freeze + # Default path strings (this is a data duplication + # with Settings which is not pulled in - see the service + # comment at the top of this file) + DEFAULT_PATHS = { + shared_path: 'shared', + encrypted_settings_path: 'encrypted_settings', + incoming_email: { + encrypted_secret_filename: 'incoming_email.yaml.enc' + }, + service_desk_email: { + encrypted_secret_filename: 'service_desk_email.yaml.enc' + } + }.freeze + class << self def enabled_configs @enabled_configs ||= configs.select { |_key, config| enabled?(config) } @@ -74,6 +91,12 @@ module Gitlab config[:log_path] = File.expand_path(config[:log_path], RAILS_ROOT_DIR) + # override password/user from any encrypted secrets + if secrets = decrypted_secrets(config_key) + config[:password] = secrets[:password] if secrets[:password] + config[:user] = secrets[:user] if secrets[:user] + end + config end @@ -108,6 +131,58 @@ module Gitlab def load_yaml @yaml ||= YAML.load_file(config_file)[rails_env].deep_symbolize_keys end + + def application_secrets_file + ENV['MAIL_ROOM_GITLAB_SECRETS_FILE'] || File.expand_path('../../config/secrets.yml', __dir__) + end + + def application_secrets + @application_secrets ||= {}.tap do |application_secrets| + # Uses Rails::Secret.parse + # from: https://github.com/rails/rails/blob/v6.1.6.1/railties/lib/rails/secrets.rb#L24 + erb_processed_yaml = ERB.new(File.read(application_secrets_file)).result + yaml_secrets = + YAML.respond_to?(:unsafe_load) ? YAML.unsafe_load(erb_processed_yaml) : YAML.safe_load(erb_processed_yaml) + application_secrets.merge!(yaml_secrets["shared"].deep_symbolize_keys) if yaml_secrets["shared"] + application_secrets.merge!(yaml_secrets[rails_env].deep_symbolize_keys) if yaml_secrets[rails_env] + end + end + + def default_encrypted_secret_filename(config_key) + DEFAULT_PATHS[config_key][:encrypted_secret_filename] + end + + def encrypted_secret_file(config_key) + config = merged_configs(config_key) + return config[:encrypted_secret_file] if config[:encrypted_secret_file] + + config_yaml = load_yaml + # Path handling for shared.path / encrypted_settings.path is a duplicate + # of the logic in config/initializers/1_settings.rb + shared_path = File.expand_path(config_yaml.dig(:shared, :path) || + DEFAULT_PATHS[:shared_path], RAILS_ROOT_DIR) + encrypted_settings_path = + File.expand_path(config_yaml.dig(:encrypted_settings, :path) || + File.join(shared_path, DEFAULT_PATHS[:encrypted_settings_path]), + RAILS_ROOT_DIR) + File.join(encrypted_settings_path, default_encrypted_secret_filename(config_key)) + end + + def encrypted_configuration_settings(config_key) + { + content_path: encrypted_secret_file(config_key), + base_key: application_secrets[:encrypted_settings_key_base], + previous_keys: application_secrets[:rotated_encrypted_settings_key_base] || [] + } + end + + def decrypted_secrets(config_key) + settings = encrypted_configuration_settings(config_key) + return if settings[:base_key].nil? + + encrypted = Gitlab::EncryptedConfiguration.new(**settings) + encrypted.active? ? encrypted.config : nil + end end end end diff --git a/lib/gitlab/memory/reporter.rb b/lib/gitlab/memory/reporter.rb index 5effafc9f5b..db0fd24983b 100644 --- a/lib/gitlab/memory/reporter.rb +++ b/lib/gitlab/memory/reporter.rb @@ -69,14 +69,14 @@ module Gitlab report_file = file_name(report) tmp_file_path = File.join(tmp_dir, report_file) - write_heap_dump_file(report, tmp_file_path) + write_compressed_file(report, tmp_file_path) File.join(@reports_path, report_file).tap do |report_file_path| FileUtils.mv(tmp_file_path, report_file_path) end end - def write_heap_dump_file(report, path) + def write_compressed_file(report, path) io_r, io_w = IO.pipe err_r, err_w = IO.pipe pid = nil diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb index c94dbed1d46..cc335c00e26 100644 --- a/lib/gitlab/memory/watchdog.rb +++ b/lib/gitlab/memory/watchdog.rb @@ -6,47 +6,6 @@ module Gitlab # into a handler when the Ruby process violates defined limits # for an extended period of time. class Watchdog - # This handler does nothing. It returns `false` to indicate to the - # caller that the situation has not been dealt with so it will - # receive calls repeatedly if fragmentation remains high. - # - # This is useful for "dress rehearsals" in production since it allows - # us to observe how frequently the handler is invoked before taking action. - class NullHandler - include Singleton - - def call - # NOP - false - end - end - - # This handler sends SIGTERM and considers the situation handled. - class TermProcessHandler - def initialize(pid = $$) - @pid = pid - end - - def call - Process.kill(:TERM, @pid) - true - end - end - - # This handler invokes Puma's graceful termination handler, which takes - # into account a configurable grace period during which a process may - # remain unresponsive to a SIGTERM. - class PumaHandler - def initialize(puma_options = ::Puma.cli_config.options) - @worker = ::Puma::Cluster::WorkerHandle.new(0, $$, 0, puma_options) - end - - def call - @worker.term - true - end - end - def initialize @configuration = Configuration.new @alive = true @@ -73,6 +32,7 @@ module Gitlab def stop stop_working(reason: 'background task stopped') + handler.stop if handler.respond_to?(:stop) end private @@ -111,7 +71,7 @@ module Gitlab def handler # This allows us to keep the watchdog running but turn it into "friendly mode" where # all that happens is we collect logs and Prometheus events for fragmentation violations. - return NullHandler.instance unless Feature.enabled?(:enforce_memory_watchdog, type: :ops) + return Handlers::NullHandler.instance unless Feature.enabled?(:enforce_memory_watchdog, type: :ops) configuration.handler end diff --git a/lib/gitlab/memory/watchdog/configuration.rb b/lib/gitlab/memory/watchdog/configuration.rb index 5c459220be8..6ab199bf816 100644 --- a/lib/gitlab/memory/watchdog/configuration.rb +++ b/lib/gitlab/memory/watchdog/configuration.rb @@ -48,7 +48,7 @@ module Gitlab end def handler - @handler ||= NullHandler.instance + @handler ||= Handlers::NullHandler.instance end def event_reporter diff --git a/lib/gitlab/memory/watchdog/configurator.rb b/lib/gitlab/memory/watchdog/configurator.rb index 04c04cbde02..4a6640ba901 100644 --- a/lib/gitlab/memory/watchdog/configurator.rb +++ b/lib/gitlab/memory/watchdog/configurator.rb @@ -12,12 +12,12 @@ module Gitlab DEFAULT_MAX_HEAP_FRAG = 0.5 DEFAULT_MAX_MEM_GROWTH = 3.0 # grace_time / sleep_interval = max_strikes allowed for Sidekiq process to violate defined limits. - DEFAULT_SIDEKIQ_GRACE_TIME_S = 300 + DEFAULT_SIDEKIQ_GRACE_TIME_S = 900 class << self def configure_for_puma ->(config) do - config.handler = Gitlab::Memory::Watchdog::PumaHandler.new + config.handler = Gitlab::Memory::Watchdog::Handlers::PumaHandler.new config.sleep_time_seconds = ENV.fetch('GITLAB_MEMWD_SLEEP_TIME_SEC', DEFAULT_SLEEP_INTERVAL_S).to_i config.monitors(&configure_monitors_for_puma) end @@ -25,7 +25,13 @@ module Gitlab def configure_for_sidekiq ->(config) do - config.handler = Gitlab::Memory::Watchdog::TermProcessHandler.new + # Give Sidekiq up to 30 seconds to allow existing jobs to finish after exceeding the limit + shutdown_timeout_seconds = ENV.fetch('SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT', 30).to_i + + config.handler = Gitlab::Memory::Watchdog::Handlers::SidekiqHandler.new( + shutdown_timeout_seconds, + sidekiq_sleep_time + ) config.sleep_time_seconds = sidekiq_sleep_time config.monitors(&configure_monitors_for_sidekiq) config.event_reporter = SidekiqEventReporter.new diff --git a/lib/gitlab/memory/watchdog/handlers/null_handler.rb b/lib/gitlab/memory/watchdog/handlers/null_handler.rb new file mode 100644 index 00000000000..127001003ce --- /dev/null +++ b/lib/gitlab/memory/watchdog/handlers/null_handler.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + module Handlers + # This handler does nothing. It returns `false` to indicate to the + # caller that the situation has not been dealt with so it will + # receive calls repeatedly if fragmentation remains high. + # + # This is useful for "dress rehearsals" in production since it allows + # us to observe how frequently the handler is invoked before taking action. + class NullHandler + include Singleton + + def call + # NOP + false + end + end + end + end + end +end diff --git a/lib/gitlab/memory/watchdog/handlers/puma_handler.rb b/lib/gitlab/memory/watchdog/handlers/puma_handler.rb new file mode 100644 index 00000000000..fffd91733c8 --- /dev/null +++ b/lib/gitlab/memory/watchdog/handlers/puma_handler.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + module Handlers + # This handler invokes Puma's graceful termination handler, which takes + # into account a configurable grace period during which a process may + # remain unresponsive to a SIGTERM. + class PumaHandler + def initialize(puma_options = ::Puma.cli_config.options) + @worker = ::Puma::Cluster::WorkerHandle.new(0, $$, 0, puma_options) + end + + def call + @worker.term + true + end + end + end + end + end +end diff --git a/lib/gitlab/memory/watchdog/handlers/sidekiq_handler.rb b/lib/gitlab/memory/watchdog/handlers/sidekiq_handler.rb new file mode 100644 index 00000000000..47ed608c576 --- /dev/null +++ b/lib/gitlab/memory/watchdog/handlers/sidekiq_handler.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + module Handlers + class SidekiqHandler + def initialize(shutdown_timeout_seconds, sleep_time_seconds) + @shutdown_timeout_seconds = shutdown_timeout_seconds + @sleep_time_seconds = sleep_time_seconds + @alive = true + end + + def call + # Tell Sidekiq to stop fetching new jobs + # We first SIGNAL and then wait given time + send_signal(:TSTP, $$, 'stop fetching new jobs', @shutdown_timeout_seconds) + return true unless @alive + + # Tell sidekiq to restart itself + # Keep extra safe to wait `Sidekiq[:timeout] + 2` seconds before SIGKILL + send_signal(:TERM, $$, 'gracefully shut down', Sidekiq[:timeout] + 2) + return true unless @alive + + # Ideally we should never reach this condition + # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't + # If process is group leader, kill the whole pgroup, so we can be sure no children are left behind + send_signal(:KILL, Process.getpgrp == $$ ? 0 : $$, 'hard shut down') + + true + end + + def stop + @alive = false + end + + private + + def send_signal(signal, pid, explanation, wait_time = nil) + Sidekiq.logger.warn( + pid: pid, + worker_id: ::Prometheus::PidProvider.worker_id, + memwd_handler_class: self.class.to_s, + memwd_signal: signal, + memwd_explanation: explanation, + memwd_wait_time: wait_time, + message: "Sending signal and waiting" + ) + + ProcessManagement.signal(pid, signal) + + return unless wait_time + + deadline = Gitlab::Metrics::System.monotonic_time + wait_time + + # Sleep until timeout reached + sleep(@sleep_time_seconds) while @alive && Gitlab::Metrics::System.monotonic_time < deadline + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb index d4f779ad79d..bdd28744137 100644 --- a/lib/gitlab/metrics/dashboard/url.rb +++ b/lib/gitlab/metrics/dashboard/url.rb @@ -100,8 +100,8 @@ module Gitlab end # Builds a metrics dashboard url based on the passed in arguments - def build_dashboard_url(*args) - Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(*args) + def build_dashboard_url(...) + Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(...) end private diff --git a/lib/gitlab/metrics/environment.rb b/lib/gitlab/metrics/environment.rb new file mode 100644 index 00000000000..43f2fa3922f --- /dev/null +++ b/lib/gitlab/metrics/environment.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Environment + class << self + def web? + service?('web') + end + + def api? + service?('api') + end + + def git? + service?('git') + end + + def service?(name) + env_var = ENV.fetch('GITLAB_METRICS_INITIALIZE', '') + return true unless env_var.present? + + env_var == name + end + end + end + end +end diff --git a/lib/gitlab/metrics/global_search_slis.rb b/lib/gitlab/metrics/global_search_slis.rb index fc2e805047a..c361d755a12 100644 --- a/lib/gitlab/metrics/global_search_slis.rb +++ b/lib/gitlab/metrics/global_search_slis.rb @@ -58,8 +58,16 @@ module Gitlab end def endpoint_ids - ['SearchController#show', 'GET /api/:version/search', 'GET /api/:version/projects/:id/(-/)search', - 'GET /api/:version/groups/:id/(-/)search'] + api_endpoints = ['GET /api/:version/search', 'GET /api/:version/projects/:id/(-/)search', + 'GET /api/:version/groups/:id/(-/)search'] + web_endpoints = ['SearchController#show'] + + endpoints = [] + + endpoints += api_endpoints if Gitlab::Metrics::Environment.api? + endpoints += web_endpoints if Gitlab::Metrics::Environment.web? + + endpoints end def possible_labels diff --git a/lib/gitlab/metrics/rails_slis.rb b/lib/gitlab/metrics/rails_slis.rb index 9fd4eec479e..6a5ef5614a3 100644 --- a/lib/gitlab/metrics/rails_slis.rb +++ b/lib/gitlab/metrics/rails_slis.rb @@ -5,8 +5,10 @@ module Gitlab module RailsSlis class << self def initialize_request_slis! - Gitlab::Metrics::Sli::Apdex.initialize_sli(:rails_request, possible_request_labels) - initialize_rails_request_error_rate + request_labels = possible_request_labels + + Gitlab::Metrics::Sli::Apdex.initialize_sli(:rails_request, request_labels) + Gitlab::Metrics::Sli::ErrorRate.initialize_sli(:rails_request, request_labels) Gitlab::Metrics::Sli::Apdex.initialize_sli(:graphql_query, possible_graphql_query_labels) end @@ -25,6 +27,8 @@ module Gitlab private def possible_graphql_query_labels + return [] unless Gitlab::Metrics::Environment.api? + ::Gitlab::Graphql::KnownOperations.default.operations.map do |op| { endpoint_id: op.to_caller_id, @@ -39,7 +43,27 @@ module Gitlab possible_controller_labels + possible_api_labels end + def possible_controller_labels + all_controller_labels.select do |labelset| + if known_git_endpoints.include?(labelset[:endpoint_id]) + Gitlab::Metrics::Environment.git? + else + Gitlab::Metrics::Environment.web? + end + end + end + def possible_api_labels + all_api_labels.select do |labelset| + if known_git_endpoints.include?(labelset[:endpoint_id]) + Gitlab::Metrics::Environment.git? + else + Gitlab::Metrics::Environment.api? + end + end + end + + def all_api_labels Gitlab::RequestEndpoints.all_api_endpoints.map do |route| endpoint_id = API::Base.endpoint_id_for_route(route) route_class = route.app.options[:for] @@ -54,7 +78,7 @@ module Gitlab end end - def possible_controller_labels + def all_controller_labels Gitlab::RequestEndpoints.all_controller_actions.map do |controller, action| { endpoint_id: controller.endpoint_id_for_action(action), @@ -64,10 +88,27 @@ module Gitlab end end - def initialize_rails_request_error_rate - return unless Feature.enabled?(:gitlab_metrics_error_rate_sli, type: :development) - - Gitlab::Metrics::Sli::ErrorRate.initialize_sli(:rails_request, possible_request_labels) + def known_git_endpoints + # This is a list of endpoints that endpoints that HAProxy redirects + # to the git fleet for GitLab.com. It is taken from + # https://thanos-query.ops.gitlab.net/graph?g0.expr=sum%20by%20(endpoint_id)(sli_aggregations%3Agitlab_sli_rails_request_total_rate6h%7Btype%3D%22git%22%2C%20env%3D%22gprd%22%7D%20%3E%200)&g0.tab=1&g0.stacked=0&g0.range_input=1h&g0.max_source_resolution=0s&g0.deduplicate=1&g0.partial_response=0&g0.store_matches=%5B%5D + [ + "GET /api/:version/internal/authorized_keys", + "GET /api/:version/internal/discover", + "POST /api/:version/internal/allowed", + "POST /api/:version/internal/lfs_authenticate", + "POST /api/:version/internal/two_factor_recovery_codes", + "ProjectsController#show", + "Repositories::GitHttpController#git_receive_pack", + "Repositories::GitHttpController#git_upload_pack", + "Repositories::GitHttpController#info_refs", + "Repositories::LfsApiController#batch", + "Repositories::LfsLocksApiController#index", + "Repositories::LfsLocksApiController#verify", + "Repositories::LfsStorageController#download", + "Repositories::LfsStorageController#upload_authorize", + "Repositories::LfsStorageController#upload_finalize" + ] end end end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index cfdac5264e0..f635deabf76 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -27,6 +27,8 @@ module Gitlab 'not_owned', 'source_code_management', FEATURE_CATEGORY_DEFAULT].freeze + REQUEST_URGENCY_KEY = 'gitlab.request_urgency' + def initialize(app) @app = app end @@ -125,8 +127,6 @@ module Gitlab end def record_error(urgency, status) - return unless Feature.enabled?(:gitlab_metrics_error_rate_sli, type: :development) - Gitlab::Metrics::RailsSlis.request_error_rate.increment( labels: labels_from_context.merge(request_urgency: urgency.name), error: ::Gitlab::Metrics.server_error?(status) @@ -142,7 +142,9 @@ module Gitlab def urgency_for_env(env) endpoint_urgency = - if env['api.endpoint'].present? + if env[REQUEST_URGENCY_KEY].present? + env[REQUEST_URGENCY_KEY] + elsif env['api.endpoint'].present? env['api.endpoint'].options[:for].try(:urgency_for_app, env['api.endpoint']) elsif env['action_controller.instance'].present? && env['action_controller.instance'].respond_to?(:urgency) env['action_controller.instance'].urgency diff --git a/lib/gitlab/metrics/subscribers/ldap.rb b/lib/gitlab/metrics/subscribers/ldap.rb index 9cac5f41090..3dae2d1fd88 100644 --- a/lib/gitlab/metrics/subscribers/ldap.rb +++ b/lib/gitlab/metrics/subscribers/ldap.rb @@ -73,8 +73,8 @@ module Gitlab def add_to_request_store(event) return unless Gitlab::SafeRequestStore.active? - Gitlab::SafeRequestStore[COUNTER] = Gitlab::SafeRequestStore[COUNTER].to_i + 1 - Gitlab::SafeRequestStore[DURATION] = Gitlab::SafeRequestStore[DURATION].to_f + event.duration.to_f + Gitlab::SafeRequestStore[COUNTER] = self.class.count + 1 + Gitlab::SafeRequestStore[DURATION] = self.class.duration + convert_to_seconds(event.duration) end # Converts the observed events into Prometheus metrics @@ -85,18 +85,23 @@ module Gitlab # and so we only want the first part, which is the # true name of the event labels = { name: event.name.split(".").first } + duration = convert_to_seconds(event.duration) current_transaction.increment(:gitlab_net_ldap_total, 1, labels) do docstring 'Net::LDAP calls' label_keys labels.keys end - current_transaction.observe(:gitlab_net_ldap_duration_seconds, event.duration, labels) do + current_transaction.observe(:gitlab_net_ldap_duration_seconds, duration, labels) do docstring 'Net::LDAP time' buckets [0.001, 0.01, 0.1, 1.0, 2.0, 5.0] label_keys labels.keys end end + + def convert_to_seconds(duration_f) + (BigDecimal(duration_f.to_s) / BigDecimal("1000.0")).to_f + end end end end diff --git a/lib/gitlab/nav/top_nav_view_model_builder.rb b/lib/gitlab/nav/top_nav_view_model_builder.rb index 8cb2729ff61..10b841f777e 100644 --- a/lib/gitlab/nav/top_nav_view_model_builder.rb +++ b/lib/gitlab/nav/top_nav_view_model_builder.rb @@ -11,12 +11,12 @@ module Gitlab # Using delegate hides the stacktrace for some errors, so we choose to be explicit. # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62047#note_579031091 - def add_primary_menu_item(**args) - @menu_builder.add_primary_menu_item(**args) + def add_primary_menu_item(...) + @menu_builder.add_primary_menu_item(...) end - def add_secondary_menu_item(**args) - @menu_builder.add_secondary_menu_item(**args) + def add_secondary_menu_item(...) + @menu_builder.add_secondary_menu_item(...) end def add_shortcut(**args) diff --git a/lib/gitlab/octokit/middleware.rb b/lib/gitlab/octokit/middleware.rb index a92860f7eb8..0e47672bb3c 100644 --- a/lib/gitlab/octokit/middleware.rb +++ b/lib/gitlab/octokit/middleware.rb @@ -11,7 +11,8 @@ module Gitlab Gitlab::UrlBlocker.validate!(env[:url], schemes: %w[http https], allow_localhost: allow_local_requests?, - allow_local_network: allow_local_requests? + allow_local_network: allow_local_requests?, + dns_rebind_protection: dns_rebind_protection? ) @app.call(env) @@ -22,6 +23,10 @@ module Gitlab def allow_local_requests? Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? end + + def dns_rebind_protection? + Gitlab::CurrentSettings.dns_rebinding_protection_enabled? + end end end end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index b78cd2a6b95..a03533dcd9a 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -23,8 +23,6 @@ module Gitlab case provider_name when 'cas3' { on_single_sign_out: cas3_signout_handler } - when 'authentiq' - { remote_sign_out_handler: authentiq_signout_handler } when 'shibboleth' { fail_with_empty_uid: true } when 'google_oauth2' @@ -53,24 +51,12 @@ module Gitlab true end end - - def authentiq_signout_handler - lambda do |request| - authentiq_session = request.params['sid'] - if Gitlab::Auth::OAuth::Session.valid?(:authentiq, authentiq_session) - Gitlab::Auth::OAuth::Session.destroy(:authentiq, authentiq_session) - true - else - false - end - end - end end private - def add_provider_to_devise(*args) - @devise_config.omniauth(*args) + def add_provider_to_devise(...) + @devise_config.omniauth(...) end def arguments_for(provider) diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb index 38618a0ac06..8e02486d8c6 100644 --- a/lib/gitlab/otp_key_rotator.rb +++ b/lib/gitlab/otp_key_rotator.rb @@ -67,7 +67,7 @@ module Gitlab attr_reader :old_key, :new_key def otp_secret_settings - @otp_secret_settings ||= User.encrypted_attributes[:otp_secret] + @otp_secret_settings ||= User.attr_encrypted_attributes[:otp_secret] end def reencrypt(user, old_key, new_key) diff --git a/lib/gitlab/pages/cache_control.rb b/lib/gitlab/pages/cache_control.rb index a24d958b7e5..81da34f1219 100644 --- a/lib/gitlab/pages/cache_control.rb +++ b/lib/gitlab/pages/cache_control.rb @@ -47,8 +47,15 @@ module Gitlab # cached settings hash to build the payload cache key to be invalidated. def clear_cache keys = cached_settings_hashes - .map { |hash| payload_cache_key_for(hash) } - .push(settings_cache_key) + .map { |hash| payload_cache_key_for(hash) } + .push(settings_cache_key) + + ::Gitlab::AppLogger.info( + message: 'clear pages cache', + pages_keys: keys, + pages_type: @type, + pages_id: @id + ) Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do Rails.cache.delete_multi(keys) diff --git a/lib/gitlab/patch/prependable.rb b/lib/gitlab/patch/prependable.rb index b974c0b2c7f..7619fc317a1 100644 --- a/lib/gitlab/patch/prependable.rb +++ b/lib/gitlab/patch/prependable.rb @@ -8,6 +8,7 @@ # 2. Allow `prepended do; end` work like `included do; end` # If we don't need anything above, we don't need this patch nor the concern! +require_dependency 'gitlab/environment' # rubocop:disable Gitlab/ModuleWithInstanceVariables module Gitlab module Patch @@ -51,7 +52,7 @@ module Gitlab end # Hack to resolve https://gitlab.com/gitlab-org/gitlab/-/issues/23932 - extend class_methods_module if ENV['STATIC_VERIFICATION'] + extend class_methods_module if Gitlab::Environment.static_verification? end def prepended(base = nil, &block) diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 8a1dcc083e8..28d195238ea 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -46,8 +46,8 @@ module Gitlab @options = parse_options(options) end - def get(*args) - options.dig(*args) + def get(...) + options.dig(...) end # Allow #to_json serialization @@ -83,7 +83,7 @@ module Gitlab end def option_multi_value?(namespace, key) - MULTI_VALUE_OPTIONS.any? { |arr| arr == [namespace, key] } + MULTI_VALUE_OPTIONS.any?([namespace, key]) end def parse_option(option) diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index d9135d1bacb..118c8aad460 100644 --- a/lib/gitlab/quick_actions/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -149,7 +149,13 @@ module Gitlab end def valid_type?(context) - types.blank? || types.any? { |type| context.quick_action_target.is_a?(type) } + types.blank? || types.any? do |type| + if context.quick_action_target.is_a?(WorkItem) + context.quick_action_target.supported_quick_action_commands.include?(name.to_sym) + else + context.quick_action_target.is_a?(type) + end + end end end end diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index a12457d89c9..96e3112f32f 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -248,7 +248,7 @@ module Gitlab if severity if quick_action_target.persisted? - ::Issues::UpdateService.new(project: quick_action_target.project, current_user: current_user, params: { severity: severity }).execute(quick_action_target) + ::Issues::UpdateService.new(container: quick_action_target.project, current_user: current_user, params: { severity: severity }).execute(quick_action_target) else quick_action_target.build_issuable_severity(severity: severity) end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index f782f2802b6..ae8bc102f57 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -320,7 +320,7 @@ module Gitlab private def zoom_link_service - ::Issues::ZoomLinkService.new(project: quick_action_target.project, current_user: current_user, params: { issue: quick_action_target }) + ::Issues::ZoomLinkService.new(container: quick_action_target.project, current_user: current_user, params: { issue: quick_action_target }) end def zoom_link_params diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index ed4f6015603..4d15022cca5 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -9,9 +9,11 @@ module Gitlab # config/initializers/7_redis.rb, instrumented, and used in health- & readiness checks. ALL_CLASSES = [ Gitlab::Redis::Cache, + Gitlab::Redis::DbLoadBalancing, Gitlab::Redis::Queues, Gitlab::Redis::RateLimiting, Gitlab::Redis::RepositoryCache, + Gitlab::Redis::ClusterRateLimiting, Gitlab::Redis::Sessions, Gitlab::Redis::SharedState, Gitlab::Redis::TraceChunks diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 043f14630d5..647573e59fe 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -2,6 +2,16 @@ module Gitlab module Redis + # Match signature in + # https://github.com/rails/rails/blob/v6.1.7.2/activesupport/lib/active_support/cache/redis_cache_store.rb#L59 + ERROR_HANDLER = ->(method:, returning:, exception:) do + Gitlab::ErrorTracking.log_exception( + exception, + method: method, + returning: returning.inspect + ) + end + class Cache < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab' @@ -12,9 +22,14 @@ module Gitlab redis: pool, compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), namespace: CACHE_NAMESPACE, - expires_in: ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 8.hours).to_i # Cache should not grow forever + expires_in: default_ttl_seconds, + error_handler: ::Gitlab::Redis::ERROR_HANDLER } end + + def self.default_ttl_seconds + ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 8.hours).to_i + end end end end diff --git a/lib/gitlab/redis/cluster_rate_limiting.rb b/lib/gitlab/redis/cluster_rate_limiting.rb new file mode 100644 index 00000000000..e9d1e4f0c3f --- /dev/null +++ b/lib/gitlab/redis/cluster_rate_limiting.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class ClusterRateLimiting < ::Gitlab::Redis::Wrapper + def self.config_fallback + Cache + end + end + end +end diff --git a/lib/gitlab/redis/db_load_balancing.rb b/lib/gitlab/redis/db_load_balancing.rb new file mode 100644 index 00000000000..01276445611 --- /dev/null +++ b/lib/gitlab/redis/db_load_balancing.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class DbLoadBalancing < ::Gitlab::Redis::Wrapper + class << self + # The data we store on DbLoadBalancing used to be stored on SharedState. + def config_fallback + SharedState + end + + private + + def redis + primary_store = ::Redis.new(params) + secondary_store = ::Redis.new(config_fallback.params) + + MultiStore.new(primary_store, secondary_store, store_name) + end + end + end + end +end diff --git a/lib/gitlab/redis/duplicate_jobs.rb b/lib/gitlab/redis/duplicate_jobs.rb deleted file mode 100644 index c76d298da18..00000000000 --- a/lib/gitlab/redis/duplicate_jobs.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Redis - # Pseudo-store to transition `Gitlab::SidekiqMiddleware::DuplicateJobs` from - # using `Sidekiq.redis` to using the `SharedState` redis store. - class DuplicateJobs < ::Gitlab::Redis::Wrapper - class << self - def store_name - 'SharedState' - end - - private - - def redis - primary_store = ::Redis.new(Gitlab::Redis::SharedState.params) - - # `Sidekiq.redis` is a namespaced redis connection. This means keys are actually being stored under - # "resque:gitlab:resque:gitlab:duplicate:". For backwards compatibility, we make the secondary store - # namespaced in the same way, but omit it from the primary so keys have proper format there. - # rubocop:disable Cop/RedisQueueUsage - secondary_store = ::Redis::Namespace.new( - Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE, redis: ::Redis.new(Gitlab::Redis::Queues.params) - ) - # rubocop:enable Cop/RedisQueueUsage - - MultiStore.new(primary_store, secondary_store, name.demodulize) - end - end - end - end -end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index aa8f390ac10..a102267d52b 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -46,13 +46,6 @@ module Gitlab # # Ref: https://www.rubydoc.info/github/redis/redis-rb/Redis/Commands # - ENUMERATOR_CACHE_HIT_VALIDATOR = { - scan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, - hscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, - sscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, - zscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? } - }.freeze - READ_CACHE_HIT_VALIDATOR = { exists: ->(val) { val != 0 }, exists?: ->(val) { val }, @@ -62,13 +55,17 @@ module Gitlab hgetall: ->(val) { val.is_a?(Hash) && !val.empty? }, hlen: ->(val) { val != 0 }, hmget: ->(val) { val.is_a?(Array) && !val.compact.empty? }, + hscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, mapped_hmget: ->(val) { val.is_a?(Hash) && !val.compact.empty? }, mget: ->(val) { val.is_a?(Array) && !val.compact.empty? }, + scan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, scard: ->(val) { val != 0 }, sismember: ->(val) { val }, smembers: ->(val) { val.is_a?(Array) && !val.empty? }, sscan: ->(val) { val != ['0', []] }, - ttl: ->(val) { val != 0 && val != -2 } + sscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, + ttl: ->(val) { val != 0 && val != -2 }, # ttl returns -2 if the key does not exist. See https://redis.io/commands/ttl/ + zscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? } }.freeze WRITE_COMMANDS = %i[ @@ -134,20 +131,6 @@ module Gitlab end end - ENUMERATOR_CACHE_HIT_VALIDATOR.each_key do |name| - define_method(name) do |*args, **kwargs, &block| - enumerator = if use_primary_and_secondary_stores? - read_command(name, *args, **kwargs) - else - default_store.send(name, *args, **kwargs) - end - - return enumerator if block.nil? - - enumerator.each(&block) - end - end - PIPELINED_COMMANDS.each do |name| define_method(name) do |*args, **kwargs, &block| if use_primary_and_secondary_stores? @@ -186,11 +169,15 @@ module Gitlab end def use_primary_and_secondary_stores? - feature_enabled?("use_primary_and_secondary_stores_for") + feature_table_exists? && + Feature.enabled?("use_primary_and_secondary_stores_for_#{instance_name.underscore}") && # rubocop:disable Cop/FeatureFlagUsage + !same_redis_store? end def use_primary_store_as_default? - feature_enabled?("use_primary_store_as_default_for") + feature_table_exists? && + Feature.enabled?("use_primary_store_as_default_for_#{instance_name.underscore}") && # rubocop:disable Cop/FeatureFlagUsage + !same_redis_store? end def increment_pipelined_command_error_count(command_name) @@ -217,6 +204,14 @@ module Gitlab extra.merge(command_name: command_name, instance_name: instance_name)) end + def default_store + use_primary_store_as_default? ? primary_store : secondary_store + end + + def fallback_store + use_primary_store_as_default? ? secondary_store : primary_store + end + def ping(message = nil) if use_primary_and_secondary_stores? # Both stores have to response success for the ping to be considered success. @@ -231,23 +226,14 @@ module Gitlab private # @return [Boolean] - def feature_enabled?(prefix) - feature_table_exists? && - Feature.enabled?("#{prefix}_#{instance_name.underscore}") && # rubocop:disable Cop/FeatureFlagUsage - !same_redis_store? - end - - # @return [Boolean] def feature_table_exists? + # Use table_exists? (which uses ActiveRecord's schema cache) instead of Feature.feature_flags_available? + # as the latter runs a ';' SQL query which causes a connection to be checked out. Feature::FlipperFeature.table_exists? rescue StandardError false end - def default_store - use_primary_store_as_default? ? primary_store : secondary_store - end - def log_method_missing(command_name, *_args) return if SKIP_LOG_METHOD_MISSING_FOR_COMMANDS.include?(command_name) @@ -275,26 +261,26 @@ module Gitlab def read_one_with_fallback(command_name, *args, **kwargs, &block) begin - value = send_command(primary_store, command_name, *args, **kwargs, &block) + value = send_command(default_store, command_name, *args, **kwargs, &block) rescue StandardError => e log_error(e, command_name, multi_store_error_message: FAILED_TO_READ_ERROR_MESSAGE) end - return value if cache_hit?(command_name, value) + return value if block.nil? && cache_hit?(command_name, value) fallback_read(command_name, *args, **kwargs, &block) end def cache_hit?(command, value) - validator = READ_CACHE_HIT_VALIDATOR[command] || ENUMERATOR_CACHE_HIT_VALIDATOR[command] + validator = READ_CACHE_HIT_VALIDATOR[command] return false unless validator !value.nil? && validator.call(value) end def fallback_read(command_name, *args, **kwargs, &block) - value = send_command(secondary_store, command_name, *args, **kwargs, &block) + value = send_command(fallback_store, command_name, *args, **kwargs, &block) if value log_error(ReadFromPrimaryError.new, command_name) diff --git a/lib/gitlab/redis/rate_limiting.rb b/lib/gitlab/redis/rate_limiting.rb index 4ae1d55e4ce..12710bafbea 100644 --- a/lib/gitlab/redis/rate_limiting.rb +++ b/lib/gitlab/redis/rate_limiting.rb @@ -3,13 +3,28 @@ module Gitlab module Redis class RateLimiting < ::Gitlab::Redis::Wrapper - # The data we store on RateLimiting used to be stored on Cache. - def self.config_fallback - Cache - end + class << self + # The data we store on RateLimiting used to be stored on Cache. + def config_fallback + Cache + end + + def cache_store + @cache_store ||= ActiveSupport::Cache::RedisCacheStore.new( + redis: pool, + namespace: Cache::CACHE_NAMESPACE, + error_handler: ::Gitlab::Redis::ERROR_HANDLER + ) + end + + private + + def redis + primary_store = ::Redis.new(::Gitlab::Redis::ClusterRateLimiting.params) + secondary_store = ::Redis.new(params) - def self.cache_store - @cache_store ||= ActiveSupport::Cache::RedisCacheStore.new(redis: pool, namespace: Cache::CACHE_NAMESPACE) + MultiStore.new(primary_store, secondary_store, name.demodulize) + end end end end diff --git a/lib/gitlab/redis/repository_cache.rb b/lib/gitlab/redis/repository_cache.rb index 8bfbfcfea60..6c7bc8c41d5 100644 --- a/lib/gitlab/redis/repository_cache.rb +++ b/lib/gitlab/redis/repository_cache.rb @@ -14,19 +14,10 @@ module Gitlab redis: pool, compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), namespace: Cache::CACHE_NAMESPACE, - # Cache should not grow forever - expires_in: ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 8.hours).to_i + expires_in: Cache.default_ttl_seconds, + error_handler: ::Gitlab::Redis::ERROR_HANDLER ) end - - private - - def redis - primary_store = ::Redis.new(params) - secondary_store = ::Redis.new(config_fallback.params) - - MultiStore.new(primary_store, secondary_store, store_name) - end end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index e5e1e1d4165..c990655769c 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -59,16 +59,11 @@ module Gitlab config_file_path("redis.#{store_name.underscore}.yml"), # The current Redis instance may have been split off from another one - # (e.g. TraceChunks was split off from SharedState). There are - # installations out there where the lowest priority config source - # (resque.yml) contains bogus values. In those cases, config_file_name - # should resolve to the instance we originated from (the - # "config_fallback") rather than resque.yml. + # (e.g. TraceChunks was split off from SharedState). config_fallback&.config_file_name, # Global config sources: - ENV['GITLAB_REDIS_CONFIG_FILE'], - config_file_path('resque.yml') + ENV['GITLAB_REDIS_CONFIG_FILE'] ].compact.first end @@ -199,11 +194,17 @@ module Gitlab def fetch_config redis_yml = read_yaml(self.class.redis_yml_path).fetch(@rails_env, {}) instance_config_yml = read_yaml(self.class.config_file_name)[@rails_env] + resque_yml = read_yaml(self.class.config_file_path('resque.yml'))[@rails_env] [ redis_yml[self.class.store_name.underscore], + # There are installations out there where the lowest priority config source (resque.yml) contains bogus + # values. In those cases, the configuration should be read for the instance we originated from (the + # "config_fallback"), either from its specific config file or from redis.yml, before falling back to + # resque.yml. instance_config_yml, - self.class.config_fallback && redis_yml[self.class.config_fallback.store_name.underscore] + self.class.config_fallback && redis_yml[self.class.config_fallback.store_name.underscore], + resque_yml ].compact.first end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 828cf65fb82..e76056709e9 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -120,7 +120,8 @@ module Gitlab @debian_version_regex ||= %r{ \A(?: (?:([0-9]{1,9}):)? (?# epoch) - ([0-9][0-9a-z\.+~]*-?){1,15} (?# version-revision) + ([0-9][0-9a-z\.+~]*) (?# version) + (-[0-9a-z\.+~]+){0,14} (?# -revision) (?<!-) )\z}xi.freeze end @@ -139,6 +140,10 @@ module Gitlab @debian_component_regex ||= %r{\A#{::Packages::Debian::COMPONENT_REGEX}\z}o.freeze end + def debian_direct_upload_filename_regex + @debian_direct_upload_filename_regex ||= %r{\A.*\.(deb|udeb)\z}o.freeze + end + def helm_channel_regex @helm_channel_regex ||= %r{\A([a-zA-Z0-9](\.|-|_)?){1,255}(?<!\.|-|_)\z}.freeze end @@ -251,24 +256,48 @@ module Gitlab extend self extend Packages - def bulk_import_namespace_path_regex - # This regexp validates the string conforms to rules for a namespace path: - # i.e does not start with a non-alphanueric character except for periods or underscores, + def bulk_import_destination_namespace_path_regex + # This regexp validates the string conforms to rules for a destination_namespace path: + # i.e does not start with a non-alphanumeric character except for periods or underscores, + # contains only alphanumeric characters, forward slashes, periods, and underscores, + # does not end with a period or forward slash, and has a relative path structure + # with no http protocol chars or leading or trailing forward slashes + # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/destination/namespace/path' + # the regex also allows for an empty string ('') to be accepted as this is allowed in + # a bulk_import POST request + @bulk_import_destination_namespace_path_regex ||= %r/((\A\z)|\A([.]?)[^\W](\/?[.]?[0-9a-z][-_]*)+\z)/i + end + + def bulk_import_source_full_path_regex + # This regexp validates the string conforms to rules for a source_full_path path: + # i.e does not start with a non-alphanumeric character except for periods or underscores, # contains only alphanumeric characters, forward slashes, periods, and underscores, # does not end with a period or forward slash, and has a relative path structure # with no http protocol chars or leading or trailing forward slashes # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/source/full/path' - @bulk_import_namespace_path_regex ||= %r/^([.]?)[^\W](\/?[.]?[0-9a-z][-_]*)+$/i + @bulk_import_source_full_path_regex ||= %r/\A([.]?)[^\W](\/?[.]?[0-9a-z][-_]*)+\z/i + end + + def bulk_import_destination_namespace_path_regex_message + "cannot start with a non-alphanumeric character except for periods or underscores, " \ + "can contain only alphanumeric characters, forward slashes, periods, and underscores, " \ + "cannot end with a period or forward slash, and has a relative path structure " \ + "with no http protocol chars or leading or trailing forward slashes" \ end def group_path_regex # This regexp validates the string conforms to rules for a group slug: - # i.e does not start with a non-alphanueric character except for periods or underscores, + # i.e does not start with a non-alphanumeric character except for periods or underscores, # contains only alphanumeric characters, periods, and underscores, - # does not end with a period or forward slash, and has a relative path structure - # with no http protocol chars or leading or trailing forward slashes - # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/source/full/path' - @group_path_regex ||= %r/^[.]?[^\W]([.]?[0-9a-z][-_]*)+$/i + # does not end with a period or forward slash, and has no leading or trailing forward slashes + # eg 'destination-path' or 'destination_pth' not 'example/com/destination/full/path' + @group_path_regex ||= %r/\A[.]?[^\W]([.]?[0-9a-z][-_]*)+\z/i + end + + def group_path_regex_message + "cannot start with a non-alphanumeric character except for periods or underscores, " \ + "can contain only alphanumeric characters, periods, and underscores, " \ + "cannot end with a period or forward slash, and has no leading or trailing forward slashes" \ end def project_name_regex @@ -406,30 +435,70 @@ module Gitlab }x.freeze end + MARKDOWN_CODE_BLOCK_REGEX = %r{ + (?<code> + # Code blocks: + # ``` + # Anything, including `>>>` blocks which are ignored by this filter + # ``` + + ^``` + .+? + \n```\ *$ + ) + }mx.freeze + + MARKDOWN_HTML_BLOCK_REGEX = %r{ + (?<html> + # HTML block: + # <tag> + # Anything, including `>>>` blocks which are ignored by this filter + # </tag> + + ^<[^>]+?>\ *\n + .+? + \n<\/[^>]+?>\ *$ + ) + }mx.freeze + + MARKDOWN_HTML_COMMENT_LINE_REGEX = %r{ + (?<html_comment_line> + # HTML comment line: + # <!-- some commented text --> + + ^<!--\ .*\ -->\ *$ + ) + }mx.freeze + + MARKDOWN_HTML_COMMENT_BLOCK_REGEX = %r{ + (?<html_comment_block> + # HTML comment block: + # <!-- some commented text + # additional text + # --> + + ^<!--.*\n + .+? + \n-->\ *$ + ) + }mx.freeze + def markdown_code_or_html_blocks @markdown_code_or_html_blocks ||= %r{ - (?<code> - # Code blocks: - # ``` - # Anything, including `>>>` blocks which are ignored by this filter - # ``` - - ^``` - .+? - \n```\ *$ - ) + #{MARKDOWN_CODE_BLOCK_REGEX} | - (?<html> - # HTML block: - # <tag> - # Anything, including `>>>` blocks which are ignored by this filter - # </tag> - - ^<[^>]+?>\ *\n - .+? - \n<\/[^>]+?>\ *$ - ) - }mx + #{MARKDOWN_HTML_BLOCK_REGEX} + }mx.freeze + end + + def markdown_code_or_html_comments + @markdown_code_or_html_comments ||= %r{ + #{MARKDOWN_CODE_BLOCK_REGEX} + | + #{MARKDOWN_HTML_COMMENT_LINE_REGEX} + | + #{MARKDOWN_HTML_COMMENT_BLOCK_REGEX} + }mx.freeze end # Based on Jira's project key format @@ -499,15 +568,6 @@ module Gitlab "Must start with a letter, and cannot end with '-' or '_'" end - def saved_reply_name_regex - @saved_reply_name_regex ||= /\A[a-z]([a-z0-9\-_]*[a-z0-9])?\z/.freeze - end - - def saved_reply_name_regex_message - "can contain only lowercase letters, digits, '_' and '-'. " \ - "Must start with a letter, and cannot end with '-' or '_'" - end - # One or more `part`s, separated by separator def sep_by_1(separator, part) %r(#{part} (#{separator} #{part})*)x diff --git a/lib/gitlab/repository_cache.rb b/lib/gitlab/repository_cache.rb index 8de2c2fe772..498eaf92381 100644 --- a/lib/gitlab/repository_cache.rb +++ b/lib/gitlab/repository_cache.rb @@ -50,12 +50,7 @@ module Gitlab end def self.store - if Feature.enabled?(:use_primary_and_secondary_stores_for_repository_cache) || - Feature.enabled?(:use_primary_store_as_default_for_repository_cache) - Gitlab::Redis::RepositoryCache.cache_store - else - Rails.cache - end + Gitlab::Redis::RepositoryCache.cache_store end end end diff --git a/lib/gitlab/repository_hash_cache.rb b/lib/gitlab/repository_hash_cache.rb index ea90a341b1e..1f3c084e194 100644 --- a/lib/gitlab/repository_hash_cache.rb +++ b/lib/gitlab/repository_hash_cache.rb @@ -140,12 +140,7 @@ module Gitlab private def cache - if Feature.enabled?(:use_primary_and_secondary_stores_for_repository_cache) || - Feature.enabled?(:use_primary_store_as_default_for_repository_cache) - Gitlab::Redis::RepositoryCache - else - Gitlab::Redis::Cache - end + Gitlab::Redis::RepositoryCache end def with(&blk) diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index c67ca92af40..838f44c0f9e 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -68,12 +68,7 @@ module Gitlab private def cache - if Feature.enabled?(:use_primary_and_secondary_stores_for_repository_cache) || - Feature.enabled?(:use_primary_store_as_default_for_repository_cache) - Gitlab::Redis::RepositoryCache - else - Gitlab::Redis::Cache - end + Gitlab::Redis::RepositoryCache end def with(&blk) diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index fd9fb8ab7e2..1684ecf6ff6 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -47,22 +47,6 @@ module Gitlab self._includers << klass end - def self.add_helpers(mod) - url_helpers.include mod - url_helpers.extend mod - - GitlabRoutingHelper.include mod - GitlabRoutingHelper.extend mod - - app_url_helpers = Gitlab::Application.routes.named_routes.url_helpers_module - app_url_helpers.include mod - app_url_helpers.extend mod - - _includers.each do |klass| - klass.include mod - end - end - # Returns the URL helpers Module. # # This method caches the output as Rails' "url_helpers" method creates an diff --git a/lib/gitlab/search/found_blob.rb b/lib/gitlab/search/found_blob.rb index 60d3e360984..79d6cfc84a3 100644 --- a/lib/gitlab/search/found_blob.rb +++ b/lib/gitlab/search/found_blob.rb @@ -9,7 +9,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize include BlobActiveModel - attr_reader :project, :content_match, :blob_path, :highlight_line + attr_reader :project, :content_match, :blob_path, :highlight_line, :matched_lines_count PATH_REGEXP = /\A(?<ref>[^:]*):(?<path>[^\x00]*)\x00/.freeze CONTENT_REGEXP = /^(?<ref>[^:]*):(?<path>[^\x00]*)\x00(?<startline>\d+)\x00/.freeze @@ -25,6 +25,7 @@ module Gitlab @binary_path = opts.fetch(:path, nil) @binary_basename = opts.fetch(:basename, nil) @ref = opts.fetch(:ref, nil) + @matched_lines_count = opts.fetch(:matched_lines_count, nil) @startline = opts.fetch(:startline, nil) @highlight_line = opts.fetch(:highlight_line, nil) @binary_data = opts.fetch(:data, nil) diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 4f7cd340461..b6e2209b475 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -251,16 +251,8 @@ module Gitlab !scheduled? && options[:if_deduplicated] == :reschedule_once end - def with_redis - if Feature.enabled?(:use_primary_and_secondary_stores_for_duplicate_jobs) || - Feature.enabled?(:use_primary_store_as_default_for_duplicate_jobs) - # TODO: Swap for Gitlab::Redis::SharedState after store transition - # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/923 - Gitlab::Redis::DuplicateJobs.with { |redis| yield redis } - else - # Keep the old behavior intact if neither feature flag is turned on - Sidekiq.redis { |redis| yield redis } # rubocop:disable Cop/SidekiqRedisCall - end + def with_redis(&block) + Sidekiq.redis(&block) # rubocop:disable Cop/SidekiqRedisCall end end end diff --git a/lib/gitlab/slash_commands/application_help.rb b/lib/gitlab/slash_commands/application_help.rb index 94abc8b4508..8d747cab2a1 100644 --- a/lib/gitlab/slash_commands/application_help.rb +++ b/lib/gitlab/slash_commands/application_help.rb @@ -17,7 +17,7 @@ module Gitlab private def trigger - "#{params[:command]} [project name or alias]" + params[:command].to_s end def commands diff --git a/lib/gitlab/slash_commands/issue_close.rb b/lib/gitlab/slash_commands/issue_close.rb index 5d33f2fe62d..885c08ce9d5 100644 --- a/lib/gitlab/slash_commands/issue_close.rb +++ b/lib/gitlab/slash_commands/issue_close.rb @@ -29,7 +29,7 @@ module Gitlab private def close_issue(issue:) - ::Issues::CloseService.new(project: project, current_user: current_user).execute(issue) + ::Issues::CloseService.new(container: project, current_user: current_user).execute(issue) end def presenter(issue) diff --git a/lib/gitlab/slash_commands/issue_move.rb b/lib/gitlab/slash_commands/issue_move.rb index 9f10da247d7..e42cdd0d433 100644 --- a/lib/gitlab/slash_commands/issue_move.rb +++ b/lib/gitlab/slash_commands/issue_move.rb @@ -29,7 +29,7 @@ module Gitlab return Gitlab::SlashCommands::Presenters::Access.new.not_found end - new_issue = ::Issues::MoveService.new(project: project, current_user: current_user) + new_issue = ::Issues::MoveService.new(container: project, current_user: current_user) .execute(old_issue, target_project) presenter(new_issue).present(old_issue) diff --git a/lib/gitlab/slash_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb index 1527fd263e0..508526ac500 100644 --- a/lib/gitlab/slash_commands/issue_new.rb +++ b/lib/gitlab/slash_commands/issue_new.rb @@ -37,7 +37,7 @@ module Gitlab private def create_issue(title:, description:) - ::Issues::CreateService.new(project: project, current_user: current_user, params: { title: title, description: description }, spam_params: nil).execute + ::Issues::CreateService.new(container: project, current_user: current_user, params: { title: title, description: description }, spam_params: nil).execute end def presenter(issue) diff --git a/lib/gitlab/slash_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb index 61b36308d20..395f7aab050 100644 --- a/lib/gitlab/slash_commands/presenters/help.rb +++ b/lib/gitlab/slash_commands/presenters/help.rb @@ -26,7 +26,7 @@ module Gitlab MESSAGE end - if text.start_with?('help') + if text && text.start_with?('help') <<~MESSAGE #{full_commands_message(trigger)} @@ -69,9 +69,9 @@ module Gitlab list = @commands .map do |command| if command < Gitlab::SlashCommands::IncidentManagement::IncidentCommand - "#{@params[:command]} #{command.help_message}" - else "#{trigger} #{command.help_message}" + else + "#{trigger} [project name or alias] #{command.help_message}" end end .join("\n") diff --git a/lib/gitlab/slug/path.rb b/lib/gitlab/slug/path.rb new file mode 100644 index 00000000000..434f36829a6 --- /dev/null +++ b/lib/gitlab/slug/path.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Slug + class Path + LEADING_DASHES = /\A-+/.freeze + # Eextract local email part if given an email. Will remove @ sign and everything following it. + EXTRACT_LOCAL_EMAIL_PART = /@.*\z/.freeze + FORBIDDEN_CHARACTERS = /[^a-zA-Z0-9_\-.]/.freeze + PATH_TRAILING_VIOLATIONS = %w[.git .atom .].freeze + DEFAULT_SLUG = 'blank' + + def initialize(input) + @input = input.dup.to_s + end + + def generate + slug = input.gsub(EXTRACT_LOCAL_EMAIL_PART, "") + slug = slug.gsub(FORBIDDEN_CHARACTERS, "") + + # Remove trailing violations ('.atom', '.git', or '.') + loop do + orig = slug + PATH_TRAILING_VIOLATIONS.each { |extension| slug = slug.chomp extension } + break if orig == slug + end + slug = slug.sub(LEADING_DASHES, "") + + # If all characters were of forbidden nature and filtered out we use this + # fallback to avoid empty paths + slug = DEFAULT_SLUG if slug.blank? + + slug + end + + alias_method :to_s, :generate + + private + + attr_reader :input + end + end +end diff --git a/lib/gitlab/sql/set_operator.rb b/lib/gitlab/sql/set_operator.rb index 18275da3ef0..8b4c43786f7 100644 --- a/lib/gitlab/sql/set_operator.rb +++ b/lib/gitlab/sql/set_operator.rb @@ -35,9 +35,12 @@ module Gitlab # By using "unprepared_statements" we remove the usage of placeholders # (thus fixing this problem), at a slight performance cost. fragments = ApplicationRecord.connection.unprepared_statement do - relations.map do |rel| - remove_order ? rel.reorder(nil).to_sql : rel.to_sql - end.reject(&:blank?) + relations.filter_map do |rel| + next if rel.is_a?(ActiveRecord::NullRelation) + + sql = remove_order ? rel.reorder(nil).to_sql : rel.to_sql + sql.presence + end end if fragments.any? diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb index 87861b61119..7094db14c5d 100644 --- a/lib/gitlab/time_tracking_formatter.rb +++ b/lib/gitlab/time_tracking_formatter.rb @@ -7,7 +7,7 @@ module Gitlab # We may want to configure it through project settings in a future version. CUSTOM_DAY_AND_MONTH_LENGTH = { hours_per_day: 8, days_per_month: 20 }.freeze - def parse(string) + def parse(string, keep_zero: false) negative_time = string.start_with?('-') string = string.delete_prefix('-') @@ -15,7 +15,7 @@ module Gitlab begin ChronicDuration.parse( string, - CUSTOM_DAY_AND_MONTH_LENGTH.merge(default_unit: 'hours')) + CUSTOM_DAY_AND_MONTH_LENGTH.merge(default_unit: 'hours', keep_zero: keep_zero)) rescue StandardError nil end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 00e609511f2..b620e9b4560 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -121,8 +121,8 @@ module Gitlab end rescue SocketError # If the dns rebinding protection is not enabled or the domain - # is allowed we avoid the dns rebinding checks - return if domain_allowed?(uri) || !dns_rebind_protection + # is allowed, or HTTP_PROXY is set we avoid the dns rebinding checks + return if domain_allowed?(uri) || !dns_rebind_protection || Gitlab.http_proxy_env? # In the test suite we use a lot of mocked urls that are either invalid or # don't exist. In order to avoid modifying a ton of tests and factories diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ci_internal_pipelines_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ci_internal_pipelines_metric.rb new file mode 100644 index 00000000000..44a2d739a4a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_ci_internal_pipelines_metric.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountCiInternalPipelinesMetric < DatabaseMetric + operation :count + + relation do + ::Ci::Pipeline.internal + end + + def value + return FALLBACK if Gitlab.com? + + super + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric.rb new file mode 100644 index 00000000000..e6093691f48 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountIssuesCreatedManuallyFromAlertsMetric < DatabaseMetric + operation :count + + start { Issue.minimum(:id) } + finish { Issue.maximum(:id) } + + cache_start_and_finish_as :issue + + relation do + Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot) + end + + def value + return FALLBACK if Gitlab.com? + + super + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ml_candidates_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ml_candidates_metric.rb new file mode 100644 index 00000000000..e6e547c155c --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_ml_candidates_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountMlCandidatesMetric < DatabaseMetric + operation :count + + relation { Ml::Candidate } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ml_experiments_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ml_experiments_metric.rb new file mode 100644 index 00000000000..3e589f28df1 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_ml_experiments_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountMlExperimentsMetric < DatabaseMetric + operation :count + + relation { Ml::Experiment } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_projects_with_ml_candidates_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_ml_candidates_metric.rb new file mode 100644 index 00000000000..bf1f5534a70 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_ml_candidates_metric.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountProjectsWithMlCandidatesMetric < DatabaseMetric + operation :distinct_count, column: 'ml_experiments.project_id' + + relation do + Ml::Experiment.where('EXISTS (?)', + Ml::Candidate.where("\"ml_experiments\".\"id\" = \"ml_candidates\".\"experiment_id\"").select(1)) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_projects_with_ml_experiments_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_ml_experiments_metric.rb new file mode 100644 index 00000000000..d575163b6b5 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_ml_experiments_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountProjectsWithMlExperimentsMetric < DatabaseMetric + operation :distinct_count, column: :project_id + + relation { Ml::Experiment } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_projects_with_monitor_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_monitor_enabled_metric.rb new file mode 100644 index 00000000000..2a560233390 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_monitor_enabled_metric.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountProjectsWithMonitorEnabledMetric < DatabaseMetric + operation :count + + metric_options do + { + batch_size: 10_000 + } + end + + relation { ProjectFeature.where.not(monitor_access_level: ProjectFeature::DISABLED) } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_users_with_ml_candidates_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_users_with_ml_candidates_metric.rb new file mode 100644 index 00000000000..0ec7173e096 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_users_with_ml_candidates_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountUsersWithMlCandidatesMetric < DatabaseMetric + operation :distinct_count, column: :user_id + + relation { Ml::Candidate } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric.rb new file mode 100644 index 00000000000..ab9c6f87023 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class IncomingEmailEncryptedSecretsEnabledMetric < GenericMetric + value do + Gitlab::IncomingEmail.encrypted_secrets.active? + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/jira_active_integrations_metric.rb b/lib/gitlab/usage/metrics/instrumentations/jira_active_integrations_metric.rb new file mode 100644 index 00000000000..13af3937f43 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/jira_active_integrations_metric.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class JiraActiveIntegrationsMetric < DatabaseMetric + operation :count + + def initialize(metric_definition) + super + + deployment_type = options[:deployment_type] + + return if deployment_type.in?(allowed_types) + + raise ArgumentError, "deployment_type '#{deployment_type}' must be one of: #{allowed_types.join(', ')}" + end + + relation do |options| + ::Integrations::Jira + .active + .joins(:jira_tracker_data) + .where(jira_tracker_data: { deployment_type: options[:deployment_type] }) + end + + private + + def allowed_types + Integrations::JiraTrackerData.deployment_types.keys + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric.rb new file mode 100644 index 00000000000..4332043de8a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class ServiceDeskEmailEncryptedSecretsEnabledMetric < GenericMetric + value do + Gitlab::ServiceDeskEmail.encrypted_secrets.active? + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index c105288fff0..53794854bd0 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -65,17 +65,10 @@ module Gitlab # rubocop: disable Metrics/AbcSize # rubocop: disable CodeReuse/ActiveRecord def system_usage_data - issues_created_manually_from_alerts = if Gitlab.com? - FALLBACK - else - count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)) - end - { counts: { assignee_lists: count(List.assignee), ci_builds: count(::Ci::Build), - ci_internal_pipelines: Gitlab.com? ? FALLBACK : count(::Ci::Pipeline.internal), ci_external_pipelines: count(::Ci::Pipeline.external), ci_pipeline_config_auto_devops: count(::Ci::Pipeline.auto_devops_source), ci_pipeline_config_repository: count(::Ci::Pipeline.repository_source), @@ -116,8 +109,6 @@ module Gitlab issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id), issues_with_embedded_grafana_charts_approx: grafana_embed_usage_data, issues_created_from_alerts: total_alert_issues, - issues_created_gitlab_alerts: issues_created_manually_from_alerts, - issues_created_manually_from_alerts: issues_created_manually_from_alerts, incident_issues: count(::Issue.incident, start: minimum_id(Issue), finish: maximum_id(Issue)), alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)), keys: count(Key), @@ -250,8 +241,8 @@ module Gitlab prometheus_enabled: alt_usage_data(fallback: nil) { Gitlab::Prometheus::Internal.prometheus_enabled? }, prometheus_metrics_enabled: alt_usage_data(fallback: nil) { Gitlab::Metrics.prometheus_metrics_enabled? }, reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::IncomingEmail.enabled? }, + web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { false }, signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? }, - web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? }, grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? }, gitpod_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gitpod_enabled? } } @@ -598,8 +589,7 @@ module Gitlab { action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(**date_range) }, action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(**date_range) }, - action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(**date_range) }, - action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(**date_range) } + action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(**date_range) } } end diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index eb040e9e819..7f6d67e01c7 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -14,14 +14,11 @@ module Gitlab::UsageDataCounters Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: project.id) namespace = project.namespace - if Feature.enabled?(:route_hll_to_snowplow, namespace) - context = Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, - event: event_name).to_context - label = 'redis_hll_counters.ci_templates.ci_templates_total_unique_counts_monthly' - Gitlab::Tracking.event(name, 'ci_templates_unique', namespace: namespace, - project: project, context: [context], user: user, - label: label) - end + context = Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, + event: event_name).to_context + label = 'redis_hll_counters.ci_templates.ci_templates_total_unique_counts_monthly' + Gitlab::Tracking.event(name, 'ci_templates_unique', namespace: namespace, + project: project, context: [context], user: user, label: label) end def ci_templates(relative_base = 'lib/gitlab/ci/templates') @@ -42,9 +39,9 @@ module Gitlab::UsageDataCounters expanded_template_name = expand_template_name(template_name) results = [expanded_template_name].tap do |result| template = Gitlab::Template::GitlabCiYmlTemplate.find(template_name.chomp('.gitlab-ci.yml')) - data = YAML.safe_load(template.content, aliases: true) - [data['include']].compact.flatten.each do |ci_include| - if ci_include_template = ci_include['template'] + data = Gitlab::Ci::Config::Yaml.load!(template.content) + [data[:include]].compact.flatten.each do |ci_include| + if ci_include_template = ci_include[:template] result.concat(all_included_templates(ci_include_template)) end end diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb index 0b448f68153..2aebc1b8813 100644 --- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -7,7 +7,6 @@ module Gitlab EDIT_BY_SFE = 'g_edit_by_sfe' EDIT_BY_WEB_IDE = 'g_edit_by_web_ide' EDIT_CATEGORY = 'ide_edit' - EDIT_BY_LIVE_PREVIEW = 'g_edit_by_live_preview' class << self def track_web_ide_edit_action(author:, time: Time.zone.now, project:) @@ -34,15 +33,6 @@ module Gitlab count_unique(EDIT_BY_SNIPPET_EDITOR, date_from, date_to) end - def count_edit_using_editor(date_from:, date_to:) - events = Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category(EDIT_CATEGORY) - count_unique(events, date_from, date_to) - end - - def track_live_preview_edit_action(author:, time: Time.zone.now, project:) - track_unique_action(EDIT_BY_LIVE_PREVIEW, author, time, project) - end - private def track_unique_action(event_name, author, time, project = nil) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 992cec2d174..b809e6c4e42 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -35,7 +35,6 @@ module Gitlab # - name: g_compliance_dashboard # Unique event name # redis_slot: compliance # Optional slot name, if not defined it will use name as a slot, used for totals # category: compliance # Group events in categories - # expiry: 29 # Optional expiration time in days, default value 29 days for daily and 6.weeks for weekly # aggregation: daily # Aggregation level, keys are stored daily or weekly # feature_flag: # The event feature flag # @@ -203,8 +202,6 @@ module Gitlab end def expiry(event) - return event[:expiry].days if event[:expiry].present? - event[:aggregation].to_sym == :daily ? DEFAULT_DAILY_KEY_EXPIRY_LENGTH : DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH end diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index 477fa288874..a59ea36961d 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -34,6 +34,7 @@ module Gitlab ISSUE_COMMENT_ADDED = 'g_project_management_issue_comment_added' ISSUE_COMMENT_EDITED = 'g_project_management_issue_comment_edited' ISSUE_COMMENT_REMOVED = 'g_project_management_issue_comment_removed' + ISSUE_DESIGN_COMMENT_REMOVED = 'g_project_management_issue_design_comments_removed' class << self def track_issue_created_action(author:, project:) @@ -171,6 +172,11 @@ module Gitlab track_unique_action(ISSUE_CLONED, author) end + def track_issue_design_comment_removed_action(author:, project:) + track_snowplow_action(ISSUE_DESIGN_COMMENT_REMOVED, author, project) + track_unique_action(ISSUE_DESIGN_COMMENT_REMOVED, author) + end + private def track_snowplow_action(event_name, author, project) diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index b9f143a3a56..b13e3d631c7 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -595,3 +595,11 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_terraform_module_base + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_terraform_module + category: ci_templates + redis_slot: ci_templates + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index a64b7c4032b..ae15530f0d0 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -3,22 +3,18 @@ - name: g_edit_by_web_ide category: ide_edit redis_slot: edit - expiry: 29 aggregation: daily - name: g_edit_by_sfe category: ide_edit redis_slot: edit - expiry: 29 aggregation: daily - name: g_edit_by_snippet_ide category: ide_edit redis_slot: edit - expiry: 29 aggregation: daily - name: g_edit_by_live_preview category: ide_edit redis_slot: edit - expiry: 29 aggregation: daily - name: i_search_total category: search @@ -216,6 +212,10 @@ category: issues_edit redis_slot: project_management aggregation: daily +- name: g_project_management_issue_design_comments_removed + category: issues_edit + redis_slot: project_management + aggregation: daily - name: g_project_management_issue_time_estimate_changed category: issues_edit redis_slot: project_management @@ -240,6 +240,11 @@ category: issues_edit redis_slot: project_management aggregation: daily +# Runner group +- name: g_runner_fleet_read_jobs_statistics + category: runner + redis_slot: runner + aggregation: weekly # Secrets Management - name: i_snippets_show category: snippets @@ -250,7 +255,7 @@ category: terraform redis_slot: terraform aggregation: weekly -# Pipeline Authoring +# Pipeline Authoring group - name: o_pipeline_authoring_unique_users_committing_ciconfigfile category: pipeline_authoring redis_slot: pipeline_authoring @@ -259,6 +264,10 @@ category: pipeline_authoring redis_slot: pipeline_authoring aggregation: weekly +- name: i_ci_secrets_management_id_tokens_build_created + category: ci_secrets_management + redis_slot: ci_secrets_management + aggregation: weekly # Merge request widgets - name: users_expanding_secure_security_report redis_slot: secure @@ -297,7 +306,6 @@ - name: unique_active_user category: manage aggregation: weekly - expiry: 42 # Environments page - name: users_visiting_environments_pages category: environments diff --git a/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml b/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml new file mode 100644 index 00000000000..e8b14de1769 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml @@ -0,0 +1,22 @@ +--- +- name: i_container_registry_push_tag_user + category: user_container_registry + aggregation: weekly + redis_slot: container_registry +- name: i_container_registry_delete_tag_user + category: user_container_registry + aggregation: weekly + redis_slot: container_registry +- name: i_container_registry_push_repository_user + category: user_container_registry + aggregation: weekly + redis_slot: container_registry +- name: i_container_registry_delete_repository_user + category: user_container_registry + aggregation: weekly + redis_slot: container_registry +- name: i_container_registry_create_repository_user + category: user_container_registry + aggregation: weekly + redis_slot: container_registry +
\ No newline at end of file diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index 10dae35d0bf..c8768164710 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -76,10 +76,10 @@ module Gitlab project: project, namespace: project.namespace, user: user, - property: MR_CREATE_ACTION, - label: 'redis_hll_counters.code_review.i_code_review_create_mr_monthly', + property: MR_USER_CREATE_ACTION, + label: 'redis_hll_counters.code_review.i_code_review_user_create_mr_monthly', context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, - event: MR_CREATE_ACTION).to_context] + event: MR_USER_CREATE_ACTION).to_context] ) end diff --git a/lib/gitlab/usage_data_counters/web_ide_counter.rb b/lib/gitlab/usage_data_counters/web_ide_counter.rb index f2753c8f215..904729f114f 100644 --- a/lib/gitlab/usage_data_counters/web_ide_counter.rb +++ b/lib/gitlab/usage_data_counters/web_ide_counter.rb @@ -27,18 +27,6 @@ module Gitlab count('pipelines') end - def increment_previews_count - return unless Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? - - count('previews') - end - - def increment_previews_success_count - return unless Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? - - count('previews_success') - end - private def redis_key(event) diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index 0d15475eebb..3a163e5dde9 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -91,6 +91,14 @@ module Gitlab count(Users::InProductMarketingEmail.where(track: track, series: series).where.not(cta_clicked_at: nil)) end # rubocop: enable CodeReuse/ActiveRecord + + def stage_manage_events(time_period) + # rubocop: disable CodeReuse/ActiveRecord + # rubocop: disable UsageData/LargeTable + estimate_batch_distinct_count(::Event.where(time_period), :author_id) + # rubocop: enable UsageData/LargeTable + # rubocop: enable CodeReuse/ActiveRecord + end end end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 8bd4cd2401d..b92e7dbb725 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -83,7 +83,11 @@ module Gitlab # Append path to host, making sure there's one single / in between def append_path(host, path) - "#{host.to_s.sub(%r{\/+$}, '')}/#{path.to_s.sub(%r{^\/+}, '')}" + "#{host.to_s.sub(%r{\/+$}, '')}/#{remove_leading_slashes(path)}" + end + + def remove_leading_slashes(str) + str.to_s.sub(%r{^/+}, '') end # A slugified version of the string, suitable for inclusion in URLs and diff --git a/lib/gitlab/utils/delegator_override.rb b/lib/gitlab/utils/delegator_override.rb index 15ba29d3916..446419378f8 100644 --- a/lib/gitlab/utils/delegator_override.rb +++ b/lib/gitlab/utils/delegator_override.rb @@ -6,7 +6,7 @@ module Gitlab # accidentally override important logic on the fabricated object. module DelegatorOverride def delegator_target(target_class) - return unless ENV['STATIC_VERIFICATION'] + return unless Gitlab::Environment.static_verification? unless self < ::SimpleDelegator raise ArgumentError, "'#{self}' is not a subclass of 'SimpleDelegator' class." @@ -16,14 +16,14 @@ module Gitlab end def delegator_override(*names) - return unless ENV['STATIC_VERIFICATION'] - raise TypeError unless names.all? { |n| n.is_a?(Symbol) } + return unless Gitlab::Environment.static_verification? + raise TypeError unless names.all?(Symbol) DelegatorOverride.validator(self).add_allowlist(names) end def delegator_override_with(mod) - return unless ENV['STATIC_VERIFICATION'] + return unless Gitlab::Environment.static_verification? raise TypeError unless mod.is_a?(Module) DelegatorOverride.validator(self).add_allowlist(mod.instance_methods) diff --git a/lib/gitlab/utils/email.rb b/lib/gitlab/utils/email.rb new file mode 100644 index 00000000000..c65d7165263 --- /dev/null +++ b/lib/gitlab/utils/email.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Utils + module Email + extend self + + # Replaces most visible characters with * to obfuscate an email address + # deform adds a fix number of * to ensure the address cannot be guessed. Also obfuscates TLD with ** + def obfuscated_email(email, deform: false) + regex = ::Gitlab::UntrustedRegexp.new('^(..?)(.*)(@.?)(.*)(\..+)$') + match = regex.match(email) + return email unless match + + if deform + # Ensure we can show two characters for the username, even if the username has + # only one character. Boring solution is to just duplicate the character. + email_start = match[1] + email_start += email_start if email_start.length == 1 + + email_start + '*' * 5 + match[3] + '*' * 5 + "#{match[5][0..1]}**" + else + match[1] + '*' * (match[2] || '').length + match[3] + '*' * (match[4] || '').length + match[5] + end + end + end + end +end diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index f83ebba7c3f..7f43e25e50d 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_dependency 'gitlab/utils' +require_dependency 'gitlab/environment' module Gitlab module Utils @@ -117,7 +118,7 @@ module Gitlab # This would make sure we're overriding something. See: # https://gitlab.com/gitlab-org/gitlab/issues/1819 def override(method_name) - return unless ENV['STATIC_VERIFICATION'] + return unless Gitlab::Environment.static_verification? Override.extensions[self] ||= Extension.new(self) Override.extensions[self].add_method_name(method_name) @@ -126,7 +127,7 @@ module Gitlab def method_added(method_name) super - return unless ENV['STATIC_VERIFICATION'] + return unless Gitlab::Environment.static_verification? return unless Override.extensions[self]&.verify_override?(method_name) method_arity = instance_method(method_name).arity @@ -163,7 +164,7 @@ module Gitlab end def queue_verification(base, verify: false) - return unless ENV['STATIC_VERIFICATION'] + return unless Gitlab::Environment.static_verification? # We could check for Class in `override` # This could be `nil` if `override` was never called. diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb index 2a57ca9ae02..cb7f0a18a88 100644 --- a/lib/gitlab/view/presenter/base.rb +++ b/lib/gitlab/view/presenter/base.rb @@ -59,9 +59,7 @@ module Gitlab end def presents(*target_classes, as: nil) - if target_classes.any? { |k| k.is_a?(Symbol) } - raise ArgumentError, "Unsupported target class type: #{target_classes}." - end + raise ArgumentError, "Unsupported target class type: #{target_classes}." if target_classes.any?(Symbol) if self < ::Gitlab::View::Presenter::Delegated target_classes.each { |k| delegator_target(k) } |