diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-12-20 13:37:47 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-12-20 13:37:47 +0000 |
| commit | aee0a117a889461ce8ced6fcf73207fe017f1d99 (patch) | |
| tree | 891d9ef189227a8445d83f35c1b0fc99573f4380 /lib/gitlab | |
| parent | 8d46af3258650d305f53b819eabf7ab18d22f59e (diff) | |
| download | gitlab-ce-aee0a117a889461ce8ced6fcf73207fe017f1d99.tar.gz | |
Add latest changes from gitlab-org/gitlab@14-6-stable-eev14.6.0-rc42
Diffstat (limited to 'lib/gitlab')
285 files changed, 2947 insertions, 3536 deletions
diff --git a/lib/gitlab/abuse.rb b/lib/gitlab/abuse.rb new file mode 100644 index 00000000000..cc95d3c1e0c --- /dev/null +++ b/lib/gitlab/abuse.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Abuse + CONFIDENCE_LEVELS = { + certain: 1.0, + likely: 0.8, + uncertain: 0.5, + unknown: 0.0 + }.freeze + + class << self + def confidence(rating) + CONFIDENCE_LEVELS.fetch(rating.to_sym) + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb index c8b11ecb4a8..2441b7a7497 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb @@ -52,3 +52,5 @@ module Gitlab end end end + +Gitlab::Analytics::CycleAnalytics::Aggregated::DataCollector.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::Aggregated::DataCollector') diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/median.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/median.rb index 181ee20948b..bf8f9f4dcc7 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/median.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/median.rb @@ -15,7 +15,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def seconds - @query = @query.select(median_duration_in_seconds.as('median')).reorder(nil) + @query = @query.select(duration_in_seconds(percentile_cont).as('median')).reorder(nil) result = @query.take || {} result['median'] || nil diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb index 7dce757cdc8..1742d396c10 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb @@ -36,7 +36,7 @@ module Gitlab def serialized_records strong_memoize(:serialized_records) do - records = ordered_and_limited_query.select(stage_event_model.arel_table[Arel.star], duration.as('total_time')) + records = ordered_and_limited_query.select(stage_event_model.arel_table[Arel.star], duration_in_seconds.as('total_time')) yield records if block_given? issuables_and_records = load_issuables(records) diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb index f23d1832df9..b00925495f2 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb @@ -27,13 +27,13 @@ module Gitlab end end - def median_duration_in_seconds - Arel::Nodes::Extract.new(percentile_cont, :epoch) - end - def in_progress? params[:end_event_filter] == :in_progress end + + def duration_in_seconds(duration_expression = duration) + Arel::Nodes::Extract.new(duration_expression, :epoch) + end end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb index 945cecfcf8c..e99ad42b0b2 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb @@ -19,7 +19,7 @@ module Gitlab raise NotImplementedError end - def markdown_description + def html_description(options = {}) self.class.name end diff --git a/lib/gitlab/anonymous_session.rb b/lib/gitlab/anonymous_session.rb index 911825eef3a..e58240e16b4 100644 --- a/lib/gitlab/anonymous_session.rb +++ b/lib/gitlab/anonymous_session.rb @@ -2,27 +2,29 @@ module Gitlab class AnonymousSession + include ::Gitlab::Redis::SessionsStoreHelper + def initialize(remote_ip) @remote_ip = remote_ip end def count_session_ip - Gitlab::Redis::SharedState.with do |redis| - redis.pipelined do - redis.incr(session_lookup_name) - redis.expire(session_lookup_name, 24.hours) + redis_store_class.with do |redis| + redis.pipelined do |pipeline| + pipeline.incr(session_lookup_name) + pipeline.expire(session_lookup_name, 24.hours) end end end def session_count - Gitlab::Redis::SharedState.with do |redis| + redis_store_class.with do |redis| redis.get(session_lookup_name).to_i end end def cleanup_session_per_ip_count - Gitlab::Redis::SharedState.with do |redis| + redis_store_class.with do |redis| redis.del(session_lookup_name) end end @@ -32,7 +34,7 @@ module Gitlab attr_reader :remote_ip def session_lookup_name - @session_lookup_name ||= "#{Gitlab::Redis::SharedState::IP_SESSIONS_LOOKUP_NAMESPACE}:#{remote_ip}" + @session_lookup_name ||= "#{Gitlab::Redis::Sessions::IP_SESSIONS_LOOKUP_NAMESPACE}:#{remote_ip}" end end end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index aa33f56582b..c3415c45b28 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -124,11 +124,8 @@ module Gitlab strong_memoize(:runner_project) do next unless runner&.project_type? - projects = ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/342147') do - runner.projects.take(2) # rubocop: disable CodeReuse/ActiveRecord - end - - projects.first if projects.one? + runner_projects = runner.runner_projects.take(2) # rubocop: disable CodeReuse/ActiveRecord + runner_projects.first.project if runner_projects.one? end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 3db2f1295f9..fb90ad9e275 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -4,12 +4,7 @@ module Gitlab # This class implements a simple rate limiter that can be used to throttle # certain actions. Unlike Rack Attack and Rack::Throttle, which operate at # the middleware level, this can be used at the controller or API level. - # - # @example - # if Gitlab::ApplicationRateLimiter.throttled?(:project_export, scope: [@project, @current_user]) - # flash[:alert] = 'error!' - # redirect_to(edit_project_path(@project), status: :too_many_requests) - # end + # See CheckRateLimit concern for usage. class ApplicationRateLimiter InvalidKeyError = Class.new(StandardError) @@ -47,7 +42,7 @@ module Gitlab project_import: { threshold: -> { application_settings.project_import_limit }, interval: 1.minute }, project_testing_hook: { threshold: 5, interval: 1.minute }, play_pipeline_schedule: { threshold: 1, interval: 1.minute }, - show_raw_controller: { threshold: -> { application_settings.raw_blob_request_limit }, interval: 1.minute }, + raw_blob: { threshold: -> { application_settings.raw_blob_request_limit }, interval: 1.minute }, group_export: { threshold: -> { application_settings.group_export_limit }, interval: 1.minute }, group_download_export: { threshold: -> { application_settings.group_download_export_limit }, interval: 1.minute }, group_import: { threshold: -> { application_settings.group_import_limit }, interval: 1.minute }, @@ -64,45 +59,47 @@ module Gitlab # be throttled. # # @param key [Symbol] Key attribute registered in `.rate_limits` - # @option scope [Array<ActiveRecord>] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project) - # @option threshold [Integer] Optional threshold value to override default one registered in `.rate_limits` - # @option users_allowlist [Array<String>] Optional list of usernames to exclude from the limit. This param will only be functional if Scope includes a current user. + # @param scope [Array<ActiveRecord>] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project) + # @param threshold [Integer] Optional threshold value to override default one registered in `.rate_limits` + # @param users_allowlist [Array<String>] Optional list of usernames to exclude from the limit. This param will only be functional if Scope includes a current user. + # @param peek [Boolean] Optional. When true the key will not be incremented but the current throttled state will be returned. # # @return [Boolean] Whether or not a request should be throttled - def throttled?(key, **options) + def throttled?(key, scope:, threshold: nil, users_allowlist: nil, peek: false) raise InvalidKeyError unless rate_limits[key] - return if scoped_user_in_allowlist?(options) + return false if scoped_user_in_allowlist?(scope, users_allowlist) - threshold_value = options[:threshold] || threshold(key) - threshold_value > 0 && - increment(key, options[:scope]) > threshold_value - end + threshold_value = threshold || threshold(key) - # Increments a cache key that is based on the current time and interval. - # So that when time passes to the next interval, the key changes and the count starts again from 0. - # - # Based on https://github.com/rack/rack-attack/blob/886ba3a18d13c6484cd511a4dc9b76c0d14e5e96/lib/rack/attack/cache.rb#L63-L68 - # - # @param key [Symbol] Key attribute registered in `.rate_limits` - # @option scope [Array<ActiveRecord>] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project) - # - # @return [Integer] incremented value - def increment(key, scope) - interval_value = interval(key) + return false if threshold_value == 0 + interval_value = interval(key) + # `period_key` is based on the current time and interval so when time passes to the next interval + # the key changes and the rate limit count starts again from 0. + # Based on https://github.com/rack/rack-attack/blob/886ba3a18d13c6484cd511a4dc9b76c0d14e5e96/lib/rack/attack/cache.rb#L63-L68 period_key, time_elapsed_in_period = Time.now.to_i.divmod(interval_value) + cache_key = cache_key(key, scope, period_key) - cache_key = "#{action_key(key, scope)}:#{period_key}" - # We add a 1 second buffer to avoid timing issues when we're at the end of a period - expiry = interval_value - time_elapsed_in_period + 1 + value = if peek + read(cache_key) + else + increment(cache_key, interval_value, time_elapsed_in_period) + end - ::Gitlab::Redis::RateLimiting.with do |redis| - redis.pipelined do - redis.incr(cache_key) - redis.expire(cache_key, expiry) - end.first - end + value > threshold_value + end + + # Returns the current rate limited state without incrementing the count. + # + # @param key [Symbol] Key attribute registered in `.rate_limits` + # @param scope [Array<ActiveRecord>] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project) + # @param threshold [Integer] Optional threshold value to override default one registered in `.rate_limits` + # @param users_allowlist [Array<String>] Optional list of usernames to exclude from the limit. This param will only be functional if Scope includes a current user. + # + # @return [Boolean] Whether or not a request is currently throttled + def peek(key, scope:, threshold: nil, users_allowlist: nil) + throttled?(key, peek: true, scope: scope, threshold: threshold, users_allowlist: users_allowlist) end # Logs request using provided logger @@ -150,7 +147,28 @@ module Gitlab action[setting] if action end - def action_key(key, scope) + # Increments the rate limit count and returns the new count value. + def increment(cache_key, interval_value, time_elapsed_in_period) + # We add a 1 second buffer to avoid timing issues when we're at the end of a period + expiry = interval_value - time_elapsed_in_period + 1 + + ::Gitlab::Redis::RateLimiting.with do |redis| + redis.pipelined do + redis.incr(cache_key) + redis.expire(cache_key, expiry) + end.first + end + end + + # Returns the rate limit count. + # Will be 0 if there is no data in the cache. + def read(cache_key) + ::Gitlab::Redis::RateLimiting.with do |redis| + redis.get(cache_key).to_i + end + end + + def cache_key(key, scope, period_key) composed_key = [key, scope].flatten.compact serialized = composed_key.map do |obj| @@ -161,20 +179,20 @@ module Gitlab end end.join(":") - "application_rate_limiter:#{serialized}" + "application_rate_limiter:#{serialized}:#{period_key}" end def application_settings Gitlab::CurrentSettings.current_application_settings end - def scoped_user_in_allowlist?(options) - return unless options[:users_allowlist].present? + def scoped_user_in_allowlist?(scope, users_allowlist) + return unless users_allowlist.present? - scoped_user = [options[:scope]].flatten.find { |s| s.is_a?(User) } + scoped_user = [scope].flatten.find { |s| s.is_a?(User) } return unless scoped_user - scoped_user.username.downcase.in?(options[:users_allowlist]) + scoped_user.username.downcase.in?(users_allowlist) end end diff --git a/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb b/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb index 6dbe6f691f6..3ada3f947ee 100644 --- a/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb +++ b/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb @@ -7,7 +7,7 @@ module Gitlab register_for 'gitlab-html-pipeline' def format(node, lang, opts) - if Feature.enabled?(:use_cmark_renderer) + if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) %(<pre #{lang ? %[lang="#{lang}"] : ''}><code>#{node.content}</code></pre>) else %(<pre><code #{lang ? %[ lang="#{lang}"] : ''}>#{node.content}</code></pre>) diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 0970b92723b..3e982168339 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -189,7 +189,7 @@ module Gitlab user = User.id_in(token.resource_owner_id).first return unless user && can_user_login_with_non_expired_password?(user) - Gitlab::Auth::Result.new(user, nil, :oauth, full_authentication_abilities) + Gitlab::Auth::Result.new(user, nil, :oauth, abilities_for_scopes(token.scopes)) end end end @@ -230,7 +230,7 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def valid_oauth_token?(token) - token && token.accessible? && valid_scoped_token?(token, [:api]) + token && token.accessible? && valid_scoped_token?(token, Doorkeeper.configuration.scopes) end def valid_scoped_token?(token, scopes) diff --git a/lib/gitlab/auth/user_access_denied_reason.rb b/lib/gitlab/auth/user_access_denied_reason.rb index 904759919ae..ff6dc7313bb 100644 --- a/lib/gitlab/auth/user_access_denied_reason.rb +++ b/lib/gitlab/auth/user_access_denied_reason.rb @@ -15,7 +15,7 @@ module Gitlab "Your account is pending approval from your administrator and hence blocked." when :terms_not_accepted "You (#{@user.to_reference}) must accept the Terms of Service in order to perform this action. "\ - "Please access GitLab from a web browser to accept these terms." + "To accept these terms, please access GitLab from a web browser at #{Gitlab.config.gitlab.url}." when :deactivated "Your account has been deactivated by your administrator. "\ "Please log back in from a web browser to reactivate your account at #{Gitlab.config.gitlab.url}" diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index 22b4b685f81..ed58508182a 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -2,11 +2,13 @@ module Gitlab module BackgroundMigration + DEFAULT_TRACKING_DATABASE = Gitlab::Database::MAIN_DATABASE_NAME + def self.coordinator_for_database(database) - JobCoordinator.for_database(database) + JobCoordinator.for_tracking_database(database) end - def self.queue(database: :main) + def self.queue(database: DEFAULT_TRACKING_DATABASE) coordinator_for_database(database).queue end @@ -22,7 +24,7 @@ module Gitlab # steal_class - The name of the class for which to steal jobs. # retry_dead_jobs - Flag to control whether jobs in Sidekiq::RetrySet or Sidekiq::DeadSet are retried. # database - tracking database this migration executes against - def self.steal(steal_class, retry_dead_jobs: false, database: :main, &block) + def self.steal(steal_class, retry_dead_jobs: false, database: DEFAULT_TRACKING_DATABASE, &block) coordinator_for_database(database).steal(steal_class, retry_dead_jobs: retry_dead_jobs, &block) end @@ -35,15 +37,15 @@ module Gitlab # arguments - The arguments to pass to the background migration's "perform" # method. # database - tracking database this migration executes against - def self.perform(class_name, arguments, database: :main) + def self.perform(class_name, arguments, database: DEFAULT_TRACKING_DATABASE) coordinator_for_database(database).perform(class_name, arguments) end - def self.exists?(migration_class, additional_queues = [], database: :main) + def self.exists?(migration_class, additional_queues = [], database: DEFAULT_TRACKING_DATABASE) coordinator_for_database(database).exists?(migration_class, additional_queues) # rubocop:disable CodeReuse/ActiveRecord end - def self.remaining(database: :main) + def self.remaining(database: DEFAULT_TRACKING_DATABASE) coordinator_for_database(database).remaining end end diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb deleted file mode 100644 index 5b9ee8a0ee2..00000000000 --- a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class AddMergeRequestDiffCommitsCount - class MergeRequestDiff < ActiveRecord::Base - self.table_name = 'merge_request_diffs' - end - - def perform(start_id, stop_id) - Gitlab::AppLogger.info("Setting commits_count for merge request diffs: #{start_id} - #{stop_id}") - - update = ' - commits_count = ( - SELECT count(*) - FROM merge_request_diff_commits - WHERE merge_request_diffs.id = merge_request_diff_commits.merge_request_diff_id - )'.squish - - MergeRequestDiff.where(id: start_id..stop_id).where(commits_count: nil).update_all(update) - end - end - end -end diff --git a/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule.rb b/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule.rb deleted file mode 100644 index 2148e96f6b4..00000000000 --- a/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule.rb +++ /dev/null @@ -1,73 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Compare all current rules to project rules - class AddModifiedToApprovalMergeRequestRule - # Stubbed class to access the Group table - class Group < ActiveRecord::Base - self.table_name = 'namespaces' - self.inheritance_column = :_type_disabled - end - - # Stubbed class to access the ApprovalMergeRequestRule table - class ApprovalMergeRequestRule < ActiveRecord::Base - self.table_name = 'approval_merge_request_rules' - - has_one :approval_merge_request_rule_source, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalMergeRequestRuleSource' - has_one :approval_project_rule, through: :approval_merge_request_rule_source - has_and_belongs_to_many :groups, - class_name: 'AddModifiedToApprovalMergeRequestRule::Group', join_table: "#{self.table_name}_groups" - end - - # Stubbed class to access the ApprovalProjectRule table - class ApprovalProjectRule < ActiveRecord::Base - self.table_name = 'approval_project_rules' - - has_many :approval_merge_request_rule_sources, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalMergeRequestRuleSource' - has_and_belongs_to_many :groups, - class_name: 'AddModifiedToApprovalMergeRequestRule::Group', join_table: "#{self.table_name}_groups" - end - - # Stubbed class to access the ApprovalMergeRequestRuleSource table - class ApprovalMergeRequestRuleSource < ActiveRecord::Base - self.table_name = 'approval_merge_request_rule_sources' - - belongs_to :approval_merge_request_rule, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalMergeRequestRule' - belongs_to :approval_project_rule, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalProjectRule' - end - - def perform(start_id, stop_id) - approval_merge_requests_rules = ApprovalMergeRequestRule - .joins(:groups, :approval_merge_request_rule_source) - .where(id: start_id..stop_id) - .pluck( - 'approval_merge_request_rule_sources.id as ars_id', - 'approval_merge_request_rules_groups.id as amrg_id' - ) - - approval_project_rules = ApprovalProjectRule - .joins(:groups, approval_merge_request_rule_sources: :approval_merge_request_rule) - .where(approval_merge_request_rules: { id: start_id..stop_id }) - .pluck( - 'approval_merge_request_rule_sources.id as ars_id', - 'approval_project_rules_groups.id as apg_id' - ) - - different_names_or_approval_sources = ApprovalMergeRequestRule.joins(:approval_project_rule, :approval_merge_request_rule_source) - .where(id: start_id..stop_id) - .where('approval_merge_request_rules.name != approval_project_rules.name OR ' \ - 'approval_merge_request_rules.approvals_required != approval_project_rules.approvals_required') - .pluck('approval_merge_request_rule_sources.id as ars_id') - - intersected_set = approval_merge_requests_rules.to_set ^ approval_project_rules.to_set - source_ids = intersected_set.collect { |rule| rule[0] }.uniq - - rule_sources = ApprovalMergeRequestRuleSource.where(id: source_ids + different_names_or_approval_sources) - changed_merge_request_rules = ApprovalMergeRequestRule.where(id: rule_sources.select(:approval_merge_request_rule_id)) - - changed_merge_request_rules.update_all(modified_from_project_rule: true) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_deployment_clusters_from_deployments.rb b/lib/gitlab/background_migration/backfill_deployment_clusters_from_deployments.rb deleted file mode 100644 index 9778f360e87..00000000000 --- a/lib/gitlab/background_migration/backfill_deployment_clusters_from_deployments.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfill deployment_clusters for a range of deployments - class BackfillDeploymentClustersFromDeployments - def perform(start_id, end_id) - ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO deployment_clusters (deployment_id, cluster_id) - SELECT deployments.id, deployments.cluster_id - FROM deployments - WHERE deployments.cluster_id IS NOT NULL - AND deployments.id BETWEEN #{start_id} AND #{end_id} - ON CONFLICT DO NOTHING - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests.rb b/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests.rb deleted file mode 100644 index 4fd3b81fda3..00000000000 --- a/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # BackfillEnvironmentIdDeploymentMergeRequests deletes duplicates - # from deployment_merge_requests table and backfills environment_id - class BackfillEnvironmentIdDeploymentMergeRequests - def perform(_start_mr_id, _stop_mr_id) - # no-op - - # Background migration removed due to - # https://gitlab.com/gitlab-org/gitlab/-/issues/217191 - end - - def backfill_range(start_mr_id, stop_mr_id) - start_mr_id = Integer(start_mr_id) - stop_mr_id = Integer(stop_mr_id) - - ActiveRecord::Base.connection.execute(<<~SQL) - DELETE FROM deployment_merge_requests - WHERE (deployment_id, merge_request_id) in ( - SELECT t.deployment_id, t.merge_request_id FROM ( - SELECT mrd.merge_request_id, mrd.deployment_id, ROW_NUMBER() OVER w AS rnum - FROM deployment_merge_requests as mrd - INNER JOIN "deployments" ON "deployments"."id" = "mrd"."deployment_id" - WHERE mrd.merge_request_id BETWEEN #{start_mr_id} AND #{stop_mr_id} - WINDOW w AS ( - PARTITION BY merge_request_id, deployments.environment_id - ORDER BY deployments.id - ) - ) t - WHERE t.rnum > 1 - ); - SQL - - ActiveRecord::Base.connection.execute(<<~SQL) - UPDATE deployment_merge_requests - SET environment_id = deployments.environment_id - FROM deployments - WHERE deployments.id = "deployment_merge_requests".deployment_id - AND "deployment_merge_requests".environment_id IS NULL - AND "deployment_merge_requests".merge_request_id BETWEEN #{start_mr_id} AND #{stop_mr_id} - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb b/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb deleted file mode 100644 index 8a58cf9b302..00000000000 --- a/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfill merge request cleanup schedules of closed/merged merge requests - # without any corresponding records. - class BackfillMergeRequestCleanupSchedules - # Model used for migration added in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46782. - class MergeRequest < ActiveRecord::Base - include EachBatch - - self.table_name = 'merge_requests' - - def self.eligible - where('merge_requests.state_id IN (2, 3)') - end - end - - def perform(start_id, end_id) - eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id) - scheduled_at_column = "COALESCE(metrics.merged_at, COALESCE(metrics.latest_closed_at, merge_requests.updated_at)) + interval '14 days'" - query = - eligible_mrs - .select("merge_requests.id, #{scheduled_at_column}, NOW(), NOW()") - .joins('LEFT JOIN merge_request_metrics metrics ON metrics.merge_request_id = merge_requests.id') - - result = ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO merge_request_cleanup_schedules (merge_request_id, scheduled_at, created_at, updated_at) - #{query.to_sql} - ON CONFLICT (merge_request_id) DO NOTHING; - SQL - - ::Gitlab::BackgroundMigration::Logger.info( - message: 'Backfilled merge_request_cleanup_schedules records', - count: result.cmd_tuples - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_namespace_settings.rb b/lib/gitlab/background_migration/backfill_namespace_settings.rb deleted file mode 100644 index a391d5f4ebe..00000000000 --- a/lib/gitlab/background_migration/backfill_namespace_settings.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfillnamespace_settings for a range of namespaces - class BackfillNamespaceSettings - def perform(start_id, end_id) - ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO namespace_settings (namespace_id, created_at, updated_at) - SELECT namespaces.id, now(), now() - FROM namespaces - WHERE namespaces.id BETWEEN #{start_id} AND #{end_id} - ON CONFLICT (namespace_id) DO NOTHING; - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_project_settings.rb b/lib/gitlab/background_migration/backfill_project_settings.rb deleted file mode 100644 index 7d7f19e1fda..00000000000 --- a/lib/gitlab/background_migration/backfill_project_settings.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfill project_settings for a range of projects - class BackfillProjectSettings - def perform(start_id, end_id) - ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO project_settings (project_id, created_at, updated_at) - SELECT projects.id, now(), now() - FROM projects - WHERE projects.id BETWEEN #{start_id} AND #{end_id} - ON CONFLICT (project_id) DO NOTHING; - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_push_rules_id_in_projects.rb b/lib/gitlab/background_migration/backfill_push_rules_id_in_projects.rb deleted file mode 100644 index 9b9ef70424a..00000000000 --- a/lib/gitlab/background_migration/backfill_push_rules_id_in_projects.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Class that will insert record into project_push_rules - # for each existing push_rule - class BackfillPushRulesIdInProjects - # Temporary AR table for push rules - class ProjectSetting < ActiveRecord::Base - self.table_name = 'project_settings' - end - - def perform(start_id, stop_id) - ProjectSetting.connection.execute(<<~SQL) - UPDATE project_settings ps1 - SET push_rule_id = pr.id - FROM project_settings ps2 - INNER JOIN push_rules pr - ON ps2.project_id = pr.project_id - WHERE pr.is_sample = false - AND pr.id BETWEEN #{start_id} AND #{stop_id} - AND ps1.project_id = ps2.project_id - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/drop_invalid_remediations.rb b/lib/gitlab/background_migration/drop_invalid_remediations.rb new file mode 100644 index 00000000000..f0a0de586f5 --- /dev/null +++ b/lib/gitlab/background_migration/drop_invalid_remediations.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class DropInvalidRemediations + def perform(start_id, stop_id) + end + end + # rubocop: enable Style/Documentation + end +end + +Gitlab::BackgroundMigration::DropInvalidRemediations.prepend_mod_with('Gitlab::BackgroundMigration::DropInvalidRemediations') diff --git a/lib/gitlab/background_migration/drop_invalid_security_findings.rb b/lib/gitlab/background_migration/drop_invalid_security_findings.rb new file mode 100644 index 00000000000..87551bb1b1e --- /dev/null +++ b/lib/gitlab/background_migration/drop_invalid_security_findings.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true +module Gitlab + module BackgroundMigration + # Drop rows from security_findings where the uuid is NULL + class DropInvalidSecurityFindings + # rubocop:disable Style/Documentation + class SecurityFinding < ActiveRecord::Base + include ::EachBatch + self.table_name = 'security_findings' + scope :no_uuid, -> { where(uuid: nil) } + end + # rubocop:enable Style/Documentation + + PAUSE_SECONDS = 0.1 + + def perform(start_id, end_id, sub_batch_size) + ranged_query = SecurityFinding + .where(id: start_id..end_id) + .no_uuid + + ranged_query.each_batch(of: sub_batch_size) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(id), max(id)')).first + + # The query need to be reconstructed because .each_batch modifies the default scope + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510 + SecurityFinding.unscoped + .where(id: first..last) + .no_uuid + .delete_all + + sleep PAUSE_SECONDS + end + + mark_job_as_succeeded(start_id, end_id, sub_batch_size) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_promoted_epics_discussion_ids.rb b/lib/gitlab/background_migration/fix_promoted_epics_discussion_ids.rb deleted file mode 100644 index 1a80ccdee92..00000000000 --- a/lib/gitlab/background_migration/fix_promoted_epics_discussion_ids.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This migration updates discussion ids for epics that were promoted from issue so that the discussion id on epics - # is different from discussion id on issue, which was causing problems when replying to epic discussions as it would - # identify the discussion as related to an issue and complaint about missing project_id - class FixPromotedEpicsDiscussionIds - # notes model to iterate through the notes to be updated - class Note < ActiveRecord::Base - self.table_name = 'notes' - self.inheritance_column = :_type_disabled - end - - def perform(discussion_ids) - Note.where(noteable_type: 'Epic') - .where(discussion_id: discussion_ids) - .update_all("discussion_id=MD5(discussion_id)||substring(discussion_id from 1 for 8)") - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_user_namespace_names.rb b/lib/gitlab/background_migration/fix_user_namespace_names.rb deleted file mode 100644 index cd5b4ab103d..00000000000 --- a/lib/gitlab/background_migration/fix_user_namespace_names.rb +++ /dev/null @@ -1,68 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This migration fixes the namespaces.name for all user-namespaces that have names - # that aren't equal to the users name. - # Then it uses the updated names of the namespaces to update the associated routes - # For more info see https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/23272 - class FixUserNamespaceNames - def perform(from_id, to_id) - fix_namespace_names(from_id, to_id) - fix_namespace_route_names(from_id, to_id) - end - - def fix_namespace_names(from_id, to_id) - ActiveRecord::Base.connection.execute <<~UPDATE_NAMESPACES - WITH namespaces_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - SELECT - namespaces.id, - users.name AS correct_name - FROM - namespaces - INNER JOIN users ON namespaces.owner_id = users.id - WHERE - namespaces.type IS NULL - AND namespaces.id BETWEEN #{from_id} AND #{to_id} - AND namespaces.name != users.name - ) - UPDATE - namespaces - SET - name = correct_name - FROM - namespaces_to_update - WHERE - namespaces.id = namespaces_to_update.id - UPDATE_NAMESPACES - end - - def fix_namespace_route_names(from_id, to_id) - ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE - WITH routes_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - SELECT - routes.id, - users.name AS correct_name - FROM - routes - INNER JOIN namespaces ON routes.source_id = namespaces.id - INNER JOIN users ON namespaces.owner_id = users.id - WHERE - namespaces.type IS NULL - AND routes.source_type = 'Namespace' - AND namespaces.id BETWEEN #{from_id} AND #{to_id} - AND (routes.name != users.name OR routes.name IS NULL) - ) - UPDATE - routes - SET - name = correct_name - FROM - routes_to_update - WHERE - routes_to_update.id = routes.id - ROUTES_UPDATE - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_user_project_route_names.rb b/lib/gitlab/background_migration/fix_user_project_route_names.rb deleted file mode 100644 index e534f2449aa..00000000000 --- a/lib/gitlab/background_migration/fix_user_project_route_names.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This migration fixes the routes.name for all user-projects that have names - # that don't start with the users name. - # For more info see https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/23272 - class FixUserProjectRouteNames - def perform(from_id, to_id) - ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE - WITH routes_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - SELECT - routes.id, - users.name || ' / ' || projects.name AS correct_name - FROM - routes - INNER JOIN projects ON routes.source_id = projects.id - INNER JOIN namespaces ON projects.namespace_id = namespaces.id - INNER JOIN users ON namespaces.owner_id = users.id - WHERE - routes.source_type = 'Project' - AND routes.id BETWEEN #{from_id} AND #{to_id} - AND namespaces.type IS NULL - AND (routes.name NOT LIKE users.name || '%' OR routes.name IS NULL) - ) - UPDATE - routes - SET - name = routes_to_update.correct_name - FROM - routes_to_update - WHERE - routes_to_update.id = routes.id - ROUTES_UPDATE - end - end - end -end diff --git a/lib/gitlab/background_migration/job_coordinator.rb b/lib/gitlab/background_migration/job_coordinator.rb index 1c8819eaa62..cfbe7167677 100644 --- a/lib/gitlab/background_migration/job_coordinator.rb +++ b/lib/gitlab/background_migration/job_coordinator.rb @@ -8,24 +8,33 @@ module Gitlab # convention of how the queues and worker classes are setup for each database. # # Also provides a database connection to the correct tracking database. - class JobCoordinator - VALID_DATABASES = %i[main].freeze - WORKER_CLASS_NAME = 'BackgroundMigrationWorker' - - def self.for_database(database) - database = database.to_sym + class JobCoordinator # rubocop:disable Metrics/ClassLength + class << self + def worker_classes + @worker_classes ||= [ + BackgroundMigrationWorker + ].freeze + end - unless VALID_DATABASES.include?(database) - raise ArgumentError, "database must be one of [#{VALID_DATABASES.join(', ')}], got '#{database}'" + def worker_for_tracking_database + @worker_for_tracking_database ||= worker_classes + .index_by(&:tracking_database) + .with_indifferent_access + .freeze end - namespace = database.to_s.capitalize unless database == :main - namespaced_worker_class = [namespace, WORKER_CLASS_NAME].compact.join('::') + def for_tracking_database(tracking_database) + worker_class = worker_for_tracking_database[tracking_database] - new(database, "::#{namespaced_worker_class}".constantize) + if worker_class.nil? + raise ArgumentError, "tracking_database must be one of [#{worker_for_tracking_database.keys.join(', ')}]" + end + + new(worker_class) + end end - attr_reader :database, :worker_class + attr_reader :worker_class def queue @queue ||= worker_class.sidekiq_options['queue'] @@ -118,15 +127,14 @@ module Gitlab private - def initialize(database, worker_class) - @database = database + def initialize(worker_class) @worker_class = worker_class end def connection @connection ||= Gitlab::Database .database_base_models - .fetch(database, Gitlab::Database::PRIMARY_DATABASE_NAME) + .fetch(worker_class.tracking_database, Gitlab::Database::PRIMARY_DATABASE_NAME) .connection end end diff --git a/lib/gitlab/background_migration/link_lfs_objects_projects.rb b/lib/gitlab/background_migration/link_lfs_objects_projects.rb deleted file mode 100644 index 983470c5121..00000000000 --- a/lib/gitlab/background_migration/link_lfs_objects_projects.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Create missing LfsObjectsProject records for forks - class LinkLfsObjectsProjects - # Model specifically used for migration. - class LfsObjectsProject < ActiveRecord::Base - include EachBatch - - self.table_name = 'lfs_objects_projects' - - def self.linkable - where( - <<~SQL - lfs_objects_projects.project_id IN ( - SELECT fork_network_members.forked_from_project_id - FROM fork_network_members - WHERE fork_network_members.forked_from_project_id IS NOT NULL - ) - SQL - ) - end - end - - # Model specifically used for migration. - class ForkNetworkMember < ActiveRecord::Base - include EachBatch - - self.table_name = 'fork_network_members' - - def self.without_lfs_object(lfs_object_id) - where( - <<~SQL - fork_network_members.project_id NOT IN ( - SELECT lop.project_id - FROM lfs_objects_projects lop - WHERE lop.lfs_object_id = #{lfs_object_id} - ) - SQL - ) - end - end - - BATCH_SIZE = 1000 - - def perform(start_id, end_id) - lfs_objects_projects = - Gitlab::BackgroundMigration::LinkLfsObjectsProjects::LfsObjectsProject - .linkable - .where(id: start_id..end_id) - - return if lfs_objects_projects.empty? - - lfs_objects_projects.find_each do |lop| - ForkNetworkMember - .select("#{lop.lfs_object_id}, fork_network_members.project_id, NOW(), NOW()") - .without_lfs_object(lop.lfs_object_id) - .where(forked_from_project_id: lop.project_id) - .each_batch(of: BATCH_SIZE) do |batch, index| - execute <<~SQL - INSERT INTO lfs_objects_projects (lfs_object_id, project_id, created_at, updated_at) - #{batch.to_sql} - SQL - - logger.info(message: "LinkLfsObjectsProjects: created missing LfsObjectsProject records for LfsObject #{lop.lfs_object_id}") - end - end - end - - private - - def execute(sql) - ::ActiveRecord::Base.connection.execute(sql) - end - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb deleted file mode 100644 index 36a339c6b80..00000000000 --- a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class is responsible to update all sha256 fingerprints within the keys table - class MigrateFingerprintSha256WithinKeys - # Temporary AR table for keys - class Key < ActiveRecord::Base - include EachBatch - - self.table_name = 'keys' - self.inheritance_column = :_type_disabled - end - - TEMP_TABLE = 'tmp_fingerprint_sha256_migration' - - def perform(start_id, stop_id) - ActiveRecord::Base.transaction do - execute(<<~SQL) - CREATE TEMPORARY TABLE #{TEMP_TABLE} - (id bigint primary key, fingerprint_sha256 bytea not null) - ON COMMIT DROP - SQL - - fingerprints = [] - Key.where(id: start_id..stop_id, fingerprint_sha256: nil).find_each do |regular_key| - if fingerprint = generate_ssh_public_key(regular_key.key) - bytea = ActiveRecord::Base.connection.escape_bytea(Base64.decode64(fingerprint)) - - fingerprints << { - id: regular_key.id, - fingerprint_sha256: bytea - } - end - end - - ApplicationRecord.legacy_bulk_insert(TEMP_TABLE, fingerprints) # rubocop:disable Gitlab/BulkInsert - - execute("ANALYZE #{TEMP_TABLE}") - - execute(<<~SQL) - UPDATE keys - SET fingerprint_sha256 = t.fingerprint_sha256 - FROM #{TEMP_TABLE} t - WHERE keys.id = t.id - SQL - end - end - - private - - def generate_ssh_public_key(regular_key) - Gitlab::SSHPublicKey.new(regular_key).fingerprint("SHA256")&.gsub("SHA256:", "") - end - - def execute(query) - ActiveRecord::Base.connection.execute(query) - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_pages_metadata.rb b/lib/gitlab/background_migration/migrate_pages_metadata.rb deleted file mode 100644 index 68fd0c17d29..00000000000 --- a/lib/gitlab/background_migration/migrate_pages_metadata.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Class that will insert record into project_pages_metadata - # for each existing project - class MigratePagesMetadata - def perform(start_id, stop_id) - perform_on_relation(Project.where(id: start_id..stop_id)) - end - - def perform_on_relation(relation) - successful_pages_deploy = <<~SQL - SELECT TRUE - FROM ci_builds - WHERE ci_builds.type = 'GenericCommitStatus' - AND ci_builds.status = 'success' - AND ci_builds.stage = 'deploy' - AND ci_builds.name = 'pages:deploy' - AND ci_builds.project_id = projects.id - LIMIT 1 - SQL - - select_from = relation - .select("projects.id", "COALESCE((#{successful_pages_deploy}), FALSE)") - .to_sql - - ActiveRecord::Base.connection_pool.with_connection do |connection| - connection.execute <<~SQL - INSERT INTO project_pages_metadata (project_id, deployed) - #{select_from} - ON CONFLICT (project_id) DO NOTHING - SQL - end - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_to_hashed_storage.rb b/lib/gitlab/background_migration/migrate_to_hashed_storage.rb deleted file mode 100644 index 4054db4fb87..00000000000 --- a/lib/gitlab/background_migration/migrate_to_hashed_storage.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Background migration to move any legacy project to Hashed Storage - class MigrateToHashedStorage - def perform - batch_size = helper.batch_size - legacy_projects_count = Project.with_unmigrated_storage.count - - if storage_migrator.rollback_pending? - logger.warn( - migrator: 'MigrateToHashedStorage', - message: 'Aborting an storage rollback operation currently in progress' - ) - - storage_migrator.abort_rollback! - end - - if legacy_projects_count == 0 - logger.info( - migrator: 'MigrateToHashedStorage', - message: 'There are no projects requiring migration to Hashed Storage' - ) - - return - end - - logger.info( - migrator: 'MigrateToHashedStorage', - message: "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}" - ) - - helper.project_id_batches_migration do |start, finish| - storage_migrator.bulk_schedule_migration(start: start, finish: finish) - - logger.info( - migrator: 'MigrateToHashedStorage', - message: "Enqueuing migration of projects in batches of #{batch_size} from ID=#{start} to ID=#{finish}", - batch_from: start, - batch_to: finish - ) - end - end - - private - - def helper - Gitlab::HashedStorage::RakeHelper - end - - def storage_migrator - @storage_migrator ||= Gitlab::HashedStorage::Migrator.new - end - - def logger - @logger ||= ::Gitlab::BackgroundMigration::Logger.build - end - end - end -end diff --git a/lib/gitlab/background_migration/move_epic_issues_after_epics.rb b/lib/gitlab/background_migration/move_epic_issues_after_epics.rb deleted file mode 100644 index 174994c7862..00000000000 --- a/lib/gitlab/background_migration/move_epic_issues_after_epics.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class MoveEpicIssuesAfterEpics - def perform(start_id, stop_id) - end - end - end -end - -Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics.prepend_mod_with('Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics') diff --git a/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb b/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb deleted file mode 100644 index 890a43800c9..00000000000 --- a/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This background migration creates any approver rule records according - # to the given merge request IDs range. A _single_ INSERT is issued for the given range. - class PopulateAnyApprovalRuleForMergeRequests - def perform(from_id, to_id) - end - end - end -end - -Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests.prepend_mod_with('Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests') diff --git a/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb b/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb deleted file mode 100644 index ac7ed18ba14..00000000000 --- a/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This background migration creates any approver rule records according - # to the given project IDs range. A _single_ INSERT is issued for the given range. - class PopulateAnyApprovalRuleForProjects - def perform(from_id, to_id) - end - end - end -end - -Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects.prepend_mod_with('Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects') diff --git a/lib/gitlab/background_migration/populate_canonical_emails.rb b/lib/gitlab/background_migration/populate_canonical_emails.rb deleted file mode 100644 index 052e75c5655..00000000000 --- a/lib/gitlab/background_migration/populate_canonical_emails.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Class to populate new rows of UserCanonicalEmail based on existing email addresses - class PopulateCanonicalEmails - def perform(start_id, stop_id) - ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO - user_canonical_emails ( - user_id, - canonical_email, - created_at, - updated_at - ) - SELECT users.id AS user_id, - concat(translate(split_part(split_part(users.email, '@', 1), '+', 1), '.', ''), '@gmail.com') AS canonical_email, - NOW() AS created_at, - NOW() AS updated_at - FROM users - WHERE users.email ILIKE '%@gmail.com' - AND users.id BETWEEN #{start_id} AND #{stop_id} - ON CONFLICT DO NOTHING; - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_dismissed_state_for_vulnerabilities.rb b/lib/gitlab/background_migration/populate_dismissed_state_for_vulnerabilities.rb deleted file mode 100644 index 68c91650d93..00000000000 --- a/lib/gitlab/background_migration/populate_dismissed_state_for_vulnerabilities.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class updates vulnerabilities entities with state dismissed - class PopulateDismissedStateForVulnerabilities - class Vulnerability < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'vulnerabilities' - end - - def perform(*vulnerability_ids) - Vulnerability.where(id: vulnerability_ids).update_all(state: 2) - PopulateMissingVulnerabilityDismissalInformation.new.perform(*vulnerability_ids) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb deleted file mode 100644 index 28ff2070209..00000000000 --- a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb +++ /dev/null @@ -1,64 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class populates missing dismissal information for - # vulnerability entries. - class PopulateHasVulnerabilities - class ProjectSetting < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'project_settings' - - def self.upsert_for(project_ids) - connection.execute(upsert_sql % { project_ids: project_ids.join(', ') }) - end - - def self.upsert_sql - <<~SQL - WITH upsert_data (project_id, has_vulnerabilities, created_at, updated_at) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - SELECT projects.id, true, current_timestamp, current_timestamp FROM projects WHERE projects.id IN (%{project_ids}) - ) - INSERT INTO project_settings - (project_id, has_vulnerabilities, created_at, updated_at) - (SELECT * FROM upsert_data) - ON CONFLICT (project_id) - DO UPDATE SET - has_vulnerabilities = true, - updated_at = EXCLUDED.updated_at - SQL - end - end - - class Vulnerability < ActiveRecord::Base # rubocop:disable Style/Documentation - include EachBatch - - self.table_name = 'vulnerabilities' - end - - def perform(*project_ids) - ProjectSetting.upsert_for(project_ids) - rescue StandardError => e - log_error(e, project_ids) - ensure - log_info(project_ids) - end - - private - - def log_error(error, project_ids) - ::Gitlab::BackgroundMigration::Logger.error( - migrator: self.class.name, - message: error.message, - project_ids: project_ids - ) - end - - def log_info(project_ids) - ::Gitlab::BackgroundMigration::Logger.info( - migrator: self.class.name, - message: 'Projects has been processed to populate `has_vulnerabilities` information', - count: project_ids.length - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb deleted file mode 100644 index 28cc4a5e3fa..00000000000 --- a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This background migration creates records on merge_request_assignees according - # to the given merge request IDs range. A _single_ INSERT is issued for the given range. - # This is required for supporting multiple assignees on merge requests. - class PopulateMergeRequestAssigneesTable - def perform(from_id, to_id) - select_sql = - MergeRequest - .where(merge_request_assignees_not_exists_clause) - .where(id: from_id..to_id) - .where.not(assignee_id: nil) - .select(:id, :assignee_id) - .to_sql - - execute("INSERT INTO merge_request_assignees (merge_request_id, user_id) #{select_sql}") - end - - def perform_all_sync(batch_size:) - MergeRequest.each_batch(of: batch_size) do |batch| - range = batch.pluck('MIN(id)', 'MAX(id)').first - - perform(*range) - end - end - - private - - def merge_request_assignees_not_exists_clause - <<~SQL - NOT EXISTS (SELECT 1 FROM merge_request_assignees - WHERE merge_request_assignees.merge_request_id = merge_requests.id) - SQL - end - - def execute(sql) - @connection ||= ActiveRecord::Base.connection - @connection.execute(sql) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb b/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb deleted file mode 100644 index 04342fdabd4..00000000000 --- a/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb +++ /dev/null @@ -1,89 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class populates missing dismissal information for - # vulnerability entries. - class PopulateMissingVulnerabilityDismissalInformation - class Vulnerability < ActiveRecord::Base # rubocop:disable Style/Documentation - include EachBatch - - self.table_name = 'vulnerabilities' - - has_one :finding, class_name: '::Gitlab::BackgroundMigration::PopulateMissingVulnerabilityDismissalInformation::Finding' - - scope :broken, -> { where('state = 2 AND (dismissed_at IS NULL OR dismissed_by_id IS NULL)') } - - def copy_dismissal_information - return unless finding&.dismissal_feedback - - update_columns( - dismissed_at: finding.dismissal_feedback.created_at, - dismissed_by_id: finding.dismissal_feedback.author_id - ) - end - end - - class Finding < ActiveRecord::Base # rubocop:disable Style/Documentation - include ShaAttribute - include ::Gitlab::Utils::StrongMemoize - - self.table_name = 'vulnerability_occurrences' - - sha_attribute :project_fingerprint - - def dismissal_feedback - strong_memoize(:dismissal_feedback) do - Feedback.dismissal.where(category: report_type, project_fingerprint: project_fingerprint, project_id: project_id).first - end - end - end - - class Feedback < ActiveRecord::Base # rubocop:disable Style/Documentation - DISMISSAL_TYPE = 0 - - self.table_name = 'vulnerability_feedback' - - scope :dismissal, -> { where(feedback_type: DISMISSAL_TYPE) } - end - - def perform(*vulnerability_ids) - Vulnerability.includes(:finding).where(id: vulnerability_ids).each { |vulnerability| populate_for(vulnerability) } - - log_info(vulnerability_ids) - end - - private - - def populate_for(vulnerability) - log_warning(vulnerability) unless vulnerability.copy_dismissal_information - rescue StandardError => error - log_error(error, vulnerability) - end - - def log_info(vulnerability_ids) - ::Gitlab::BackgroundMigration::Logger.info( - migrator: self.class.name, - message: 'Dismissal information has been copied', - count: vulnerability_ids.length - ) - end - - def log_warning(vulnerability) - ::Gitlab::BackgroundMigration::Logger.warn( - migrator: self.class.name, - message: 'Could not update vulnerability!', - vulnerability_id: vulnerability.id - ) - end - - def log_error(error, vulnerability) - ::Gitlab::BackgroundMigration::Logger.error( - migrator: self.class.name, - message: error.message, - vulnerability_id: vulnerability.id - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb b/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb deleted file mode 100644 index ed7ffce8018..00000000000 --- a/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class creates/updates those personal snippets statistics - # that haven't been created nor initialized. - # It also updates the related root storage namespace stats - class PopulatePersonalSnippetStatistics - def perform(snippet_ids) - personal_snippets(snippet_ids).group_by(&:author).each do |author, author_snippets| - upsert_snippet_statistics(author_snippets) - update_namespace_statistics(author.namespace) - end - end - - private - - def personal_snippets(snippet_ids) - PersonalSnippet - .where(id: snippet_ids) - .includes(author: :namespace) - .includes(:statistics) - .includes(snippet_repository: :shard) - end - - def upsert_snippet_statistics(snippets) - snippets.each do |snippet| - response = Snippets::UpdateStatisticsService.new(snippet).execute - - error_message("#{response.message} snippet: #{snippet.id}") if response.error? - end - end - - def update_namespace_statistics(namespace) - Namespaces::StatisticsRefresherService.new.execute(namespace) - rescue StandardError => e - error_message("Error updating statistics for namespace #{namespace.id}: #{e.message}") - end - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build - end - - def error_message(message) - logger.error(message: "Snippet Statistics Migration: #{message}") - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_project_snippet_statistics.rb b/lib/gitlab/background_migration/populate_project_snippet_statistics.rb deleted file mode 100644 index 37af320f044..00000000000 --- a/lib/gitlab/background_migration/populate_project_snippet_statistics.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class creates/updates those project snippets statistics - # that haven't been created nor initialized. - # It also updates the related project statistics and its root storage namespace stats - class PopulateProjectSnippetStatistics - def perform(snippet_ids) - project_snippets(snippet_ids).group_by(&:namespace_id).each do |namespace_id, namespace_snippets| - namespace_snippets.group_by(&:project).each do |project, snippets| - upsert_snippet_statistics(snippets) - update_project_statistics(project) - rescue StandardError - error_message("Error updating statistics for project #{project.id}") - end - - update_namespace_statistics(namespace_snippets.first.project.root_namespace) - rescue StandardError => e - error_message("Error updating statistics for namespace #{namespace_id}: #{e.message}") - end - end - - private - - def project_snippets(snippet_ids) - ProjectSnippet - .select('snippets.*, projects.namespace_id') - .where(id: snippet_ids) - .joins(:project) - .includes(:statistics) - .includes(snippet_repository: :shard) - .includes(project: [:route, :statistics, :namespace]) - end - - def upsert_snippet_statistics(snippets) - snippets.each do |snippet| - response = Snippets::UpdateStatisticsService.new(snippet).execute - - error_message("#{response.message} snippet: #{snippet.id}") if response.error? - end - end - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build - end - - def error_message(message) - logger.error(message: "Snippet Statistics Migration: #{message}") - end - - def update_project_statistics(project) - project.statistics&.refresh!(only: [:snippets_size]) - end - - def update_namespace_statistics(namespace) - Namespaces::StatisticsRefresherService.new.execute(namespace) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb b/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb deleted file mode 100644 index 8241fea66db..00000000000 --- a/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class updates vulnerability feedback entities with no pipeline id assigned. - class PopulateVulnerabilityFeedbackPipelineId - def perform(project_ids) - end - end - end -end - -Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId.prepend_mod_with('Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId') diff --git a/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb deleted file mode 100644 index 9a9f23e29ea..00000000000 --- a/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class creates/updates those project historical vulnerability statistics - # that haven't been created nor initialized. It should only be executed in EE. - class PopulateVulnerabilityHistoricalStatistics - def perform(project_ids, retention_period = 90) - end - end - end -end - -Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics.prepend_mod_with('Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics') diff --git a/lib/gitlab/background_migration/prune_orphaned_geo_events.rb b/lib/gitlab/background_migration/prune_orphaned_geo_events.rb deleted file mode 100644 index 0efbe72775c..00000000000 --- a/lib/gitlab/background_migration/prune_orphaned_geo_events.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true -# -# rubocop:disable Style/Documentation - -# This job is added to fix https://gitlab.com/gitlab-org/gitlab/issues/30229 -# It's not used anywhere else. -# Can be removed in GitLab 13.* -module Gitlab - module BackgroundMigration - class PruneOrphanedGeoEvents - def perform(table_name) - end - end - end -end - -Gitlab::BackgroundMigration::PruneOrphanedGeoEvents.prepend_mod_with('Gitlab::BackgroundMigration::PruneOrphanedGeoEvents') diff --git a/lib/gitlab/background_migration/recalculate_project_authorizations_with_min_max_user_id.rb b/lib/gitlab/background_migration/recalculate_project_authorizations_with_min_max_user_id.rb deleted file mode 100644 index b66fdfd5c65..00000000000 --- a/lib/gitlab/background_migration/recalculate_project_authorizations_with_min_max_user_id.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop:disable Style/Documentation - class RecalculateProjectAuthorizationsWithMinMaxUserId - def perform(min_user_id, max_user_id) - User.where(id: min_user_id..max_user_id).find_each do |user| - service = Users::RefreshAuthorizedProjectsService.new( - user, - incorrect_auth_found_callback: - ->(project_id, access_level) do - logger.info(message: 'Removing ProjectAuthorizations', - user_id: user.id, - project_id: project_id, - access_level: access_level) - end, - missing_auth_found_callback: - ->(project_id, access_level) do - logger.info(message: 'Creating ProjectAuthorizations', - user_id: user.id, - project_id: project_id, - access_level: access_level) - end - ) - - service.execute - end - end - - private - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_security_scans.rb b/lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb index 0ae984f2dbc..20200a1d508 100644 --- a/lib/gitlab/background_migration/migrate_security_scans.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb @@ -3,11 +3,11 @@ module Gitlab module BackgroundMigration # rubocop: disable Style/Documentation - class MigrateSecurityScans + class RecalculateVulnerabilityFindingSignaturesForFindings def perform(start_id, stop_id) end end end end -Gitlab::BackgroundMigration::MigrateSecurityScans.prepend_mod_with('Gitlab::BackgroundMigration::MigrateSecurityScans') +Gitlab::BackgroundMigration::RecalculateVulnerabilityFindingSignaturesForFindings.prepend_mod diff --git a/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb b/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb deleted file mode 100644 index 17ef6dec4c0..00000000000 --- a/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class RemoveDuplicateCsFindings - def perform(start_id, stop_id) - end - end - end -end - -Gitlab::BackgroundMigration::RemoveDuplicateCsFindings.prepend_mod_with('Gitlab::BackgroundMigration::RemoveDuplicateCsFindings') diff --git a/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb deleted file mode 100644 index e5772fc7375..00000000000 --- a/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class RemoveDuplicatedCsFindingsWithoutVulnerabilityId - def perform(start_id, stop_id) - end - end - end -end - -Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId.prepend_mod_with('Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId') diff --git a/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb b/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb deleted file mode 100644 index cb6a600a525..00000000000 --- a/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop:disable Style/Documentation - class RemoveInaccessibleEpicTodos - def perform(start_id, stop_id) - end - end - end -end - -Gitlab::BackgroundMigration::RemoveInaccessibleEpicTodos.prepend_mod_with('Gitlab::BackgroundMigration::RemoveInaccessibleEpicTodos') diff --git a/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb b/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb new file mode 100644 index 00000000000..7fe5a427d10 --- /dev/null +++ b/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +# This migration will look for Vulnerabilities::Finding objects that would have a duplicate UUIDv5 if the UUID was +# recalculated. Then it removes Vulnerabilities::FindingPipeline objects associated with those Findings. +# We can't just drop those Findings directly since the cascade drop will timeout if any given Finding has too many +# associated FindingPipelines +class Gitlab::BackgroundMigration::RemoveOccurrencePipelinesAndDuplicateVulnerabilitiesFindings + # rubocop:disable Gitlab/NamespacedClass, Style/Documentation + class VulnerabilitiesFinding < ActiveRecord::Base + self.table_name = "vulnerability_occurrences" + end + + class VulnerabilitiesFindingPipeline < ActiveRecord::Base + include EachBatch + self.table_name = "vulnerability_occurrence_pipelines" + end + # rubocop:enable Gitlab/NamespacedClass, Style/Documentation + + def perform(start_id, end_id) + ids_to_look_for = findings_that_would_produce_duplicate_uuids(start_id, end_id) + + ids_to_look_for.each do |finding_id| + VulnerabilitiesFindingPipeline.where(occurrence_id: finding_id).each_batch(of: 1000) do |pipelines| + pipelines.delete_all + end + end + + VulnerabilitiesFinding.where(id: ids_to_look_for).delete_all + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def findings_that_would_produce_duplicate_uuids(start_id, end_id) + VulnerabilitiesFinding + .from("vulnerability_occurrences to_delete") + .where("to_delete.id BETWEEN ? AND ?", start_id, end_id) + .where( + "EXISTS ( + SELECT 1 + FROM vulnerability_occurrences duplicates + WHERE duplicates.report_type = to_delete.report_type + AND duplicates.location_fingerprint = to_delete.location_fingerprint + AND duplicates.primary_identifier_id = to_delete.primary_identifier_id + AND duplicates.project_id = to_delete.project_id + AND duplicates.id > to_delete.id + )" + ) + .pluck("to_delete.id") + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end +end diff --git a/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb b/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb deleted file mode 100644 index 4be61bfb689..00000000000 --- a/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class RemoveUndefinedVulnerabilityConfidenceLevel - def perform(start_id, stop_id) - end - end - end -end - -Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel.prepend_mod_with('Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel') diff --git a/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb b/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb new file mode 100644 index 00000000000..31fb5e97c5d --- /dev/null +++ b/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Remove vulnerability finding link records + # The records will be repopulated from the `raw_metadata` + # column of `vulnerability_occurrences` once the unique + # index is in place. + class RemoveVulnerabilityFindingLinks + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, stop_id) + define_batchable_model('vulnerability_finding_links').where(id: start_id..stop_id).delete_all + end + end + end +end diff --git a/lib/gitlab/background_migration/replace_blocked_by_links.rb b/lib/gitlab/background_migration/replace_blocked_by_links.rb deleted file mode 100644 index 0c29887bb00..00000000000 --- a/lib/gitlab/background_migration/replace_blocked_by_links.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class ReplaceBlockedByLinks - class IssueLink < ActiveRecord::Base - self.table_name = 'issue_links' - end - - def perform(start_id, stop_id) - blocked_by_links = IssueLink.where(id: start_id..stop_id).where(link_type: 2) - - ActiveRecord::Base.transaction do - # There could be two edge cases: - # 1) issue1 is blocked by issue2 AND issue2 blocks issue1 (type 1) - # 2) issue1 is blocked by issue2 AND issue2 is related to issue1 (type 0) - # In both cases cases we couldn't convert blocked by relation to - # `issue2 blocks issue` because there is already a link with the same - # source/target id. To avoid these conflicts, we first delete any - # "opposite" links before we update `blocked by` relation. This - # should be rare as we have a pre-create check which checks if issues - # are already linked - opposite_ids = blocked_by_links - .select('opposite_links.id') - .joins('INNER JOIN issue_links as opposite_links ON issue_links.source_id = opposite_links.target_id AND issue_links.target_id = opposite_links.source_id') - IssueLink.where(id: opposite_ids).delete_all - - blocked_by_links.update_all('source_id=target_id,target_id=source_id,link_type=1') - end - end - end - end -end diff --git a/lib/gitlab/background_migration/reset_merge_status.rb b/lib/gitlab/background_migration/reset_merge_status.rb deleted file mode 100644 index d040b4931be..00000000000 --- a/lib/gitlab/background_migration/reset_merge_status.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Updates the range of given MRs to merge_status "unchecked", if they're opened - # and mergeable. - class ResetMergeStatus - def perform(from_id, to_id) - relation = MergeRequest.where(id: from_id..to_id, - state_id: 1, # opened - merge_status: 'can_be_merged') - - relation.update_all(merge_status: 'unchecked') - end - end - end -end diff --git a/lib/gitlab/background_migration/reset_shared_runners_for_transferred_projects.rb b/lib/gitlab/background_migration/reset_shared_runners_for_transferred_projects.rb deleted file mode 100644 index 0053cafb4ac..00000000000 --- a/lib/gitlab/background_migration/reset_shared_runners_for_transferred_projects.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Resets inconsistent state of shared_runners_enabled for projects that have been transferred - class ResetSharedRunnersForTransferredProjects - # Model specifically used for migration. - class Namespace < ActiveRecord::Base - include EachBatch - - self.table_name = 'namespaces' - end - - # Model specifically used for migration. - class Project < ActiveRecord::Base - self.table_name = 'projects' - end - - def perform(start_id, stop_id) - Project.reset_column_information - - Namespace.where(id: start_id..stop_id).each_batch(of: 1_000) do |relation| - ids = relation.where(shared_runners_enabled: false, allow_descendants_override_disabled_shared_runners: false).select(:id) - - Project.where(namespace_id: ids).update_all(shared_runners_enabled: false) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb b/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb deleted file mode 100644 index 527dd2a0a83..00000000000 --- a/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Sets the MergeRequestDiff#files_count value for old rows - class SetMergeRequestDiffFilesCount - # Some historic data has a *lot* of files. Apply a sentinel to these cases - FILES_COUNT_SENTINEL = 2**15 - 1 - - def self.count_subquery - <<~SQL - files_count = ( - SELECT LEAST(#{FILES_COUNT_SENTINEL}, count(*)) - FROM merge_request_diff_files - WHERE merge_request_diff_files.merge_request_diff_id = merge_request_diffs.id - ) - SQL - end - - class MergeRequestDiff < ActiveRecord::Base # rubocop:disable Style/Documentation - include EachBatch - - self.table_name = 'merge_request_diffs' - end - - def perform(start_id, end_id) - MergeRequestDiff.where(id: start_id..end_id).each_batch do |relation| - relation.update_all(self.class.count_subquery) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/update_existing_subgroup_to_match_visibility_level_of_parent.rb b/lib/gitlab/background_migration/update_existing_subgroup_to_match_visibility_level_of_parent.rb deleted file mode 100644 index 9e330f7c008..00000000000 --- a/lib/gitlab/background_migration/update_existing_subgroup_to_match_visibility_level_of_parent.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This background migration updates children of group to match visibility of a parent - class UpdateExistingSubgroupToMatchVisibilityLevelOfParent - def perform(parents_groups_ids, level) - groups_ids = Gitlab::ObjectHierarchy.new(Group.where(id: parents_groups_ids)) - .base_and_descendants - .where("visibility_level > ?", level) - .select(:id) - - return if groups_ids.empty? - - Group - .where(id: groups_ids) - .update_all(visibility_level: level) - end - end - end -end diff --git a/lib/gitlab/background_migration/update_existing_users_that_require_two_factor_auth.rb b/lib/gitlab/background_migration/update_existing_users_that_require_two_factor_auth.rb deleted file mode 100644 index d97765cd398..00000000000 --- a/lib/gitlab/background_migration/update_existing_users_that_require_two_factor_auth.rb +++ /dev/null @@ -1,110 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class UpdateExistingUsersThatRequireTwoFactorAuth # rubocop:disable Metrics/ClassLength - def perform(start_id, stop_id) - ActiveRecord::Base.connection.execute <<~SQL - UPDATE - users - SET - require_two_factor_authentication_from_group = FALSE - WHERE - users.id BETWEEN #{start_id} - AND #{stop_id} - AND users.require_two_factor_authentication_from_group = TRUE - AND users.id NOT IN ( SELECT DISTINCT - users_groups_query.user_id - FROM ( - SELECT - users.id AS user_id, - members.source_id AS group_ids - FROM - users - LEFT JOIN members ON members.source_type = 'Namespace' - AND members.requested_at IS NULL - AND members.user_id = users.id - AND members.type = 'GroupMember' - WHERE - users.require_two_factor_authentication_from_group = TRUE - AND users.id BETWEEN #{start_id} - AND #{stop_id}) AS users_groups_query - INNER JOIN LATERAL ( WITH RECURSIVE "base_and_ancestors" AS ( - ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "namespaces" - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."id" = users_groups_query.group_ids) - UNION ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "namespaces", - "base_and_ancestors" - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."id" = "base_and_ancestors"."parent_id")), - "base_and_descendants" AS ( - ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "namespaces" - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."id" = users_groups_query.group_ids) - UNION ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "namespaces", - "base_and_descendants" - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."parent_id" = "base_and_descendants"."id")) - SELECT - "namespaces".* - FROM (( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "base_and_ancestors" AS "namespaces" - WHERE - "namespaces"."type" = 'Group') - UNION ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "base_and_descendants" AS "namespaces" - WHERE - "namespaces"."type" = 'Group')) namespaces - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."require_two_factor_authentication" = TRUE) AS hierarchy_tree ON TRUE); - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb b/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb deleted file mode 100644 index 054b918dade..00000000000 --- a/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class UpdateLocationFingerprintForContainerScanningFindings - def perform(start_id, stop_id) - end - end - end -end - -Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings.prepend_mod_with('Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings') diff --git a/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb b/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb new file mode 100644 index 00000000000..c95ef9f5515 --- /dev/null +++ b/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class to populate spent_at for timelogs + class UpdateTimelogsNullSpentAt + include Gitlab::Database::DynamicModelHelpers + + BATCH_SIZE = 100 + + def perform(start_id, stop_id) + define_batchable_model('timelogs').where(spent_at: nil, id: start_id..stop_id).each_batch(of: 100) do |subbatch| + batch_start, batch_end = subbatch.pluck('min(id), max(id)').first + + update_timelogs(batch_start, batch_end) + end + end + + def update_timelogs(batch_start, batch_stop) + execute(<<~SQL) + UPDATE timelogs + SET spent_at = created_at + WHERE spent_at IS NULL + AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop}; + SQL + end + + def execute(sql) + @connection ||= ::ActiveRecord::Base.connection + @connection.execute(sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb b/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb deleted file mode 100644 index 1cc03f061fb..00000000000 --- a/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class UpdateVulnerabilitiesFromDismissalFeedback - def perform(project_id) - end - end - end -end - -Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback.prepend_mod_with('Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback') diff --git a/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb b/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb deleted file mode 100644 index b3876018553..00000000000 --- a/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - class CreateResourceUserMention - # Resources that have mentions to be migrated: - # issue, merge_request, epic, commit, snippet, design - - BULK_INSERT_SIZE = 1_000 - ISOLATION_MODULE = 'Gitlab::BackgroundMigration::UserMentions::Models' - - def perform(resource_model, join, conditions, with_notes, start_id, end_id) - return unless Feature.enabled?(:migrate_user_mentions, default_enabled: true) - - resource_model = "#{ISOLATION_MODULE}::#{resource_model}".constantize if resource_model.is_a?(String) - model = with_notes ? Gitlab::BackgroundMigration::UserMentions::Models::Note : resource_model - resource_user_mention_model = resource_model.user_mention_model - - records = model.joins(join).where(conditions).where(id: start_id..end_id) - - records.each_batch(of: BULK_INSERT_SIZE) do |records| - mentions = [] - records.each do |record| - mention_record = record.build_mention_values(resource_user_mention_model.resource_foreign_key) - mentions << mention_record unless mention_record.blank? - end - - resource_user_mention_model.insert_all(mentions) unless mentions.empty? - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb deleted file mode 100644 index 3def5eb3369..00000000000 --- a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Lib - module Banzai - # isolated Banzai::ReferenceParser - module ReferenceParser - # Returns the reference parser class for the given type - # - # Example: - # - # Banzai::ReferenceParser['isolated_mentioned_group'] - # - # This would return the `::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser::IsolatedMentionedGroupParser` class. - def self.[](name) - const_get("::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser::#{name.to_s.camelize}Parser", false) - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb deleted file mode 100644 index d3d032ba433..00000000000 --- a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Lib - module Banzai - module ReferenceParser - # isolated Banzai::ReferenceParser::MentionedGroupParser - class IsolatedMentionedGroupParser < ::Banzai::ReferenceParser::MentionedGroupParser - extend ::Gitlab::Utils::Override - - self.reference_type = :user - - override :references_relation - def references_relation - ::Gitlab::BackgroundMigration::UserMentions::Models::Group - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb deleted file mode 100644 index 5930d65bc2c..00000000000 --- a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Lib - module Banzai - module ReferenceParser - # isolated Banzai::ReferenceParser::MentionedGroupParser - class IsolatedMentionedProjectParser < ::Banzai::ReferenceParser::MentionedProjectParser - extend ::Gitlab::Utils::Override - - self.reference_type = :user - - override :references_relation - def references_relation - ::Gitlab::BackgroundMigration::UserMentions::Models::Project - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb deleted file mode 100644 index f5f98517433..00000000000 --- a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Lib - module Banzai - module ReferenceParser - # isolated Banzai::ReferenceParser::MentionedGroupParser - class IsolatedMentionedUserParser < ::Banzai::ReferenceParser::MentionedUserParser - extend ::Gitlab::Utils::Override - - self.reference_type = :user - - override :references_relation - def references_relation - ::Gitlab::BackgroundMigration::UserMentions::Models::User - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb deleted file mode 100644 index 8610129533d..00000000000 --- a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Lib - module Gitlab - # Extract possible GFM references from an arbitrary String for further processing. - class IsolatedReferenceExtractor < ::Gitlab::ReferenceExtractor - REFERABLES = %i(isolated_mentioned_group isolated_mentioned_user isolated_mentioned_project).freeze - - REFERABLES.each do |type| - define_method("#{type}s") do - @references[type] ||= isolated_references(type) - end - end - - def isolated_references(type) - context = ::Banzai::RenderContext.new(project, current_user) - processor = ::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser[type].new(context) - - refs = processor.process(html_documents) - refs[:visible] - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb deleted file mode 100644 index 0334ea1dd08..00000000000 --- a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb +++ /dev/null @@ -1,60 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Lib - module Gitlab - # Gitlab::IsolatedVisibilityLevel module - # - # Define allowed public modes that can be used for - # GitLab projects to determine project public mode - # - module IsolatedVisibilityLevel - extend ::ActiveSupport::Concern - - included do - scope :public_to_user, -> (user = nil) do - where(visibility_level: IsolatedVisibilityLevel.levels_for_user(user)) - end - end - - PRIVATE = 0 unless const_defined?(:PRIVATE) - INTERNAL = 10 unless const_defined?(:INTERNAL) - PUBLIC = 20 unless const_defined?(:PUBLIC) - - class << self - def levels_for_user(user = nil) - return [PUBLIC] unless user - - if user.can_read_all_resources? - [PRIVATE, INTERNAL, PUBLIC] - elsif user.external? - [PUBLIC] - else - [INTERNAL, PUBLIC] - end - end - end - - def private? - visibility_level_value == PRIVATE - end - - def internal? - visibility_level_value == INTERNAL - end - - def public? - visibility_level_value == PUBLIC - end - - def visibility_level_value - self[visibility_level_field] - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/commit.rb b/lib/gitlab/background_migration/user_mentions/models/commit.rb deleted file mode 100644 index 65f4a7a25b6..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/commit.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - class Commit - include EachBatch - include Concerns::IsolatedMentionable - include Concerns::MentionableMigrationMethods - - def self.user_mention_model - Gitlab::BackgroundMigration::UserMentions::Models::CommitUserMention - end - - def user_mention_model - self.class.user_mention_model - end - - def user_mention_resource_id - id - end - - def user_mention_note_id - 'NULL' - end - - def self.no_quote_columns - [:note_id] - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb deleted file mode 100644 index f4cc96c8bc0..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - class CommitUserMention < ActiveRecord::Base - self.table_name = 'commit_user_mentions' - self.inheritance_column = :_type_disabled - - def self.resource_foreign_key - :commit_id - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb deleted file mode 100644 index ba6b783f9f1..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - module Concerns - # isolated FeatureGate module - module IsolatedFeatureGate - def flipper_id - return if new_record? - - "#{self.class.name}:#{id}" - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb deleted file mode 100644 index f684f789ea9..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb +++ /dev/null @@ -1,104 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - module Concerns - # == IsolatedMentionable concern - # - # Shortcutted for isolation version of Mentionable to be used in mentions migrations - # - module IsolatedMentionable - extend ::ActiveSupport::Concern - - class_methods do - # Indicate which attributes of the Mentionable to search for GFM references. - def attr_mentionable(attr, options = {}) - attr = attr.to_s - mentionable_attrs << [attr, options] - end - end - - included do - # Accessor for attributes marked mentionable. - cattr_accessor :mentionable_attrs, instance_accessor: false do - [] - end - - if self < Participable - participant -> (user, ext) { all_references(user, extractor: ext) } - end - end - - def all_references(current_user = nil, extractor: nil) - # Use custom extractor if it's passed in the function parameters. - if extractor - extractors[current_user] = extractor - else - extractor = extractors[current_user] ||= - Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedReferenceExtractor.new(project, current_user) - - extractor.reset_memoized_values - end - - self.class.mentionable_attrs.each do |attr, options| - text = __send__(attr) # rubocop:disable GitlabSecurity/PublicSend - options = options.merge( - cache_key: [self, attr], - author: author, - skip_project_check: skip_project_check? - ).merge(mentionable_params) - - cached_html = self.try(:updated_cached_html_for, attr.to_sym) - options[:rendered] = cached_html if cached_html - - extractor.analyze(text, options) - end - - extractor - end - - def extractors - @extractors ||= {} - end - - def skip_project_check? - false - end - - def build_mention_values(resource_foreign_key) - refs = all_references(author) - - mentioned_users_ids = array_to_sql(refs.isolated_mentioned_users.pluck(:id)) - mentioned_projects_ids = array_to_sql(refs.isolated_mentioned_projects.pluck(:id)) - mentioned_groups_ids = array_to_sql(refs.isolated_mentioned_groups.pluck(:id)) - - return if mentioned_users_ids.blank? && mentioned_projects_ids.blank? && mentioned_groups_ids.blank? - - { - "#{resource_foreign_key}": user_mention_resource_id, - note_id: user_mention_note_id, - mentioned_users_ids: mentioned_users_ids, - mentioned_projects_ids: mentioned_projects_ids, - mentioned_groups_ids: mentioned_groups_ids - } - end - - def array_to_sql(ids_array) - return unless ids_array.present? - - '{' + ids_array.join(", ") + '}' - end - - private - - def mentionable_params - {} - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/mentionable_migration_methods.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/mentionable_migration_methods.rb deleted file mode 100644 index efb08d44100..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/mentionable_migration_methods.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - module Concerns - # Extract common no_quote_columns method used in determining the columns that do not need - # to be quoted for corresponding models - module MentionableMigrationMethods - extend ::ActiveSupport::Concern - - class_methods do - def no_quote_columns - [ - :note_id, - user_mention_model.resource_foreign_key - ] - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb deleted file mode 100644 index 75759ed0111..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb +++ /dev/null @@ -1,74 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - module Concerns - module Namespace - # isolate recursive traversal code for namespace hierarchy - module RecursiveTraversal - extend ActiveSupport::Concern - - def root_ancestor - return self if persisted? && parent_id.nil? - - strong_memoize(:root_ancestor) do - Gitlab::ObjectHierarchy - .new(self.class.where(id: id)) - .base_and_ancestors - .reorder(nil) - .find_by(parent_id: nil) - end - end - - # Returns all ancestors, self, and descendants of the current namespace. - def self_and_hierarchy - Gitlab::ObjectHierarchy - .new(self.class.where(id: id)) - .all_objects - end - - # Returns all the ancestors of the current namespaces. - def ancestors - return self.class.none unless parent_id - - Gitlab::ObjectHierarchy - .new(self.class.where(id: parent_id)) - .base_and_ancestors - end - - # returns all ancestors upto but excluding the given namespace - # when no namespace is given, all ancestors upto the top are returned - def ancestors_upto(top = nil, hierarchy_order: nil) - Gitlab::ObjectHierarchy.new(self.class.where(id: id)) - .ancestors(upto: top, hierarchy_order: hierarchy_order) - end - - def self_and_ancestors(hierarchy_order: nil) - return self.class.where(id: id) unless parent_id - - Gitlab::ObjectHierarchy - .new(self.class.where(id: id)) - .base_and_ancestors(hierarchy_order: hierarchy_order) - end - - # Returns all the descendants of the current namespace. - def descendants - Gitlab::ObjectHierarchy - .new(self.class.where(parent_id: id)) - .base_and_descendants - end - - def self_and_descendants - Gitlab::ObjectHierarchy - .new(self.class.where(id: id)) - .base_and_descendants - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb deleted file mode 100644 index d010d68600d..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - module DesignManagement - class Design < ActiveRecord::Base - include EachBatch - include Concerns::MentionableMigrationMethods - - self.table_name = 'design_management_designs' - self.inheritance_column = :_type_disabled - - def self.user_mention_model - Gitlab::BackgroundMigration::UserMentions::Models::DesignUserMention - end - - def user_mention_model - self.class.user_mention_model - end - - def user_mention_resource_id - id - end - - def user_mention_note_id - 'NULL' - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb deleted file mode 100644 index eb00f6cfa3f..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - class DesignUserMention < ActiveRecord::Base - self.table_name = 'design_user_mentions' - self.inheritance_column = :_type_disabled - - def self.resource_foreign_key - :design_id - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/epic.rb b/lib/gitlab/background_migration/user_mentions/models/epic.rb deleted file mode 100644 index cfd9a4faa9b..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/epic.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - class Epic < ActiveRecord::Base - include EachBatch - include Concerns::IsolatedMentionable - include Concerns::MentionableMigrationMethods - include CacheMarkdownField - - attr_mentionable :title, pipeline: :single_line - attr_mentionable :description - cache_markdown_field :title, pipeline: :single_line - cache_markdown_field :description, issuable_state_filter_enabled: true - - self.table_name = 'epics' - self.inheritance_column = :_type_disabled - - belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" - belongs_to :group, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Group" - - def self.user_mention_model - Gitlab::BackgroundMigration::UserMentions::Models::EpicUserMention - end - - def user_mention_model - self.class.user_mention_model - end - - def project - nil - end - - def mentionable_params - { group: group, label_url_method: :group_epics_url } - end - - def user_mention_resource_id - id - end - - def user_mention_note_id - 'NULL' - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb deleted file mode 100644 index 579e4d99612..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - class EpicUserMention < ActiveRecord::Base - self.table_name = 'epic_user_mentions' - self.inheritance_column = :_type_disabled - - def self.resource_foreign_key - :epic_id - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/group.rb b/lib/gitlab/background_migration/user_mentions/models/group.rb deleted file mode 100644 index 310723570c2..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/group.rb +++ /dev/null @@ -1,97 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - # isolated Group model - class Group < ::Gitlab::BackgroundMigration::UserMentions::Models::Namespace - self.store_full_sti_class = false - self.inheritance_column = :_type_disabled - - has_one :saml_provider - - def root_saml_provider - root_ancestor.saml_provider - end - - def self.declarative_policy_class - "GroupPolicy" - end - - def max_member_access_for_user(user) - return GroupMember::NO_ACCESS unless user - - return GroupMember::OWNER if user.admin? - - max_member_access = members_with_parents.where(user_id: user) - .reorder(access_level: :desc) - .first - &.access_level - - max_member_access || GroupMember::NO_ACCESS - end - - def members_with_parents - # Avoids an unnecessary SELECT when the group has no parents - source_ids = - if has_parent? - self_and_ancestors.reorder(nil).select(:id) - else - id - end - - group_hierarchy_members = GroupMember.active_without_invites_and_requests - .where(source_id: source_ids) - - GroupMember.from_union([group_hierarchy_members, - members_from_self_and_ancestor_group_shares]) - end - - # rubocop: disable Metrics/AbcSize - def members_from_self_and_ancestor_group_shares - group_group_link_table = GroupGroupLink.arel_table - group_member_table = GroupMember.arel_table - - source_ids = - if has_parent? - self_and_ancestors.reorder(nil).select(:id) - else - id - end - - group_group_links_query = GroupGroupLink.where(shared_group_id: source_ids) - cte = Gitlab::SQL::CTE.new(:group_group_links_cte, group_group_links_query) - cte_alias = cte.table.alias(GroupGroupLink.table_name) - - # Instead of members.access_level, we need to maximize that access_level at - # the respective group_group_links.group_access. - member_columns = GroupMember.attribute_names.map do |column_name| - if column_name == 'access_level' - smallest_value_arel([cte_alias[:group_access], group_member_table[:access_level]], - 'access_level') - else - group_member_table[column_name] - end - end - - GroupMember - .with(cte.to_arel) - .select(*member_columns) - .from([group_member_table, cte.alias_to(group_group_link_table)]) - .where(group_member_table[:requested_at].eq(nil)) - .where(group_member_table[:source_id].eq(group_group_link_table[:shared_with_group_id])) - .where(group_member_table[:source_type].eq('Namespace')) - end - # rubocop: enable Metrics/AbcSize - - def smallest_value_arel(args, column_alias) - Arel::Nodes::As.new( - Arel::Nodes::NamedFunction.new('LEAST', args), - Arel::Nodes::SqlLiteral.new(column_alias)) - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb deleted file mode 100644 index 13addcc3c55..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - class MergeRequest < ActiveRecord::Base - include EachBatch - include Concerns::IsolatedMentionable - include CacheMarkdownField - include Concerns::MentionableMigrationMethods - - attr_mentionable :title, pipeline: :single_line - attr_mentionable :description - cache_markdown_field :title, pipeline: :single_line - cache_markdown_field :description, issuable_state_filter_enabled: true - - self.table_name = 'merge_requests' - self.inheritance_column = :_type_disabled - - belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" - belongs_to :target_project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" - belongs_to :source_project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" - - alias_attribute :project, :target_project - - def self.user_mention_model - Gitlab::BackgroundMigration::UserMentions::Models::MergeRequestUserMention - end - - def user_mention_model - self.class.user_mention_model - end - - def user_mention_resource_id - id - end - - def user_mention_note_id - 'NULL' - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb deleted file mode 100644 index 4a85892d7b8..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - class MergeRequestUserMention < ActiveRecord::Base - self.table_name = 'merge_request_user_mentions' - self.inheritance_column = :_type_disabled - - def self.resource_foreign_key - :merge_request_id - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb deleted file mode 100644 index d76d06606ee..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/namespace.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - # isolated Namespace model - class Namespace < ActiveRecord::Base - self.inheritance_column = :_type_disabled - - include Concerns::IsolatedFeatureGate - include Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedVisibilityLevel - include ::Gitlab::Utils::StrongMemoize - include Gitlab::BackgroundMigration::UserMentions::Models::Concerns::Namespace::RecursiveTraversal - - belongs_to :parent, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Namespace" - - def visibility_level_field - :visibility_level - end - - def has_parent? - parent_id.present? || parent.present? - end - - # Deprecated, use #licensed_feature_available? instead. Remove once Namespace#feature_available? isn't used anymore. - def feature_available?(feature) - licensed_feature_available?(feature) - end - - # Overridden in EE::Namespace - def licensed_feature_available?(_feature) - false - end - end - end - end - end -end - -Namespace.prepend_mod_with('Namespace') diff --git a/lib/gitlab/background_migration/user_mentions/models/note.rb b/lib/gitlab/background_migration/user_mentions/models/note.rb deleted file mode 100644 index 4026a91903f..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/note.rb +++ /dev/null @@ -1,72 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - class Note < ActiveRecord::Base - include EachBatch - include Concerns::IsolatedMentionable - include CacheMarkdownField - - self.table_name = 'notes' - self.inheritance_column = :_type_disabled - - attr_mentionable :note, pipeline: :note - cache_markdown_field :note, pipeline: :note, issuable_state_filter_enabled: true - - belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" - belongs_to :noteable, polymorphic: true - belongs_to :project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" - - def for_personal_snippet? - noteable && noteable.instance_of?(PersonalSnippet) - end - - def for_project_noteable? - !for_personal_snippet? && !for_epic? - end - - def skip_project_check? - !for_project_noteable? - end - - def for_epic? - noteable && noteable_type == 'Epic' - end - - def user_mention_resource_id - noteable_id || commit_id - end - - def user_mention_note_id - id - end - - def noteable - super unless for_commit? - end - - def for_commit? - noteable_type == "Commit" - end - - private - - def mentionable_params - return super unless for_epic? - - super.merge(banzai_context_params) - end - - def banzai_context_params - return {} unless noteable - - { group: noteable.group, label_url_method: :group_epics_url } - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/project.rb b/lib/gitlab/background_migration/user_mentions/models/project.rb deleted file mode 100644 index 4e02bf97d12..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/project.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - # isolated Namespace model - class Project < ActiveRecord::Base - include Concerns::IsolatedFeatureGate - include Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedVisibilityLevel - - self.table_name = 'projects' - self.inheritance_column = :_type_disabled - - belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id', class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Group" - belongs_to :namespace, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Namespace" - alias_method :parent, :namespace - - # Returns a collection of projects that is either public or visible to the - # logged in user. - def self.public_or_visible_to_user(user = nil, min_access_level = nil) - min_access_level = nil if user&.can_read_all_resources? - - return public_to_user unless user - - if user.is_a?(::Gitlab::BackgroundMigration::UserMentions::Models::User) - where('EXISTS (?) OR projects.visibility_level IN (?)', - user.authorizations_for_projects(min_access_level: min_access_level), - levels_for_user(user)) - end - end - - def grafana_integration - nil - end - - def default_issues_tracker? - true # we do not care of the issue tracker type(internal or external) when parsing mentions - end - - def visibility_level_field - :visibility_level - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/user.rb b/lib/gitlab/background_migration/user_mentions/models/user.rb deleted file mode 100644 index a30220b6934..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/user.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - # isolated Namespace model - class User < ActiveRecord::Base - include Concerns::IsolatedFeatureGate - - self.table_name = 'users' - self.inheritance_column = :_type_disabled - - has_many :project_authorizations, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent - - def authorizations_for_projects(min_access_level: nil, related_project_column: 'projects.id') - authorizations = project_authorizations - .select(1) - .where("project_authorizations.project_id = #{related_project_column}") - - return authorizations unless min_access_level.present? - - authorizations.where('project_authorizations.access_level >= ?', min_access_level) - end - - def can_read_all_resources? - can?(:read_all_resources) - end - - def can?(action, subject = :global) - Ability.allowed?(self, action, subject) - end - end - end - end - end -end diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index 899e2e6c1c5..242979da367 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -19,7 +19,8 @@ module Gitlab end def self.refmap - [:heads, :tags, '+refs/pull-requests/*/to:refs/merge-requests/*/head'] + # We omit :heads and :tags since these are fetched in the import_repository + ['+refs/pull-requests/*/to:refs/merge-requests/*/head'] end # Unlike GitHub, you can't grab the commit SHAs for pull requests that @@ -140,11 +141,11 @@ module Gitlab def import_repository log_info(stage: 'import_repository', message: 'starting import') - project.ensure_repository + project.repository.import_repository(project.import_url) project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap) log_info(stage: 'import_repository', message: 'finished import') - rescue Gitlab::Shell::Error => e + rescue ::Gitlab::Git::CommandError => e Gitlab::ErrorTracking.log_exception( e, stage: 'import_repository', message: 'failed import', error: e.message diff --git a/lib/gitlab/chat/output.rb b/lib/gitlab/chat/output.rb index 4a55b81a9eb..b4ce05ca06d 100644 --- a/lib/gitlab/chat/output.rb +++ b/lib/gitlab/chat/output.rb @@ -48,10 +48,10 @@ module Gitlab # follows it will produce a nil. For example: # # "\n".split("\n") # => [] - # "\n".split("\n")[1..-1] # => nil + # "\n".split("\n")[1..] # => nil # # To work around this we only "join" if we're given an Array. - if (converted = output.split("\n")[1..-1]) + if (converted = output.split("\n")[1..]) converted.join("\n") else '' diff --git a/lib/gitlab/ci/badge/coverage/report.rb b/lib/gitlab/ci/badge/coverage/report.rb index 78b51dbdaf0..4d1193176ad 100644 --- a/lib/gitlab/ci/badge/coverage/report.rb +++ b/lib/gitlab/ci/badge/coverage/report.rb @@ -30,7 +30,7 @@ module Gitlab::Ci @coverage ||= raw_coverage return unless @coverage - @coverage.to_f.round(2) + @coverage.round(2) end def metadata diff --git a/lib/gitlab/ci/badge/metadata.rb b/lib/gitlab/ci/badge/metadata.rb index eec9fedfaa9..244e3aff851 100644 --- a/lib/gitlab/ci/badge/metadata.rb +++ b/lib/gitlab/ci/badge/metadata.rb @@ -8,14 +8,13 @@ module Gitlab::Ci class Metadata include Gitlab::Routing include ActionView::Helpers::AssetTagHelper - include ActionView::Helpers::UrlHelper def initialize(badge) @badge = badge end def to_html - link_to(image_tag(image_url, alt: title), link_url) + ApplicationController.helpers.link_to(image_tag(image_url, alt: title), link_url) end def to_markdown diff --git a/lib/gitlab/ci/build/context/base.rb b/lib/gitlab/ci/build/context/base.rb index c7ea7c78e2f..81f96e822f4 100644 --- a/lib/gitlab/ci/build/context/base.rb +++ b/lib/gitlab/ci/build/context/base.rb @@ -17,6 +17,12 @@ module Gitlab raise NotImplementedError end + def variables_hash + strong_memoize(:variables_hash) do + variables.to_hash + end + end + def project pipeline.project end diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb index 7b1ce6330f0..810523052ae 100644 --- a/lib/gitlab/ci/build/policy/variables.rb +++ b/lib/gitlab/ci/build/policy/variables.rb @@ -10,7 +10,7 @@ module Gitlab end def satisfied_by?(pipeline, context) - variables = context.variables + variables = context.variables_hash statements = @expressions.map do |statement| ::Gitlab::Ci::Pipeline::Expression::Statement diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb index 9c2f6eea1dd..82a59fdb4e1 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -23,7 +23,7 @@ module Gitlab return @globs unless context @globs.map do |glob| - ExpandVariables.expand_existing(glob, context.variables) + ExpandVariables.expand_existing(glob, -> { context.variables_hash }) end end end diff --git a/lib/gitlab/ci/build/rules/rule/clause/if.rb b/lib/gitlab/ci/build/rules/rule/clause/if.rb index 6143a736ca6..499a265a1e2 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/if.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/if.rb @@ -10,7 +10,7 @@ module Gitlab def satisfied_by?(pipeline, context) ::Gitlab::Ci::Pipeline::Expression::Statement.new( - @expression, context.variables).truthful? + @expression, context.variables_hash).truthful? end end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 6f149385969..42b487fdf81 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -17,21 +17,27 @@ module Gitlab Config::Yaml::Tags::TagError ].freeze - attr_reader :root, :context, :source_ref_path, :source + attr_reader :root, :context, :source_ref_path, :source, :logger - def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil) + def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, logger: nil) + @logger = logger || ::Gitlab::Ci::Pipeline::Logger.new(project: project) @source_ref_path = pipeline&.source_ref_path - @context = build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline) + @context = self.logger.instrument(:config_build_context) do + build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline) + end + @context.set_deadline(TIMEOUT_SECONDS) @source = source - @config = expand_config(config) - - @root = Entry::Root.new(@config) - @root.compose! + @config = self.logger.instrument(:config_expand) do + expand_config(config) + end + @root = self.logger.instrument(:config_compose) do + Entry::Root.new(@config).tap(&:compose!) + end rescue *rescue_errors => e raise Config::ConfigError, e.message end @@ -94,11 +100,25 @@ module Gitlab end def build_config(config) - initial_config = Config::Yaml.load!(config) - initial_config = Config::External::Processor.new(initial_config, @context).perform - initial_config = Config::Extendable.new(initial_config).to_hash - initial_config = Config::Yaml::Tags::Resolver.new(initial_config).to_hash - Config::EdgeStagesInjector.new(initial_config).to_hash + initial_config = logger.instrument(:config_yaml_load) do + Config::Yaml.load!(config) + end + + initial_config = logger.instrument(:config_external_process) do + Config::External::Processor.new(initial_config, @context).perform + end + + initial_config = logger.instrument(:config_yaml_extend) do + Config::Extendable.new(initial_config).to_hash + end + + initial_config = logger.instrument(:config_tags_resolve) do + Config::Yaml::Tags::Resolver.new(initial_config).to_hash + end + + logger.instrument(:config_stages_inject) do + Config::EdgeStagesInjector.new(initial_config).to_hash + end end def find_sha(project) @@ -115,10 +135,20 @@ module Gitlab sha: sha || find_sha(project), user: user, parent_pipeline: parent_pipeline, - variables: build_variables(project: project, pipeline: pipeline)) + variables: build_variables(project: project, pipeline: pipeline), + logger: logger) end def build_variables(project:, pipeline:) + logger.instrument(:config_build_variables) do + build_variables_without_instrumentation( + project: project, + pipeline: pipeline + ) + end + end + + def build_variables_without_instrumentation(project:, pipeline:) Gitlab::Ci::Variables::Collection.new.tap do |variables| break variables unless project diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 520b1ce6119..43475742214 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -26,7 +26,7 @@ module Gitlab validates :name, length: { maximum: 255 }, if: -> { ::Feature.enabled?(:ci_validate_job_length, default_enabled: :yaml) } validates :config, disallowed_keys: { - in: %i[only except when start_in], + in: %i[only except start_in], message: 'key may not be used with `rules`' }, if: :has_rules? diff --git a/lib/gitlab/ci/config/entry/tags.rb b/lib/gitlab/ci/config/entry/tags.rb index ca3b48372e2..6044cfddbdc 100644 --- a/lib/gitlab/ci/config/entry/tags.rb +++ b/lib/gitlab/ci/config/entry/tags.rb @@ -16,8 +16,6 @@ module Gitlab validates :config, array_of_strings: true validate do - next unless ::Feature.enabled?(:ci_build_tags_limit, default_enabled: :yaml) - if config.is_a?(Array) && config.size >= TAGS_LIMIT errors.add(:config, _("must be less than the limit of %{tag_limit} tags") % { tag_limit: TAGS_LIMIT }) end diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 51624dc30ea..308414af47d 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -9,17 +9,22 @@ module Gitlab TimeoutError = Class.new(StandardError) + include ::Gitlab::Utils::StrongMemoize + attr_reader :project, :sha, :user, :parent_pipeline, :variables - attr_reader :expandset, :execution_deadline + attr_reader :expandset, :execution_deadline, :logger + + delegate :instrument, to: :logger - def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: []) + def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil, logger: nil) @project = project @sha = sha @user = user @parent_pipeline = parent_pipeline - @variables = variables + @variables = variables || Ci::Variables::Collection.new @expandset = Set.new @execution_deadline = 0 + @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) yield self if block_given? end @@ -36,10 +41,17 @@ module Gitlab end end + def variables_hash + strong_memoize(:variables_hash) do + variables.to_hash + end + end + def mutate(attrs = {}) self.class.new(**attrs) do |ctx| ctx.expandset = expandset ctx.execution_deadline = execution_deadline + ctx.logger = logger end end @@ -60,7 +72,7 @@ module Gitlab protected - attr_writer :expandset, :execution_deadline + attr_writer :expandset, :execution_deadline, :logger private diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 95f1a842c50..a5bf066c81f 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -30,6 +30,18 @@ module Gitlab def process return [] if locations.empty? + logger.instrument(:config_mapper_process) do + process_without_instrumentation + end + end + + private + + attr_reader :locations, :context + + delegate :expandset, :logger, to: :context + + def process_without_instrumentation locations .compact .map(&method(:normalize_location)) @@ -41,14 +53,14 @@ module Gitlab .map(&method(:select_first_matching)) end - private - - attr_reader :locations, :context - - delegate :expandset, to: :context + def normalize_location(location) + logger.instrument(:config_mapper_normalize) do + normalize_location_without_instrumentation(location) + end + end # convert location if String to canonical form - def normalize_location(location) + def normalize_location_without_instrumentation(location) if location.is_a?(String) expanded_location = expand_variables(location) normalize_location_string(expanded_location) @@ -58,6 +70,12 @@ module Gitlab end def verify_rules(location) + logger.instrument(:config_mapper_rules) do + verify_rules_without_instrumentation(location) + end + end + + def verify_rules_without_instrumentation(location) return unless Rules.new(location[:rules]).evaluate(context).pass? location @@ -72,6 +90,12 @@ module Gitlab end def expand_wildcard_paths(location) + logger.instrument(:config_mapper_wildcards) do + expand_wildcard_paths_without_instrumentation(location) + end + end + + def expand_wildcard_paths_without_instrumentation(location) # We only support local files for wildcard paths return location unless location[:local] && location[:local].include?('*') @@ -89,6 +113,12 @@ module Gitlab end def verify_duplicates!(location) + logger.instrument(:config_mapper_verify) do + verify_duplicates_without_instrumentation!(location) + end + end + + def verify_duplicates_without_instrumentation!(location) if expandset.count >= MAX_INCLUDES raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!" end @@ -106,6 +136,12 @@ module Gitlab end def select_first_matching(location) + logger.instrument(:config_mapper_select) do + select_first_matching_without_instrumentation(location) + end + end + + def select_first_matching_without_instrumentation(location) matching = FILE_CLASSES.map do |file_class| file_class.new(location, context) end.select(&:matching?) @@ -116,6 +152,12 @@ module Gitlab end def expand_variables(data) + logger.instrument(:config_mapper_variables) do + expand_variables_without_instrumentation(data) + end + end + + def expand_variables_without_instrumentation(data) if data.is_a?(String) expand(data) else @@ -137,7 +179,7 @@ module Gitlab end def expand(data) - ExpandVariables.expand(data, context.variables) + ExpandVariables.expand(data, -> { context.variables_hash }) end end end diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb index de69a1b1e8f..6a4aee26d80 100644 --- a/lib/gitlab/ci/config/external/processor.rb +++ b/lib/gitlab/ci/config/external/processor.rb @@ -7,10 +7,13 @@ module Gitlab class Processor IncludeError = Class.new(StandardError) + attr_reader :context, :logger + def initialize(values, context) @values = values @external_files = External::Mapper.new(values, context).process @content = {} + @logger = context.logger rescue External::Mapper::Error, OpenSSL::SSL::SSLError => e raise IncludeError, e.message @@ -29,13 +32,17 @@ module Gitlab def validate_external_files! @external_files.each do |file| - raise IncludeError, file.error_message unless file.valid? + logger.instrument(:config_external_verify) do + raise IncludeError, file.error_message unless file.valid? + end end end def merge_external_files! @external_files.each do |file| - @content.deep_merge!(file.to_hash) + logger.instrument(:config_external_merge) do + @content.deep_merge!(file.to_hash) + end end end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb deleted file mode 100644 index 51051b0490f..00000000000 --- a/lib/gitlab/ci/features.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - ## - # Deprecated: Ci::Features is a class that aggregates all CI/CD feature flags in one place. - # - module Features - # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` - # is a safe switch to disable the feature for a particular project when something went wrong, - # therefore it's not supposed to be enabled by default. - def self.disallow_to_create_merge_request_pipelines_in_target_project?(target_project) - ::Feature.enabled?(:ci_disallow_to_create_merge_request_pipelines_in_target_project, target_project) - end - - def self.accept_trace?(project) - ::Feature.enabled?(:ci_enable_live_trace, project) && - ::Feature.enabled?(:ci_accept_trace, project, type: :ops, default_enabled: true) - end - - def self.log_invalid_trace_chunks?(project) - ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) - end - - def self.gldropdown_tags_enabled? - ::Feature.enabled?(:gldropdown_tags, default_enabled: :yaml) - end - end - end -end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 73cfa02ce4b..651ed23eb25 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -34,7 +34,7 @@ module Gitlab end def file_name - "#{report_type.to_s.dasherize}-report-format.json" + report_type == :api_fuzzing ? "dast-report-format.json" : "#{report_type.to_s.dasherize}-report-format.json" end end diff --git a/lib/gitlab/ci/parsers/terraform/tfplan.rb b/lib/gitlab/ci/parsers/terraform/tfplan.rb index f9afa58f915..041d73cd914 100644 --- a/lib/gitlab/ci/parsers/terraform/tfplan.rb +++ b/lib/gitlab/ci/parsers/terraform/tfplan.rb @@ -34,7 +34,7 @@ module Gitlab def job_details(job) { 'job_id' => job.id.to_s, - 'job_name' => job.options.dig(:artifacts, :name).to_s, + 'job_name' => job.name, 'job_path' => Gitlab::Routing.url_helpers.project_job_path(job.project, job) } end diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb index 9b494f3a7ec..28567437719 100644 --- a/lib/gitlab/ci/pipeline/chain/base.rb +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -7,7 +7,7 @@ module Gitlab class Base attr_reader :pipeline, :command, :config - delegate :project, :current_user, :parent_pipeline, to: :command + delegate :project, :current_user, :parent_pipeline, :logger, to: :command def initialize(pipeline, command) @pipeline = pipeline diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index 6feb693221b..bbdc6b65b96 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -21,6 +21,10 @@ module Gitlab merge_request: @command.merge_request, external_pull_request: @command.external_pull_request, locked: @command.project.default_pipeline_lock) + + # Initialize the feature flag at the beginning of the pipeline creation process + # so that the flag references in the latter chains return the same value. + @pipeline.create_deployment_in_separate_transaction? end def break? diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index beb8801096b..c466b8b36d0 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -11,7 +11,7 @@ module Gitlab :trigger_request, :schedule, :merge_request, :external_pull_request, :ignore_skip_ci, :save_incompleted, :seeds_block, :variables_attributes, :push_options, - :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, + :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, :logger, # These attributes are set by Chains during processing: :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed ) do @@ -88,7 +88,14 @@ module Gitlab @metrics ||= ::Gitlab::Ci::Pipeline::Metrics end + def logger + self[:logger] ||= ::Gitlab::Ci::Pipeline::Logger.new(project: project) + end + def observe_step_duration(step_class, duration) + step = step_class.name.underscore.parameterize(separator: '_') + logger.observe("pipeline_step_#{step}_duration_s", duration) + if Feature.enabled?(:ci_pipeline_creation_step_duration_tracking, type: :ops, default_enabled: :yaml) metrics.pipeline_creation_step_duration_histogram .observe({ step: step_class.name }, duration.seconds) @@ -96,11 +103,15 @@ module Gitlab end def observe_creation_duration(duration) + logger.observe(:pipeline_creation_duration_s, duration) + metrics.pipeline_creation_duration_histogram .observe({}, duration.seconds) end def observe_pipeline_size(pipeline) + logger.observe(:pipeline_size_count, pipeline.total_size) + metrics.pipeline_size_histogram .observe({ source: pipeline.source.to_s }, pipeline.total_size) end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index f3c937ddd28..64d1b001e3c 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -11,16 +11,21 @@ module Gitlab def perform! raise ArgumentError, 'missing config content' unless @command.config_content - result = ::Gitlab::Ci::YamlProcessor.new( - @command.config_content, { - project: project, - pipeline: @pipeline, - sha: @pipeline.sha, - source: @pipeline.source, - user: current_user, - parent_pipeline: parent_pipeline - } - ).execute + result = logger.instrument(:pipeline_config_process) do + processor = ::Gitlab::Ci::YamlProcessor.new( + @command.config_content, { + project: project, + pipeline: @pipeline, + sha: @pipeline.sha, + source: @pipeline.source, + user: current_user, + parent_pipeline: parent_pipeline, + logger: logger + } + ) + + processor.execute + end add_warnings_to_pipeline(result.warnings) diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 81ef3bb074d..15b0ff3c04d 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -6,10 +6,18 @@ module Gitlab module Chain class Create < Chain::Base include Chain::Helpers + include Gitlab::Utils::StrongMemoize def perform! - BulkInsertableAssociations.with_bulk_insert do - pipeline.save! + logger.instrument(:pipeline_save) do + BulkInsertableAssociations.with_bulk_insert do + tags = extract_tag_list_by_status + + pipeline.transaction do + pipeline.save! + CommitStatus.bulk_insert_tags!(statuses, tags) if bulk_insert_tags? + end + end end rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") @@ -18,6 +26,37 @@ module Gitlab def break? !pipeline.persisted? end + + private + + def statuses + strong_memoize(:statuses) do + pipeline.stages.flat_map(&:statuses) + end + end + + # We call `job.tag_list=` to assign tags to the jobs from the + # Chain::Seed step which uses the `@tag_list` instance variable to + # store them on the record. We remove them here because we want to + # bulk insert them, otherwise they would be inserted and assigned one + # by one with callbacks. We must use `remove_instance_variable` + # because having the instance variable defined would still run the callbacks + def extract_tag_list_by_status + return {} unless bulk_insert_tags? + + statuses.each.with_object({}) do |job, acc| + tag_list = job.clear_memoization(:tag_list) + next unless tag_list + + acc[job.name] = tag_list + end + end + + def bulk_insert_tags? + strong_memoize(:bulk_insert_tags) do + ::Feature.enabled?(:ci_bulk_insert_tags, project, default_enabled: :yaml) + end + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/create_deployments.rb b/lib/gitlab/ci/pipeline/chain/create_deployments.rb new file mode 100644 index 00000000000..b92aa89d62d --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/create_deployments.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class CreateDeployments < Chain::Base + DeploymentCreationError = Class.new(StandardError) + + def perform! + return unless pipeline.create_deployment_in_separate_transaction? + + create_deployments! + end + + def break? + false + end + + private + + def create_deployments! + pipeline.stages.map(&:statuses).flatten.map(&method(:create_deployment)) + end + + def create_deployment(build) + return unless build.instance_of?(::Ci::Build) && build.persisted_environment.present? + + deployment = ::Gitlab::Ci::Pipeline::Seed::Deployment + .new(build, build.persisted_environment).to_resource + + return unless deployment + + deployment.deployable = build + deployment.save! + rescue ActiveRecord::RecordInvalid => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception( + DeploymentCreationError.new(e.message), build_id: build.id) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb new file mode 100644 index 00000000000..424e1d87fb4 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class EnsureEnvironments < Chain::Base + def perform! + return unless pipeline.create_deployment_in_separate_transaction? + + pipeline.stages.map(&:statuses).flatten.each(&method(:ensure_environment)) + end + + def break? + false + end + + private + + def ensure_environment(build) + return unless build.instance_of?(::Ci::Build) && build.has_environment? + + environment = ::Gitlab::Ci::Pipeline::Seed::Environment.new(build).to_resource + + if environment.persisted? + build.persisted_environment = environment + build.assign_attributes(metadata_attributes: { expanded_environment_name: environment.name }) + else + build.assign_attributes(status: :failed, failure_reason: :environment_creation_failure) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/ensure_resource_groups.rb b/lib/gitlab/ci/pipeline/chain/ensure_resource_groups.rb new file mode 100644 index 00000000000..f4e5e6e467a --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/ensure_resource_groups.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class EnsureResourceGroups < Chain::Base + def perform! + return unless pipeline.create_deployment_in_separate_transaction? + + pipeline.stages.map(&:statuses).flatten.each(&method(:ensure_resource_group)) + end + + def break? + false + end + + private + + def ensure_resource_group(processable) + return unless processable.is_a?(::Ci::Processable) + + key = processable.options.delete(:resource_group_key) + + resource_group = ::Gitlab::Ci::Pipeline::Seed::Processable::ResourceGroup + .new(processable, key).to_resource + + processable.resource_group = resource_group + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index ef7447fa83d..356eeb76908 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -13,8 +13,10 @@ module Gitlab raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result # Allocate next IID. This operation must be outside of transactions of pipeline creations. - pipeline.ensure_project_iid! - pipeline.ensure_ci_ref! + logger.instrument(:pipeline_allocate_seed_attributes) do + pipeline.ensure_project_iid! + pipeline.ensure_ci_ref! + end # Protect the pipeline. This is assigned in Populate instead of # Build to prevent erroring out on ambiguous refs. @@ -23,8 +25,12 @@ module Gitlab ## # Gather all runtime build/stage errors # - if pipeline_seed.errors - return error(pipeline_seed.errors.join("\n"), config_error: true) + seed_errors = logger.instrument(:pipeline_seed_evaluation) do + pipeline_seed.errors + end + + if seed_errors + return error(seed_errors.join("\n"), config_error: true) end @command.pipeline_seed = pipeline_seed @@ -38,8 +44,11 @@ module Gitlab def pipeline_seed strong_memoize(:pipeline_seed) do - stages_attributes = @command.yaml_processor_result.stages_attributes - Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes) + logger.instrument(:pipeline_seed_initialization) do + stages_attributes = @command.yaml_processor_result.stages_attributes + + Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes) + end end end @@ -48,9 +57,11 @@ module Gitlab end def root_variables - ::Gitlab::Ci::Variables::Helpers.merge_variables( - @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables - ) + logger.instrument(:pipeline_seed_merge_variables) do + ::Gitlab::Ci::Variables::Helpers.merge_variables( + @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables + ) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index 845eb6c7a42..de147914850 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -9,30 +9,36 @@ module Gitlab @pipeline = pipeline @command = command @sequence = sequence - @start = Time.now + @start = current_monotonic_time end def build! @sequence.each do |step_class| - step_start = ::Gitlab::Metrics::System.monotonic_time + step_start = current_monotonic_time step = step_class.new(@pipeline, @command) step.perform! @command.observe_step_duration( step_class, - ::Gitlab::Metrics::System.monotonic_time - step_start + current_monotonic_time - step_start ) break if step.break? end - @command.observe_creation_duration(Time.now - @start) + @command.observe_creation_duration(current_monotonic_time - @start) @command.observe_pipeline_size(@pipeline) @command.observe_jobs_count_in_alive_pipelines @pipeline end + + private + + def current_monotonic_time + ::Gitlab::Metrics::System.monotonic_time + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 28ba1cd4d47..85bd5f0a7c1 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -113,7 +113,7 @@ module Gitlab name: build[:name], stage: build[:stage], image: build.dig(:options, :image, :name), - services: build.dig(:options, :services)&.map { |service| service[:name] }, + services: service_names(build), script: [ build.dig(:options, :before_script), build.dig(:options, :script), @@ -122,6 +122,14 @@ module Gitlab } end + def service_names(build) + services = build.dig(:options, :services) + + return unless services + + services.compact.map { |service| service[:name] } + end + def stages_attributes command.yaml_processor_result.stages_attributes end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb index 11d2010909f..6da88fd287e 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -9,7 +9,11 @@ module Gitlab PATTERN = /\$(?<name>\w+)/.freeze def evaluate(variables = {}) - variables.with_indifferent_access.fetch(@value, nil) + unless variables.is_a?(ActiveSupport::HashWithIndifferentAccess) + variables = variables.with_indifferent_access + end + + variables.fetch(@value, nil) end def inspect diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index 5f3310dd668..4b13cae792e 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -9,7 +9,7 @@ module Gitlab def initialize(statement, variables = nil) @lexer = Expression::Lexer.new(statement) - @variables = variables&.to_hash + @variables = variables || {} end def parse_tree @@ -19,7 +19,7 @@ module Gitlab end def evaluate - parse_tree.evaluate(@variables.to_h) + parse_tree.evaluate(@variables) end def truthful? diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb new file mode 100644 index 00000000000..97f7dddd09a --- /dev/null +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + class Logger + include ::Gitlab::Utils::StrongMemoize + + def self.current_monotonic_time + ::Gitlab::Metrics::System.monotonic_time + end + + def initialize(project:, destination: Gitlab::AppJsonLogger) + @started_at = current_monotonic_time + @project = project + @destination = destination + @log_conditions = [] + + yield(self) if block_given? + end + + def log_when(&block) + log_conditions.push(block) + end + + def instrument(operation) + return yield unless enabled? + + raise ArgumentError, 'block not given' unless block_given? + + op_started_at = current_monotonic_time + + result = yield + + observe("#{operation}_duration_s", current_monotonic_time - op_started_at) + + result + end + + def observe(operation, value) + return unless enabled? + + observations[operation.to_s].push(value) + end + + def commit(pipeline:, caller:) + return unless log? + + attributes = { + class: self.class.name.to_s, + pipeline_creation_caller: caller, + project_id: project.id, + pipeline_id: pipeline.id, + pipeline_persisted: pipeline.persisted?, + pipeline_source: pipeline.source, + pipeline_creation_service_duration_s: age + }.stringify_keys.merge(observations_hash) + + destination.info(attributes) + end + + def observations_hash + observations.transform_values do |values| + next if values.empty? + + { + 'count' => values.size, + 'min' => values.min, + 'max' => values.max, + 'avg' => values.sum / values.size + } + end.compact + end + + private + + attr_reader :project, :destination, :started_at, :log_conditions + delegate :current_monotonic_time, to: :class + + def age + current_monotonic_time - started_at + end + + def log? + return false unless enabled? + return true if log_conditions.empty? + + log_conditions.any? { |cond| cond.call(observations) } + end + + def enabled? + strong_memoize(:enabled) do + ::Feature.enabled?(:ci_pipeline_creation_logger, project, type: :ops, default_enabled: :yaml) + end + end + + def observations + @observations ||= Hash.new { |hash, key| hash[key] = [] } + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 72837b8ec22..762292f0fa3 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -7,8 +7,6 @@ module Gitlab class Build < Seed::Base include Gitlab::Utils::StrongMemoize - EnvironmentCreationFailure = Class.new(StandardError) - delegate :dig, to: :@seed_attributes def initialize(context, attributes, stages_for_needs_lookup = []) @@ -30,7 +28,7 @@ module Gitlab @except = Gitlab::Ci::Build::Policy .fabricate(attributes.delete(:except)) @rules = Gitlab::Ci::Build::Rules - .new(attributes.delete(:rules), default_when: 'on_success') + .new(attributes.delete(:rules), default_when: attributes[:when]) @cache = Gitlab::Ci::Build::Cache .new(attributes.delete(:cache), @pipeline) @@ -80,7 +78,7 @@ module Gitlab def to_resource strong_memoize(:resource) do processable = initialize_processable - assign_resource_group(processable) + assign_resource_group(processable) unless @pipeline.create_deployment_in_separate_transaction? processable end end @@ -90,7 +88,9 @@ module Gitlab ::Ci::Bridge.new(attributes) else ::Ci::Build.new(attributes).tap do |build| - build.assign_attributes(self.class.deployment_attributes_for(build)) + unless @pipeline.create_deployment_in_separate_transaction? + build.assign_attributes(self.class.deployment_attributes_for(build)) + end end end end @@ -107,20 +107,7 @@ module Gitlab environment = Seed::Environment.new(build).to_resource if environment.nil? unless environment.persisted? - if Feature.enabled?(:surface_environment_creation_failure, build.project, default_enabled: :yaml) && - Feature.disabled?(:surface_environment_creation_failure_override, build.project) - return { status: :failed, failure_reason: :environment_creation_failure } - end - - # If there is a validation error on environment creation, such as - # the name contains invalid character, the build falls back to a - # non-environment job. - Gitlab::ErrorTracking.track_exception( - EnvironmentCreationFailure.new, - project_id: build.project_id, - reason: environment.errors.full_messages.to_sentence) - - return { environment: nil } + return { status: :failed, failure_reason: :environment_creation_failure } end build.persisted_environment = environment @@ -215,12 +202,14 @@ module Gitlab end def runner_tags - { tag_list: evaluate_runner_tags }.compact + strong_memoize(:runner_tags) do + { tag_list: evaluate_runner_tags }.compact + end end def evaluate_runner_tags - @seed_attributes[:tag_list]&.map do |tag| - ExpandVariables.expand_existing(tag, evaluate_context.variables) + @seed_attributes.delete(:tag_list)&.map do |tag| + ExpandVariables.expand_existing(tag, -> { evaluate_context.variables_hash }) end end diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb index 417319cb5be..3e4a44a2e70 100644 --- a/lib/gitlab/ci/reports/security/report.rb +++ b/lib/gitlab/ci/reports/security/report.rb @@ -51,7 +51,7 @@ module Gitlab def replace_with!(other) instance_variables.each do |ivar| - instance_variable_set(ivar, other.public_send(ivar.to_s[1..-1])) # rubocop:disable GitlabSecurity/PublicSend + instance_variable_set(ivar, other.public_send(ivar.to_s[1..])) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb index d66d4b20bba..eaa87157716 100644 --- a/lib/gitlab/ci/status/bridge/common.rb +++ b/lib/gitlab/ci/status/bridge/common.rb @@ -16,7 +16,11 @@ module Gitlab def details_path return unless can?(user, :read_pipeline, downstream_pipeline) - project_pipeline_path(downstream_project, downstream_pipeline) + if Feature.enabled?(:ci_retry_downstream_pipeline, subject.project, default_enabled: :yaml) + project_job_path(subject.project, subject) + else + project_pipeline_path(downstream_project, downstream_pipeline) + end end def has_action? diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index b0f12ff7517..5dd28157b1f 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -34,7 +34,8 @@ module Gitlab no_matching_runner: 'no matching runner available', trace_size_exceeded: 'log size limit exceeded', builds_disabled: 'project builds are disabled', - environment_creation_failure: 'environment creation failure' + environment_creation_failure: 'environment creation failure', + deployment_rejected: 'deployment rejected' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/tags/bulk_insert.rb b/lib/gitlab/ci/tags/bulk_insert.rb new file mode 100644 index 00000000000..a299df7e2d9 --- /dev/null +++ b/lib/gitlab/ci/tags/bulk_insert.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Tags + class BulkInsert + TAGGINGS_BATCH_SIZE = 1000 + TAGS_BATCH_SIZE = 500 + + def initialize(statuses, tag_list_by_status) + @statuses = statuses + @tag_list_by_status = tag_list_by_status + end + + def insert! + return false if tag_list_by_status.empty? + + persist_build_tags! + end + + private + + attr_reader :statuses, :tag_list_by_status + + def persist_build_tags! + all_tags = tag_list_by_status.values.flatten.uniq.reject(&:blank?) + tag_records_by_name = create_tags(all_tags).index_by(&:name) + taggings = build_taggings_attributes(tag_records_by_name) + + return false if taggings.empty? + + taggings.each_slice(TAGGINGS_BATCH_SIZE) do |taggings_slice| + ActsAsTaggableOn::Tagging.insert_all!(taggings) + end + + true + end + + # rubocop: disable CodeReuse/ActiveRecord + def create_tags(tags) + existing_tag_records = ActsAsTaggableOn::Tag.where(name: tags).to_a + missing_tags = detect_missing_tags(tags, existing_tag_records) + return existing_tag_records if missing_tags.empty? + + missing_tags + .map { |tag| { name: tag } } + .each_slice(TAGS_BATCH_SIZE) do |tags_attributes| + ActsAsTaggableOn::Tag.insert_all!(tags_attributes) + end + + ActsAsTaggableOn::Tag.where(name: tags).to_a + end + # rubocop: enable CodeReuse/ActiveRecord + + def build_taggings_attributes(tag_records_by_name) + taggings = statuses.flat_map do |status| + tag_list = tag_list_by_status[status.name] + next unless tag_list + + tags = tag_records_by_name.values_at(*tag_list) + taggings_for(tags, status) + end + + taggings.compact! + taggings + end + + def taggings_for(tags, status) + tags.map do |tag| + { + tag_id: tag.id, + taggable_type: CommitStatus.name, + taggable_id: status.id, + created_at: Time.current, + context: 'tags' + } + end + end + + def detect_missing_tags(tags, tag_records) + if tags.size != tag_records.size + tags - tag_records.map(&:name) + else + [] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 89fd59d98f4..fddcc1492a8 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -53,6 +53,9 @@ variables: # KUBE_INGRESS_BASE_DOMAIN is the application deployment domain and should be set as a variable at the group or project level. # KUBE_INGRESS_BASE_DOMAIN: domain.example.com + # Allows Container-Scanning to correctly correlate image names when using Jobs/Build.gitlab-ci.yml + CS_DEFAULT_BRANCH_IMAGE: $CI_REGISTRY_IMAGE/$CI_DEFAULT_BRANCH:$CI_COMMIT_SHA + POSTGRES_USER: user POSTGRES_PASSWORD: testing-password POSTGRES_ENABLED: "true" diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml index b763705857e..fa7f6ffa2b7 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml @@ -24,7 +24,7 @@ kics-iac-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 0 + SAST_ANALYZER_IMAGE_TAG: 1 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml index 17ed1d2e87f..d32444833fb 100644 --- a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -9,6 +9,7 @@ pages: script: - mkdir .public - cp -r * .public + - rm -rf public - mv .public public artifacts: paths: diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index aec41c137a4..4917abf6ae9 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -23,7 +23,7 @@ cache: - venv/ before_script: - - python -V # Print out python version for debugging + - python --version # For debugging - pip install virtualenv - virtualenv venv - source venv/bin/activate diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml index ff8f9601189..de54d64dc42 100644 --- a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml @@ -13,8 +13,10 @@ before_script: - apt-get update -yqq - apt-get install apt-transport-https -yqq # Add keyserver for SBT - - echo "deb http://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list - - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 + - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list + - mkdir -p /root/.gnupg + - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823 + - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg # Install SBT - apt-get update -yqq - apt-get install sbt -yqq diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 544774d3b06..01041f4f056 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -11,11 +11,11 @@ variables: FUZZAPI_VERSION: "1" SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION} + FUZZAPI_IMAGE: api-fuzzing apifuzzer_fuzz: stage: fuzz - image: $FUZZAPI_IMAGE + image: $SECURE_ANALYZERS_PREFIX/$FUZZAPI_IMAGE:$FUZZAPI_VERSION allow_failure: true rules: - if: $API_FUZZING_DISABLED diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 89e6743b0e4..65a2b20d5c0 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -38,7 +38,8 @@ container_scanning: artifacts: reports: container_scanning: gl-container-scanning-report.json - paths: [gl-container-scanning-report.json] + dependency_scanning: gl-dependency-scanning-report.json + paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] dependencies: [] script: - gtcs scan diff --git a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml new file mode 100644 index 00000000000..57f1993921d --- /dev/null +++ b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml @@ -0,0 +1,52 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Dast-API.gitlab-ci.yml + +# To use this template, add the following to your .gitlab-ci.yml file: +# +# include: +# template: DAST-API.latest.gitlab-ci.yml +# +# You also need to add a `dast` stage to your `stages:` configuration. A sample configuration for DAST API: +# +# stages: +# - build +# - test +# - deploy +# - dast + +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html + +# Configure DAST API scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables + +variables: + # Setting this variable affects all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + # + DAST_API_VERSION: "1" + DAST_API_IMAGE: api-fuzzing + +dast_api: + stage: dast + image: $SECURE_ANALYZERS_PREFIX/$DAST_API_IMAGE:$DAST_API_VERSION + allow_failure: true + rules: + - if: $DAST_API_DISABLED + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH + script: + - /peach/analyzer-dast-api + artifacts: + when: always + paths: + - gl-assets + - gl-dast-api-report.json + - gl-*.log + reports: + dast: gl-dast-api-report.json diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml index 4f63ff93d4d..8f4a836441d 100644 --- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -13,7 +13,7 @@ stages: a11y: stage: accessibility - image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.0.1 + image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.1.1 script: /gitlab-accessibility.sh $a11y_urls allow_failure: true artifacts: diff --git a/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml index edd0fb0ba07..09fce67db2d 100644 --- a/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml @@ -1,4 +1,4 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: +# To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/dotNET-Core.yml @@ -14,10 +14,10 @@ # The 'latest' tag targets the latest available version of .NET Core SDK image. # If preferred, you can explicitly specify version of .NET Core (e.g. using '2.2-sdk' tag). # -# See other available tags for .NET Core: https://hub.docker.com/r/microsoft/dotnet +# See other available tags for .NET Core: https://hub.docker.com/_/microsoft-dotnet # Learn more about Docker tags: https://docs.docker.com/glossary/?term=tag # and the Docker itself: https://opensource.com/resources/what-docker -image: microsoft/dotnet:latest +image: mcr.microsoft.com/dotnet/sdk:latest # ### Define variables # diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index f4c5a06af97..3e2c2c7fc1a 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -12,7 +12,7 @@ module Gitlab def scoped_variables(job, environment:, dependencies:) Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.concat(predefined_variables(job)) if pipeline.predefined_vars_in_builder_enabled? + variables.concat(predefined_variables(job)) end end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 1aa3dbc5e47..296b0cfded2 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -29,10 +29,8 @@ module Gitlab run_logical_validations! Result.new(ci_config: @ci_config, warnings: @ci_config&.warnings) - rescue Gitlab::Ci::Config::ConfigError => e Result.new(ci_config: @ci_config, errors: [e.message], warnings: @ci_config&.warnings) - rescue ValidationError => e Result.new(ci_config: @ci_config, errors: [e.message], warnings: @ci_config&.warnings) end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 6215ba40ebe..f14279dca2d 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -92,6 +92,7 @@ module Gitlab script: job[:script], after_script: job[:after_script], environment: job[:environment], + resource_group_key: job[:resource_group], retry: job[:retry], parallel: job[:parallel], instance: job[:instance], diff --git a/lib/gitlab/config/entry/undefined.rb b/lib/gitlab/config/entry/undefined.rb index 5f708abc80c..55393890693 100644 --- a/lib/gitlab/config/entry/undefined.rb +++ b/lib/gitlab/config/entry/undefined.rb @@ -31,6 +31,10 @@ module Gitlab false end + def type + nil + end + def inspect "#<#{self.class.name}>" end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index bdae59e7e3c..87bc2ace204 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -36,6 +36,7 @@ module Gitlab if Rails.env.development? allow_webpack_dev_server(directives) allow_letter_opener(directives) + allow_snowplow_micro(directives) if Gitlab::Tracking.snowplow_micro_enabled? allow_customersdot(directives) if ENV['CUSTOMER_PORTAL_URL'].present? end @@ -138,13 +139,15 @@ module Gitlab append_to_directive(directives, 'frame_src', Gitlab::Utils.append_path(Gitlab.config.gitlab.url, '/rails/letter_opener/')) end + def self.allow_snowplow_micro(directives) + url = URI.join(Gitlab::Tracking::Destinations::SnowplowMicro.new.uri, '/').to_s + append_to_directive(directives, 'connect_src', url) + end + # Using 'self' in the CSP introduces several CSP bypass opportunities # for this reason we list the URLs where GitLab frames itself instead def self.allow_framed_gitlab_paths(directives) - # We need the version without trailing / for the sidekiq page itself - # and we also need the version with trailing / for "deeper" pages - # like /admin/sidekiq/busy - ['/admin/sidekiq', '/admin/sidekiq/', '/-/speedscope/index.html'].map do |path| + ['/admin/', '/assets/', '/-/speedscope/index.html'].map do |path| append_to_directive(directives, 'frame_src', Gitlab::Utils.append_path(Gitlab.config.gitlab.url, path)) end end diff --git a/lib/gitlab/content_security_policy/directives.rb b/lib/gitlab/content_security_policy/directives.rb index 30f3c16247d..3b958f8c92e 100644 --- a/lib/gitlab/content_security_policy/directives.rb +++ b/lib/gitlab/content_security_policy/directives.rb @@ -8,7 +8,7 @@ module Gitlab module ContentSecurityPolicy module Directives def self.frame_src - "https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com" + "https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com https://www.googletagmanager.com/ns.html" end def self.script_src diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index deaaab953aa..f48ba27888c 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -10,7 +10,7 @@ module Gitlab def initialize(contributor, current_user = nil) @contributor = contributor - @contributor_time_instance = local_time_instance(contributor.timezone) + @contributor_time_instance = local_timezone_instance(contributor.timezone).now @current_user = current_user @projects = if @contributor.include_private_contributions? ContributedProjectsFinder.new(@contributor).execute(@contributor) @@ -21,27 +21,33 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def activity_dates + return {} if @projects.empty? return @activity_dates if @activity_dates.present? + start_time = @contributor_time_instance.years_ago(1).beginning_of_day + end_time = @contributor_time_instance.end_of_day + + date_interval = "INTERVAL '#{@contributor_time_instance.utc_offset} seconds'" + # Can't use Event.contributions here because we need to check 3 different # project_features for the (currently) 3 different contribution types - date_from = @contributor_time_instance.now.years_ago(1) - repo_events = event_counts(date_from, :repository) - .having(action: :pushed) - issue_events = event_counts(date_from, :issues) - .having(action: [:created, :closed], target_type: "Issue") - mr_events = event_counts(date_from, :merge_requests) - .having(action: [:merged, :created, :closed], target_type: "MergeRequest") - note_events = event_counts(date_from, :merge_requests) - .having(action: :commented) + repo_events = events_created_between(start_time, end_time, :repository) + .where(action: :pushed) + issue_events = events_created_between(start_time, end_time, :issues) + .where(action: [:created, :closed], target_type: "Issue") + mr_events = events_created_between(start_time, end_time, :merge_requests) + .where(action: [:merged, :created, :closed], target_type: "MergeRequest") + note_events = events_created_between(start_time, end_time, :merge_requests) + .where(action: :commented) events = Event - .select(:project_id, :target_type, :action, :date, :total_amount) - .from_union([repo_events, issue_events, mr_events, note_events]) + .select("date(created_at + #{date_interval}) AS date", 'COUNT(*) AS num_events') + .from_union([repo_events, issue_events, mr_events, note_events], remove_duplicates: false) + .group(:date) .map(&:attributes) @activity_dates = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| - activities[event["date"]] += event["total_amount"] + activities[event["date"]] += event["num_events"] end end # rubocop: enable CodeReuse/ActiveRecord @@ -50,7 +56,7 @@ module Gitlab def events_by_date(date) return Event.none unless can_read_cross_project? - date_in_time_zone = date.in_time_zone(@contributor_time_instance) + date_in_time_zone = date.in_time_zone(@contributor_time_instance.time_zone) Event.contributions.where(author_id: contributor.id) .where(created_at: date_in_time_zone.beginning_of_day..date_in_time_zone.end_of_day) @@ -60,11 +66,11 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def starting_year - @contributor_time_instance.now.years_ago(1).year + @contributor_time_instance.years_ago(1).year end def starting_month - @contributor_time_instance.today.month + @contributor_time_instance.month end private @@ -74,29 +80,31 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord - def event_counts(date_from, feature) - t = Event.arel_table - + def events_created_between(start_time, end_time, feature) # re-running the contributed projects query in each union is expensive, so # use IN(project_ids...) instead. It's the intersection of two users so # the list will be (relatively) short @contributed_project_ids ||= projects.distinct.pluck(:id) - authed_projects = Project.where(id: @contributed_project_ids) - .with_feature_available_for_user(feature, current_user) - .reorder(nil) - .select(:id) - - conditions = t[:created_at].gteq(date_from.beginning_of_day) - .and(t[:created_at].lteq(@contributor_time_instance.today.end_of_day)) - .and(t[:author_id].eq(contributor.id)) - date_interval = "INTERVAL '#{@contributor_time_instance.now.utc_offset} seconds'" + # no need to check feature access of current user, if the contributor opted-in + # to show all private events anyway - otherwise they would get filtered out again + authed_projects = if @contributor.include_private_contributions? + @contributed_project_ids + else + ProjectFeature + .with_feature_available_for_user(feature, current_user) + .where(project_id: @contributed_project_ids) + .reorder(nil) + .select(:project_id) + end Event.reorder(nil) - .select(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval}) AS date", 'count(id) as total_amount') - .group(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval})") - .where(conditions) - .where("events.project_id in (#{authed_projects.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection + .select(:created_at) + .where( + author_id: contributor.id, + created_at: start_time..end_time, + events: { project_id: authed_projects } + ) end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb index ddb9d907640..058fe1c8139 100644 --- a/lib/gitlab/daemon.rb +++ b/lib/gitlab/daemon.rb @@ -2,16 +2,16 @@ module Gitlab class Daemon - def self.initialize_instance(*args) + def self.initialize_instance(...) raise "#{name} singleton instance already initialized" if @instance - @instance = new(*args) + @instance = new(...) Kernel.at_exit(&@instance.method(:stop)) @instance end - def self.instance(*args) - @instance ||= initialize_instance(*args) + def self.instance(...) + @instance ||= initialize_instance(...) end attr_reader :thread @@ -20,7 +20,8 @@ module Gitlab !thread.nil? end - def initialize + def initialize(**options) + @synchronous = options[:synchronous] @mutex = Mutex.new end @@ -43,6 +44,10 @@ module Gitlab Thread.current.name = thread_name run_thread end + + @thread.join if @synchronous + + @thread end end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 9c74e5d2ca8..f9c346a272f 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -63,6 +63,19 @@ module Gitlab }.compact.with_indifferent_access.freeze end + # This returns a list of base models with connection associated for a given gitlab_schema + def self.schemas_to_base_models + @schemas_to_base_models ||= { + gitlab_main: [self.database_base_models.fetch(:main)], + gitlab_ci: [self.database_base_models[:ci] || self.database_base_models.fetch(:main)], # use CI or fallback to main + gitlab_shared: self.database_base_models.values # all models + }.with_indifferent_access.freeze + end + + def self.all_database_names + DATABASE_NAMES + end + # We configure the database connection pool size automatically based on the # configured concurrency. We also add some headroom, to make sure we don't # run out of connections when more threads besides the 'user-facing' ones diff --git a/lib/gitlab/database/async_indexes/index_creator.rb b/lib/gitlab/database/async_indexes/index_creator.rb index 994a1deba57..2fb4cc8f675 100644 --- a/lib/gitlab/database/async_indexes/index_creator.rb +++ b/lib/gitlab/database/async_indexes/index_creator.rb @@ -47,6 +47,10 @@ module Gitlab TIMEOUT_PER_ACTION end + def lease_key + [super, async_index.connection_db_config.name].join('/') + end + def set_statement_timeout connection.execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT) yield diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 32765cb6a56..503172dd750 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -21,6 +21,7 @@ module Gitlab from_union([failed_jobs, self.stuck]) } + scope :except_succeeded, -> { where(status: self.statuses.except(:succeeded).values) } enum status: { pending: 0, diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index d9fc2ea48f6..2844cbe4a74 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -18,6 +18,8 @@ module Gitlab scope: [:job_class_name, :table_name, :column_name] } + validate :validate_batched_jobs_status, if: -> { status_changed? && finished? } + scope :queue_order, -> { order(id: :asc) } scope :queued, -> { where(status: [:active, :paused]) } scope :for_configuration, ->(job_class_name, table_name, column_name, job_arguments) do @@ -92,11 +94,11 @@ module Gitlab end def job_class_name=(class_name) - write_attribute(:job_class_name, class_name.demodulize) + write_attribute(:job_class_name, class_name.delete_prefix("::")) end def batch_class_name=(class_name) - write_attribute(:batch_class_name, class_name.demodulize) + write_attribute(:batch_class_name, class_name.delete_prefix("::")) end def migrated_tuple_count @@ -133,6 +135,12 @@ module Gitlab def optimize! BatchOptimizer.new(self).optimize! end + + private + + def validate_batched_jobs_status + errors.add(:batched_jobs, 'jobs need to be succeeded') if batched_jobs.except_succeeded.exists? + end end end end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index 870cf25984b..68a0c15480a 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -32,12 +32,12 @@ module Gitlab # Models using single-type inheritance (STI) don't work with # reltuple count estimates. We just have to ignore them and # use another strategy to compute them. - def non_sti_models + def non_sti_models(models) models.reject { |model| sti_model?(model) } end - def non_sti_table_names - non_sti_models.map(&:table_name) + def non_sti_table_names(models) + non_sti_models(models).map(&:table_name) end def sti_model?(model) @@ -45,21 +45,34 @@ module Gitlab model.base_class != model end - def table_names - models.map(&:table_name) + def table_to_model_mapping + @table_to_model_mapping ||= models.each_with_object({}) { |model, h| h[model.table_name] = model } + end + + def table_to_model(table_name) + table_to_model_mapping[table_name] end def size_estimates(check_statistics: true) - table_to_model = models.each_with_object({}) { |model, h| h[model.table_name] = model } - - # Querying tuple stats only works on the primary. Due to load balancing, the - # easiest way to do this is to start a transaction. - ActiveRecord::Base.transaction do # rubocop: disable Database/MultipleDatabases - get_statistics(non_sti_table_names, check_statistics: check_statistics).each_with_object({}) do |row, data| - model = table_to_model[row.table_name] - data[model] = row.estimate + results = {} + + models.group_by { |model| model.connection_db_config.name }.map do |db_name, models_for_db| + base_model = Gitlab::Database.database_base_models[db_name] + tables = non_sti_table_names(models_for_db) + + # Querying tuple stats only works on the primary. Due to load balancing, the + # easiest way to do this is to start a transaction. + base_model.transaction do + Gitlab::Database::SharedModel.using_connection(base_model.connection) do + get_statistics(tables, check_statistics: check_statistics).each do |row| + model = table_to_model(row.table_name) + results[model] = row.estimate + end + end end end + + results end # Generates the PostgreSQL query to return the tuples for tables diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb index 489bc0aacea..92c8de9aeac 100644 --- a/lib/gitlab/database/count/tablesample_count_strategy.rb +++ b/lib/gitlab/database/count/tablesample_count_strategy.rb @@ -61,7 +61,7 @@ module Gitlab #{where_clause(model)} SQL - rows = ActiveRecord::Base.connection.select_all(query) # rubocop: disable Database/MultipleDatabases + rows = model.connection.select_all(query) Integer(rows.first['count']) end diff --git a/lib/gitlab/database/gitlab_loose_foreign_keys.yml b/lib/gitlab/database/gitlab_loose_foreign_keys.yml new file mode 100644 index 00000000000..0343c054f23 --- /dev/null +++ b/lib/gitlab/database/gitlab_loose_foreign_keys.yml @@ -0,0 +1,69 @@ +ci_pipeline_chat_data: + - table: chat_names + column: chat_name_id + on_delete: async_delete +dast_scanner_profiles_builds: + - table: ci_builds + column: ci_build_id + on_delete: async_delete +dast_scanner_profiles_builds: + - table: ci_builds + column: ci_build_id + on_delete: async_delete +dast_profiles_pipelines: + - table: ci_pipelines + column: ci_pipeline_id + on_delete: async_delete +clusters_applications_runners: + - table: ci_runners + column: runner_id + on_delete: async_nullify +ci_namespace_mirrors: + - table: namespaces + column: namespace_id + on_delete: async_delete +ci_builds: + - table: users + column: user_id + on_delete: async_nullify +ci_pipelines: + - table: merge_requests + column: merge_request_id + on_delete: async_delete + - table: external_pull_requests + column: external_pull_request_id + on_delete: async_nullify + - table: users + column: user_id + on_delete: async_nullify +ci_project_mirrors: + - table: projects + column: project_id + on_delete: async_delete + - table: namespaces + column: namespace_id + on_delete: async_delete +packages_build_infos: + - table: ci_pipelines + column: pipeline_id + on_delete: async_nullify +packages_package_file_build_infos: + - table: ci_pipelines + column: pipeline_id + on_delete: async_nullify +pages_deployments: + - table: ci_builds + column: ci_build_id + on_delete: async_nullify +terraform_state_versions: + - table: ci_builds + column: ci_build_id + on_delete: async_nullify +merge_request_metrics: + - table: ci_pipelines + column: pipeline_id + on_delete: async_delete +project_pages_metadata: + - table: ci_job_artifacts + column: artifacts_archive_id + on_delete: async_nullify diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 66157e998a0..24c2d634780 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -1,4 +1,5 @@ abuse_reports: :gitlab_main +agent_activity_events: :gitlab_main agent_group_authorizations: :gitlab_main agent_project_authorizations: :gitlab_main alert_management_alert_assignees: :gitlab_main @@ -85,6 +86,7 @@ ci_job_token_project_scope_links: :gitlab_ci ci_job_variables: :gitlab_ci ci_minutes_additional_packs: :gitlab_ci ci_namespace_monthly_usages: :gitlab_ci +ci_namespace_mirrors: :gitlab_ci ci_pending_builds: :gitlab_ci ci_pipeline_artifacts: :gitlab_ci ci_pipeline_chat_data: :gitlab_ci @@ -96,6 +98,7 @@ ci_pipelines: :gitlab_ci ci_pipeline_variables: :gitlab_ci ci_platform_metrics: :gitlab_ci ci_project_monthly_usages: :gitlab_ci +ci_project_mirrors: :gitlab_ci ci_refs: :gitlab_ci ci_resource_groups: :gitlab_ci ci_resources: :gitlab_ci @@ -161,6 +164,7 @@ dependency_proxy_group_settings: :gitlab_main dependency_proxy_image_ttl_group_policies: :gitlab_main dependency_proxy_manifests: :gitlab_main deploy_keys_projects: :gitlab_main +deployment_approvals: :gitlab_main deployment_clusters: :gitlab_main deployment_merge_requests: :gitlab_main deployments: :gitlab_main @@ -249,6 +253,7 @@ incident_management_oncall_schedules: :gitlab_main incident_management_oncall_shifts: :gitlab_main incident_management_pending_alert_escalations: :gitlab_main incident_management_pending_issue_escalations: :gitlab_main +incident_management_timeline_events: :gitlab_main index_statuses: :gitlab_main in_product_marketing_emails: :gitlab_main insights: :gitlab_main @@ -260,6 +265,7 @@ issuable_severities: :gitlab_main issuable_slas: :gitlab_main issue_assignees: :gitlab_main issue_customer_relations_contacts: :gitlab_main +issue_emails: :gitlab_main issue_email_participants: :gitlab_main issue_links: :gitlab_main issue_metrics: :gitlab_main @@ -281,6 +287,7 @@ ldap_group_links: :gitlab_main lfs_file_locks: :gitlab_main lfs_objects: :gitlab_main lfs_objects_projects: :gitlab_main +lfs_object_states: :gitlab_main licenses: :gitlab_main lists: :gitlab_main list_user_preferences: :gitlab_main @@ -290,6 +297,7 @@ members: :gitlab_main merge_request_assignees: :gitlab_main merge_request_blocks: :gitlab_main merge_request_cleanup_schedules: :gitlab_main +merge_requests_compliance_violations: :gitlab_main merge_request_context_commit_diff_files: :gitlab_main merge_request_context_commits: :gitlab_main merge_request_diff_commits: :gitlab_main @@ -314,6 +322,7 @@ namespace_package_settings: :gitlab_main namespace_root_storage_statistics: :gitlab_main namespace_settings: :gitlab_main namespaces: :gitlab_main +namespaces_sync_events: :gitlab_main namespace_statistics: :gitlab_main note_diff_files: :gitlab_main notes: :gitlab_main @@ -363,6 +372,7 @@ packages_pypi_metadata: :gitlab_main packages_rubygems_metadata: :gitlab_main packages_tags: :gitlab_main pages_deployments: :gitlab_main +pages_deployment_states: :gitlab_main pages_domain_acme_orders: :gitlab_main pages_domains: :gitlab_main partitioned_foreign_keys: :gitlab_main @@ -408,6 +418,7 @@ project_repository_storage_moves: :gitlab_main project_security_settings: :gitlab_main project_settings: :gitlab_main projects: :gitlab_main +projects_sync_events: :gitlab_main project_statistics: :gitlab_main project_topics: :gitlab_main project_tracing_settings: :gitlab_main @@ -485,6 +496,7 @@ trending_projects: :gitlab_main u2f_registrations: :gitlab_main upcoming_reconciliations: :gitlab_main uploads: :gitlab_main +upload_states: :gitlab_main user_agent_details: :gitlab_main user_callouts: :gitlab_main user_canonical_emails: :gitlab_main @@ -526,6 +538,7 @@ vulnerability_issue_links: :gitlab_main vulnerability_occurrence_identifiers: :gitlab_main vulnerability_occurrence_pipelines: :gitlab_main vulnerability_occurrences: :gitlab_main +vulnerability_reads: :gitlab_main vulnerability_remediations: :gitlab_main vulnerability_scanners: :gitlab_main vulnerability_statistics: :gitlab_main diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb index 52eb0764ae3..e16db5af8ce 100644 --- a/lib/gitlab/database/load_balancing.rb +++ b/lib/gitlab/database/load_balancing.rb @@ -30,6 +30,10 @@ module Gitlab end end + def self.primary_only? + each_load_balancer.all?(&:primary_only?) + end + def self.release_hosts each_load_balancer(&:release_host) end diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb index da313361073..e769cb5c35c 100644 --- a/lib/gitlab/database/load_balancing/configuration.rb +++ b/lib/gitlab/database/load_balancing/configuration.rb @@ -107,7 +107,11 @@ module Gitlab hosts.any? || service_discovery_enabled? end + # This is disabled for Rake tasks to ensure e.g. database migrations + # always produce consistent results. def service_discovery_enabled? + return false if Gitlab::Runtime.rake? + service_discovery[:record].present? end diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb index b9acc36b4cc..5d91292b8de 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -6,6 +6,8 @@ module Gitlab class SidekiqServerMiddleware JobReplicaNotUpToDate = Class.new(StandardError) + MINIMUM_DELAY_INTERVAL_SECONDS = 0.8 + def call(worker, job, _queue) worker_class = worker.class strategy = select_load_balancing_strategy(worker_class, job) @@ -42,11 +44,15 @@ module Gitlab wal_locations = get_wal_locations(job) - return :primary_no_wal unless wal_locations + return :primary_no_wal if wal_locations.blank? + + # Happy case: we can read from a replica. + return replica_strategy(worker_class, job) if databases_in_sync?(wal_locations) + + sleep_if_needed(job) if databases_in_sync?(wal_locations) - # Happy case: we can read from a replica. - retried_before?(worker_class, job) ? :replica_retried : :replica + replica_strategy(worker_class, job) elsif can_retry?(worker_class, job) # Optimistic case: The worker allows retries and we have retries left. :retry @@ -56,17 +62,14 @@ module Gitlab end end - def get_wal_locations(job) - job['dedup_wal_locations'] || job['wal_locations'] || legacy_wal_location(job) - end + def sleep_if_needed(job) + remaining_delay = MINIMUM_DELAY_INTERVAL_SECONDS - (Time.current.to_f - job['created_at'].to_f) - # Already scheduled jobs could still contain legacy database write location. - # TODO: remove this in the next iteration - # https://gitlab.com/gitlab-org/gitlab/-/issues/338213 - def legacy_wal_location(job) - wal_location = job['database_write_location'] || job['database_replica_location'] + sleep remaining_delay if remaining_delay > 0 && remaining_delay < MINIMUM_DELAY_INTERVAL_SECONDS + end - { ::Gitlab::Database::MAIN_DATABASE_NAME.to_sym => wal_location } if wal_location + def get_wal_locations(job) + job['dedup_wal_locations'] || job['wal_locations'] end def load_balancing_available?(worker_class) @@ -79,6 +82,10 @@ module Gitlab worker_class.get_data_consistency == :delayed && not_yet_retried?(job) end + def replica_strategy(worker_class, job) + retried_before?(worker_class, job) ? :replica_retried : :replica + end + def retried_before?(worker_class, job) worker_class.get_data_consistency == :delayed && !not_yet_retried?(job) end diff --git a/lib/gitlab/database/load_balancing/sticking.rb b/lib/gitlab/database/load_balancing/sticking.rb index 834e9c6d3c6..8e5dc98e96e 100644 --- a/lib/gitlab/database/load_balancing/sticking.rb +++ b/lib/gitlab/database/load_balancing/sticking.rb @@ -123,21 +123,18 @@ module Gitlab def unstick(namespace, id) Gitlab::Redis::SharedState.with do |redis| redis.del(redis_key_for(namespace, id)) - redis.del(old_redis_key_for(namespace, id)) end end def set_write_location_for(namespace, id, location) Gitlab::Redis::SharedState.with do |redis| redis.set(redis_key_for(namespace, id), location, ex: EXPIRATION) - redis.set(old_redis_key_for(namespace, id), location, ex: EXPIRATION) end end def last_write_location_for(namespace, id) Gitlab::Redis::SharedState.with do |redis| - redis.get(redis_key_for(namespace, id)) || - redis.get(old_redis_key_for(namespace, id)) + redis.get(redis_key_for(namespace, id)) end end @@ -146,10 +143,6 @@ module Gitlab "database-load-balancing/write-location/#{name}/#{namespace}/#{id}" end - - def old_redis_key_for(namespace, id) - "database-load-balancing/write-location/#{namespace}/#{id}" - end end end end diff --git a/lib/gitlab/database/loose_foreign_keys.rb b/lib/gitlab/database/loose_foreign_keys.rb new file mode 100644 index 00000000000..1ecfb5ce47f --- /dev/null +++ b/lib/gitlab/database/loose_foreign_keys.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LooseForeignKeys + def self.definitions_by_table + @definitions_by_table ||= definitions.group_by(&:to_table).with_indifferent_access.freeze + end + + def self.definitions + @definitions ||= loose_foreign_keys_yaml.flat_map do |child_table_name, configs| + configs.map { |config| build_definition(child_table_name, config) } + end.freeze + end + + def self.build_definition(child_table_name, config) + parent_table_name = config.fetch('table') + + ActiveRecord::ConnectionAdapters::ForeignKeyDefinition.new( + child_table_name, + parent_table_name, + { + column: config.fetch('column'), + on_delete: config.fetch('on_delete').to_sym, + gitlab_schema: GitlabSchema.table_schema(child_table_name) + } + ) + end + + def self.loose_foreign_keys_yaml + @loose_foreign_keys_yaml ||= YAML.load_file(Rails.root.join('lib/gitlab/database/gitlab_loose_foreign_keys.yml')) + end + + private_class_method :build_definition + private_class_method :loose_foreign_keys_yaml + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 7dce4fa0ce2..4245dd80714 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -4,6 +4,7 @@ module Gitlab module Database module MigrationHelpers include Migrations::BackgroundMigrationHelpers + include Migrations::BatchedBackgroundMigrationHelpers include DynamicModelHelpers include RenameTableHelpers include AsyncIndexes::MigrationHelpers diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index bdaf0d35a83..8c33c41ce77 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -5,59 +5,7 @@ module Gitlab module Migrations module BackgroundMigrationHelpers BATCH_SIZE = 1_000 # Number of rows to process per job - SUB_BATCH_SIZE = 100 # Number of rows to process per sub-batch JOB_BUFFER_SIZE = 1_000 # Number of jobs to bulk queue at a time - BATCH_CLASS_NAME = 'PrimaryKeyBatchingStrategy' # Default batch class for batched migrations - BATCH_MIN_VALUE = 1 # Default minimum value for batched migrations - BATCH_MIN_DELAY = 2.minutes.freeze # Minimum delay between batched migrations - - # Bulk queues background migration jobs for an entire table, batched by ID range. - # "Bulk" meaning many jobs will be pushed at a time for efficiency. - # If you need a delay interval per job, then use `queue_background_migration_jobs_by_range_at_intervals`. - # - # model_class - The table being iterated over - # job_class_name - The background migration job class as a string - # batch_size - The maximum number of rows per job - # - # Example: - # - # class Route < ActiveRecord::Base - # include EachBatch - # self.table_name = 'routes' - # end - # - # bulk_queue_background_migration_jobs_by_range(Route, 'ProcessRoutes') - # - # Where the model_class includes EachBatch, and the background migration exists: - # - # class Gitlab::BackgroundMigration::ProcessRoutes - # def perform(start_id, end_id) - # # do something - # end - # end - def bulk_queue_background_migration_jobs_by_range(model_class, job_class_name, batch_size: BATCH_SIZE) - raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') - - jobs = [] - table_name = model_class.quoted_table_name - - model_class.each_batch(of: batch_size) do |relation| - start_id, end_id = relation.pluck("MIN(#{table_name}.id)", "MAX(#{table_name}.id)").first - - if jobs.length >= JOB_BUFFER_SIZE - # Note: This code path generally only helps with many millions of rows - # We push multiple jobs at a time to reduce the time spent in - # Sidekiq/Redis operations. We're using this buffer based approach so we - # don't need to run additional queries for every range. - bulk_migrate_async(jobs) - jobs.clear - end - - jobs << [job_class_name, [start_id, end_id]] - end - - bulk_migrate_async(jobs) unless jobs.empty? - end # Queues background migration jobs for an entire table in batches. # The default batching column used is the standard primary key `id`. @@ -137,6 +85,7 @@ module Gitlab # Requeue pending jobs previously queued with #queue_background_migration_jobs_by_range_at_intervals # # This method is useful to schedule jobs that had previously failed. + # It can only be used if the previous background migration used job tracking like the queue_background_migration_jobs_by_range_at_intervals helper. # # job_class_name - The background migration job class as a string # delay_interval - The duration between each job's scheduled time @@ -170,100 +119,6 @@ module Gitlab duration end - # Creates a batched background migration for the given table. A batched migration runs one job - # at a time, computing the bounds of the next batch based on the current migration settings and the previous - # batch bounds. Each job's execution status is tracked in the database as the migration runs. The given job - # class must be present in the Gitlab::BackgroundMigration module, and the batch class (if specified) must be - # present in the Gitlab::BackgroundMigration::BatchingStrategies module. - # - # If migration with same job_class_name, table_name, column_name, and job_aruments already exists, this helper - # will log an warning and not create a new one. - # - # job_class_name - The background migration job class as a string - # batch_table_name - The name of the table the migration will batch over - # batch_column_name - The name of the column the migration will batch over - # job_arguments - Extra arguments to pass to the job instance when the migration runs - # job_interval - The pause interval between each job's execution, minimum of 2 minutes - # batch_min_value - The value in the column the batching will begin at - # batch_max_value - The value in the column the batching will end at, defaults to `SELECT MAX(batch_column)` - # batch_class_name - The name of the class that will be called to find the range of each next batch - # batch_size - The maximum number of rows per job - # sub_batch_size - The maximum number of rows processed per "iteration" within the job - # - # - # *Returns the created BatchedMigration record* - # - # Example: - # - # queue_batched_background_migration( - # 'CopyColumnUsingBackgroundMigrationJob', - # :events, - # :id, - # job_interval: 2.minutes, - # other_job_arguments: ['column1', 'column2']) - # - # Where the the background migration exists: - # - # class Gitlab::BackgroundMigration::CopyColumnUsingBackgroundMigrationJob - # def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, *other_args) - # # do something - # end - # end - def queue_batched_background_migration( # rubocop:disable Metrics/ParameterLists - job_class_name, - batch_table_name, - batch_column_name, - *job_arguments, - job_interval:, - batch_min_value: BATCH_MIN_VALUE, - batch_max_value: nil, - batch_class_name: BATCH_CLASS_NAME, - batch_size: BATCH_SIZE, - sub_batch_size: SUB_BATCH_SIZE - ) - - if Gitlab::Database::BackgroundMigration::BatchedMigration.for_configuration(job_class_name, batch_table_name, batch_column_name, job_arguments).exists? - Gitlab::AppLogger.warn "Batched background migration not enqueued because it already exists: " \ - "job_class_name: #{job_class_name}, table_name: #{batch_table_name}, column_name: #{batch_column_name}, " \ - "job_arguments: #{job_arguments.inspect}" - return - end - - job_interval = BATCH_MIN_DELAY if job_interval < BATCH_MIN_DELAY - - batch_max_value ||= connection.select_value(<<~SQL) - SELECT MAX(#{connection.quote_column_name(batch_column_name)}) - FROM #{connection.quote_table_name(batch_table_name)} - SQL - - migration_status = batch_max_value.nil? ? :finished : :active - batch_max_value ||= batch_min_value - - migration = Gitlab::Database::BackgroundMigration::BatchedMigration.create!( - job_class_name: job_class_name, - table_name: batch_table_name, - column_name: batch_column_name, - job_arguments: job_arguments, - interval: job_interval, - min_value: batch_min_value, - max_value: batch_max_value, - batch_class_name: batch_class_name, - batch_size: batch_size, - sub_batch_size: sub_batch_size, - status: migration_status) - - # This guard is necessary since #total_tuple_count was only introduced schema-wise, - # after this migration helper had been used for the first time. - return migration unless migration.respond_to?(:total_tuple_count) - - # We keep track of the estimated number of tuples to reason later - # about the overall progress of a migration. - migration.total_tuple_count = Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate - migration.save! - - migration - end - # Force a background migration to complete. # # WARNING: This method will block the caller and move the background migration from an @@ -275,6 +130,7 @@ module Gitlab # 4. Optionally remove job tracking information. # # This method does not garauntee that all jobs completed successfully. + # It can only be used if the previous background migration used the queue_background_migration_jobs_by_range_at_intervals helper. def finalize_background_migration(class_name, delete_tracking_jobs: ['succeeded']) # Empty the sidekiq queue. Gitlab::BackgroundMigration.steal(class_name) diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb new file mode 100644 index 00000000000..dcaf7fad05f --- /dev/null +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + # BatchedBackgroundMigrations are a new approach to scheduling and executing background migrations, which uses + # persistent state in the database to track each migration. This avoids having to batch over an entire table and + # schedule a large number of sidekiq jobs upfront. It also provides for more flexibility as the migration runs, + # as it can be paused and restarted, and have configuration values like the batch size updated dynamically as the + # migration runs. + # + # For now, these migrations are not considered ready for general use, for more information see the tracking epic: + # https://gitlab.com/groups/gitlab-org/-/epics/6751 + module BatchedBackgroundMigrationHelpers + BATCH_SIZE = 1_000 # Number of rows to process per job + SUB_BATCH_SIZE = 100 # Number of rows to process per sub-batch + BATCH_CLASS_NAME = 'PrimaryKeyBatchingStrategy' # Default batch class for batched migrations + BATCH_MIN_VALUE = 1 # Default minimum value for batched migrations + BATCH_MIN_DELAY = 2.minutes.freeze # Minimum delay between batched migrations + + # Creates a batched background migration for the given table. A batched migration runs one job + # at a time, computing the bounds of the next batch based on the current migration settings and the previous + # batch bounds. Each job's execution status is tracked in the database as the migration runs. The given job + # class must be present in the Gitlab::BackgroundMigration module, and the batch class (if specified) must be + # present in the Gitlab::BackgroundMigration::BatchingStrategies module. + # + # If migration with same job_class_name, table_name, column_name, and job_aruments already exists, this helper + # will log an warning and not create a new one. + # + # job_class_name - The background migration job class as a string + # batch_table_name - The name of the table the migration will batch over + # batch_column_name - The name of the column the migration will batch over + # job_arguments - Extra arguments to pass to the job instance when the migration runs + # job_interval - The pause interval between each job's execution, minimum of 2 minutes + # batch_min_value - The value in the column the batching will begin at + # batch_max_value - The value in the column the batching will end at, defaults to `SELECT MAX(batch_column)` + # batch_class_name - The name of the class that will be called to find the range of each next batch + # batch_size - The maximum number of rows per job + # sub_batch_size - The maximum number of rows processed per "iteration" within the job + # + # *Returns the created BatchedMigration record* + # + # Example: + # + # queue_batched_background_migration( + # 'CopyColumnUsingBackgroundMigrationJob', + # :events, + # :id, + # job_interval: 2.minutes, + # other_job_arguments: ['column1', 'column2']) + # + # Where the the background migration exists: + # + # class Gitlab::BackgroundMigration::CopyColumnUsingBackgroundMigrationJob + # def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, *other_args) + # # do something + # end + # end + def queue_batched_background_migration( # rubocop:disable Metrics/ParameterLists + job_class_name, + batch_table_name, + batch_column_name, + *job_arguments, + job_interval:, + batch_min_value: BATCH_MIN_VALUE, + batch_max_value: nil, + batch_class_name: BATCH_CLASS_NAME, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + + if Gitlab::Database::BackgroundMigration::BatchedMigration.for_configuration(job_class_name, batch_table_name, batch_column_name, job_arguments).exists? + Gitlab::AppLogger.warn "Batched background migration not enqueued because it already exists: " \ + "job_class_name: #{job_class_name}, table_name: #{batch_table_name}, column_name: #{batch_column_name}, " \ + "job_arguments: #{job_arguments.inspect}" + return + end + + job_interval = BATCH_MIN_DELAY if job_interval < BATCH_MIN_DELAY + + batch_max_value ||= connection.select_value(<<~SQL) + SELECT MAX(#{connection.quote_column_name(batch_column_name)}) + FROM #{connection.quote_table_name(batch_table_name)} + SQL + + migration_status = batch_max_value.nil? ? :finished : :active + batch_max_value ||= batch_min_value + + migration = Gitlab::Database::BackgroundMigration::BatchedMigration.create!( + job_class_name: job_class_name, + table_name: batch_table_name, + column_name: batch_column_name, + job_arguments: job_arguments, + interval: job_interval, + min_value: batch_min_value, + max_value: batch_max_value, + batch_class_name: batch_class_name, + batch_size: batch_size, + sub_batch_size: sub_batch_size, + status: migration_status) + + # This guard is necessary since #total_tuple_count was only introduced schema-wise, + # after this migration helper had been used for the first time. + return migration unless migration.respond_to?(:total_tuple_count) + + # We keep track of the estimated number of tuples to reason later + # about the overall progress of a migration. + migration.total_tuple_count = Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate + end + migration.save! + + migration + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb index 6e5ffb74411..1f7e81cae84 100644 --- a/lib/gitlab/database/migrations/instrumentation.rb +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -14,11 +14,11 @@ module Gitlab @result_dir = result_dir end - def observe(version:, name:, &block) + def observe(version:, name:, connection:, &block) observation = Observation.new(version, name) observation.success = true - observers = observer_classes.map { |c| c.new(observation, @result_dir) } + observers = observer_classes.map { |c| c.new(observation, @result_dir, connection) } exception = nil diff --git a/lib/gitlab/database/migrations/observers/migration_observer.rb b/lib/gitlab/database/migrations/observers/migration_observer.rb index 106f8f1f829..0006af73f6c 100644 --- a/lib/gitlab/database/migrations/observers/migration_observer.rb +++ b/lib/gitlab/database/migrations/observers/migration_observer.rb @@ -7,8 +7,8 @@ module Gitlab class MigrationObserver attr_reader :connection, :observation, :output_dir - def initialize(observation, output_dir) - @connection = ActiveRecord::Base.connection + def initialize(observation, output_dir, connection) + @connection = connection @observation = observation @output_dir = output_dir end diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index b267a64256b..f0bac594119 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -69,7 +69,7 @@ module Gitlab instrumentation = Instrumentation.new(result_dir: result_dir) sorted_migrations.each do |migration| - instrumentation.observe(version: migration.version, name: migration.name) do + instrumentation.observe(version: migration.version, name: migration.name, connection: ActiveRecord::Migration.connection) do ActiveRecord::Migrator.new(direction, migration_context.migrations, migration_context.schema_migration, migration.version).run end end diff --git a/lib/gitlab/database/partitioning/detached_partition_dropper.rb b/lib/gitlab/database/partitioning/detached_partition_dropper.rb index 593824384b5..5e32ecad4ca 100644 --- a/lib/gitlab/database/partitioning/detached_partition_dropper.rb +++ b/lib/gitlab/database/partitioning/detached_partition_dropper.rb @@ -4,8 +4,6 @@ module Gitlab module Partitioning class DetachedPartitionDropper def perform - return unless Feature.enabled?(:drop_detached_partitions, default_enabled: :yaml) - Gitlab::AppLogger.info(message: "Checking for previously detached partitions to drop") Postgresql::DetachedPartition.ready_to_drop.find_each do |detached_partition| diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb index c93e775d7ed..9c8cccb3dc6 100644 --- a/lib/gitlab/database/partitioning/monthly_strategy.rb +++ b/lib/gitlab/database/partitioning/monthly_strategy.rb @@ -36,6 +36,10 @@ module Gitlab partitions end + def after_adding_partitions + # No-op, required by the partition manager + end + private def desired_partitions diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index 8742c0ff166..aa824dfbd2f 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -25,10 +25,8 @@ module Gitlab partitions_to_create = missing_partitions create(partitions_to_create) unless partitions_to_create.empty? - if Feature.enabled?(:partition_pruning, default_enabled: :yaml) - partitions_to_detach = extra_partitions - detach(partitions_to_detach) unless partitions_to_detach.empty? - end + partitions_to_detach = extra_partitions + detach(partitions_to_detach) unless partitions_to_detach.empty? end rescue StandardError => e Gitlab::AppLogger.error(message: "Failed to create / detach partition(s)", @@ -73,6 +71,8 @@ module Gitlab partition_name: partition.partition_name, table_name: partition.table) end + + model.partitioning_strategy.after_adding_partitions end end end diff --git a/lib/gitlab/database/partitioning/single_numeric_list_partition.rb b/lib/gitlab/database/partitioning/single_numeric_list_partition.rb new file mode 100644 index 00000000000..23ac73a0e53 --- /dev/null +++ b/lib/gitlab/database/partitioning/single_numeric_list_partition.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class SingleNumericListPartition + include Comparable + + def self.from_sql(table, partition_name, definition) + # A list partition can support multiple values, but we only support a single number + matches = definition.match(/FOR VALUES IN \('(?<value>\d+)'\)/) + + raise ArgumentError, 'Unknown partition definition' unless matches + + value = Integer(matches[:value]) + + new(table, value, partition_name: partition_name) + end + + attr_reader :table, :value + + def initialize(table, value, partition_name: nil ) + @table = table + @value = value + @partition_name = partition_name + end + + def partition_name + @partition_name || "#{table}_#{value}" + end + + def to_sql + <<~SQL + CREATE TABLE IF NOT EXISTS #{fully_qualified_partition} + PARTITION OF #{conn.quote_table_name(table)} + FOR VALUES IN (#{conn.quote(value)}) + SQL + end + + def to_detach_sql + <<~SQL + ALTER TABLE #{conn.quote_table_name(table)} + DETACH PARTITION #{fully_qualified_partition} + SQL + end + + def ==(other) + table == other.table && + partition_name == other.partition_name && + value == other.value + end + alias_method :eql?, :== + + def hash + [table, partition_name, value].hash + end + + def <=>(other) + return if table != other.table + + value <=> other.value + end + + private + + def fully_qualified_partition + "%s.%s" % [conn.quote_table_name(Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA), conn.quote_table_name(partition_name)] + end + + def conn + @conn ||= Gitlab::Database::SharedModel.connection + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/sliding_list_strategy.rb b/lib/gitlab/database/partitioning/sliding_list_strategy.rb new file mode 100644 index 00000000000..21b86b43ae7 --- /dev/null +++ b/lib/gitlab/database/partitioning/sliding_list_strategy.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class SlidingListStrategy + attr_reader :model, :partitioning_key, :next_partition_if, :detach_partition_if + + delegate :table_name, to: :model + + def initialize(model, partitioning_key, next_partition_if:, detach_partition_if:) + @model = model + @partitioning_key = partitioning_key + @next_partition_if = next_partition_if + @detach_partition_if = detach_partition_if + + ensure_partitioning_column_ignored! + end + + def current_partitions + Gitlab::Database::PostgresPartition.for_parent_table(table_name).map do |partition| + SingleNumericListPartition.from_sql(table_name, partition.name, partition.condition) + end.sort + end + + def missing_partitions + if no_partitions_exist? + [initial_partition] + elsif next_partition_if.call(active_partition.value) + [next_partition] + else + [] + end + end + + def initial_partition + SingleNumericListPartition.new(table_name, 1) + end + + def next_partition + SingleNumericListPartition.new(table_name, active_partition.value + 1) + end + + def extra_partitions + possibly_extra = current_partitions[0...-1] # Never consider the most recent partition + + possibly_extra.take_while { |p| detach_partition_if.call(p.value) } + end + + def after_adding_partitions + active_value = active_partition.value + model.connection.change_column_default(model.table_name, partitioning_key, active_value) + end + + def active_partition + # The current partitions list is sorted, so the last partition has the highest value + # This is the only partition that receives inserts. + current_partitions.last + end + + def no_partitions_exist? + current_partitions.empty? + end + + private + + def ensure_partitioning_column_ignored! + unless model.ignored_columns.include?(partitioning_key.to_s) + raise "Add #{partitioning_key} to #{model.name}.ignored_columns to use it with SlidingListStrategy" + end + end + end + end + end +end diff --git a/lib/gitlab/database/pg_class.rb b/lib/gitlab/database/pg_class.rb index 0ce9eebc14c..bd582d903c6 100644 --- a/lib/gitlab/database/pg_class.rb +++ b/lib/gitlab/database/pg_class.rb @@ -2,7 +2,7 @@ module Gitlab module Database - class PgClass < ActiveRecord::Base + class PgClass < SharedModel self.table_name = 'pg_class' def self.for_table(relname) diff --git a/lib/gitlab/database/postgres_hll/buckets.rb b/lib/gitlab/database/postgres_hll/buckets.rb index 429e823379f..76818bbf340 100644 --- a/lib/gitlab/database/postgres_hll/buckets.rb +++ b/lib/gitlab/database/postgres_hll/buckets.rb @@ -65,8 +65,7 @@ module Gitlab ).to_i if num_zero_buckets > 0 && num_uniques < 2.5 * TOTAL_BUCKETS - ((0.7213 / (1 + 1.079 / TOTAL_BUCKETS)) * (TOTAL_BUCKETS * - Math.log2(TOTAL_BUCKETS.to_f / num_zero_buckets))) + TOTAL_BUCKETS * Math.log(TOTAL_BUCKETS.to_f / num_zero_buckets) else num_uniques end diff --git a/lib/gitlab/database/query_analyzer.rb b/lib/gitlab/database/query_analyzer.rb index 0f285688876..2736f9d18dc 100644 --- a/lib/gitlab/database/query_analyzer.rb +++ b/lib/gitlab/database/query_analyzer.rb @@ -58,17 +58,15 @@ module Gitlab return unless parsed analyzers.each do |analyzer| - next if analyzer.suppressed? + next if analyzer.suppressed? && !analyzer.requires_tracking?(parsed) analyzer.analyze(parsed) - rescue StandardError => e + rescue StandardError, QueryAnalyzers::Base::QueryAnalyzerError => e # We catch all standard errors to prevent validation errors to introduce fatal errors in production Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end end - private - # Enable query analyzers def begin! analyzers = all_analyzers.select do |analyzer| @@ -77,7 +75,7 @@ module Gitlab true end - rescue StandardError => e + rescue StandardError, QueryAnalyzers::Base::QueryAnalyzerError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) false @@ -90,13 +88,15 @@ module Gitlab def end! enabled_analyzers.select do |analyzer| analyzer.end! - rescue StandardError => e + rescue StandardError, QueryAnalyzers::Base::QueryAnalyzerError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end Thread.current[:query_analyzer_enabled_analyzers] = nil end + private + def enabled_analyzers Thread.current[:query_analyzer_enabled_analyzers] end diff --git a/lib/gitlab/database/query_analyzers/base.rb b/lib/gitlab/database/query_analyzers/base.rb index e8066f7a706..0802d3c8013 100644 --- a/lib/gitlab/database/query_analyzers/base.rb +++ b/lib/gitlab/database/query_analyzers/base.rb @@ -4,10 +4,17 @@ module Gitlab module Database module QueryAnalyzers class Base + # `Exception` to ensure that is not easily rescued when running in test env + QueryAnalyzerError = Class.new(Exception) # rubocop:disable Lint/InheritException + def self.suppressed? Thread.current[self.suppress_key] end + def self.requires_tracking?(parsed) + false + end + def self.suppress=(value) Thread.current[self.suppress_key] = value end diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index 2233f3c4646..2e3db2a5c6e 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -4,7 +4,7 @@ module Gitlab module Database module QueryAnalyzers class PreventCrossDatabaseModification < Database::QueryAnalyzers::Base - CrossDatabaseModificationAcrossUnsupportedTablesError = Class.new(StandardError) + CrossDatabaseModificationAcrossUnsupportedTablesError = Class.new(QueryAnalyzerError) # This method will allow cross database modifications within the block # Example: @@ -36,29 +36,36 @@ module Gitlab Feature.enabled?(:detect_cross_database_modification, default_enabled: :yaml) end + def self.requires_tracking?(parsed) + # The transaction boundaries always needs to be tracked regardless of suppress behavior + self.transaction_begin?(parsed) || self.transaction_end?(parsed) + end + # rubocop:disable Metrics/AbcSize def self.analyze(parsed) - return if in_factory_bot_create? - database = ::Gitlab::Database.db_config_name(parsed.connection) sql = parsed.sql # We ignore BEGIN in tests as this is the outer transaction for # DatabaseCleaner - if sql.start_with?('SAVEPOINT') || (!Rails.env.test? && sql.start_with?('BEGIN')) + if self.transaction_begin?(parsed) context[:transaction_depth_by_db][database] += 1 return - elsif sql.start_with?('RELEASE SAVEPOINT', 'ROLLBACK TO SAVEPOINT') || (!Rails.env.test? && sql.start_with?('ROLLBACK', 'COMMIT')) + elsif self.transaction_end?(parsed) context[:transaction_depth_by_db][database] -= 1 - if context[:transaction_depth_by_db][database] <= 0 + if context[:transaction_depth_by_db][database] == 0 context[:modified_tables_by_db][database].clear + elsif context[:transaction_depth_by_db][database] < 0 + context[:transaction_depth_by_db][database] = 0 + raise CrossDatabaseModificationAcrossUnsupportedTablesError, "Misaligned cross-DB transactions discovered at query #{sql}. This could be a bug in #{self.class} or a valid issue to investigate. Read more at https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions ." end return end - return if context[:transaction_depth_by_db].values.all?(&:zero?) + return unless self.in_transaction? + return if in_factory_bot_create? # PgQuery might fail in some cases due to limited nesting: # https://github.com/pganalyze/pg_query/issues/209 @@ -97,6 +104,42 @@ module Gitlab end # rubocop:enable Metrics/AbcSize + def self.transaction_begin?(parsed) + # We ignore BEGIN or START in tests + unless Rails.env.test? + return true if transaction_stmt?(parsed, :TRANS_STMT_BEGIN) + return true if transaction_stmt?(parsed, :TRANS_STMT_START) + end + + # SAVEPOINT + return true if transaction_stmt?(parsed, :TRANS_STMT_SAVEPOINT) + + false + end + + def self.transaction_end?(parsed) + # We ignore ROLLBACK or COMMIT in tests + unless Rails.env.test? + return true if transaction_stmt?(parsed, :TRANS_STMT_COMMIT) + return true if transaction_stmt?(parsed, :TRANS_STMT_COMMIT_PREPARED) + return true if transaction_stmt?(parsed, :TRANS_STMT_ROLLBACK) + return true if transaction_stmt?(parsed, :TRANS_STMT_ROLLBACK_PREPARED) + end + + # RELEASE (SAVEPOINT) or ROLLBACK TO (SAVEPOINT) + return true if transaction_stmt?(parsed, :TRANS_STMT_RELEASE) + return true if transaction_stmt?(parsed, :TRANS_STMT_ROLLBACK_TO) + + false + end + + # Known kinds: https://github.com/pganalyze/pg_query/blob/f6588703deb9d7a94b87b34b7c3bab240087fbc4/ext/pg_query/include/nodes/parsenodes.h#L3050 + def self.transaction_stmt?(parsed, kind) + parsed.pg.tree.stmts.map(&:stmt).any? do |stmt| + stmt.node == :transaction_stmt && stmt.transaction_stmt.kind == kind + end + end + # We only raise in tests for now otherwise some features will be broken # in development. For now we've mostly only added allowlist based on # spec names. Until we have allowed all the violations inline we don't @@ -105,13 +148,21 @@ module Gitlab Rails.env.test? end + def self.in_transaction? + context[:transaction_depth_by_db].values.any?(&:positive?) + end + # We ignore execution in the #create method from FactoryBot # because it is not representative of real code we run in # production. There are far too many false positives caused # by instantiating objects in different `gitlab_schema` in a # FactoryBot `create`. def self.in_factory_bot_create? - Rails.env.test? && caller_locations.any? { |l| l.path.end_with?('lib/factory_bot/evaluation.rb') && l.label == 'create' } + Rails.env.test? && caller_locations.any? do |l| + l.path.end_with?('lib/factory_bot/evaluation.rb') && l.label == 'create' || + l.path.end_with?('lib/factory_bot/strategy/create.rb') || + l.path.end_with?('shoulda/matchers/active_record/validate_uniqueness_of_matcher.rb') && l.label == 'create_existing_record' + end end end end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 7a22e324bdb..6ffe14249f0 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -15,6 +15,26 @@ module Gitlab # on e.g. vacuum. REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30 + def self.enabled? + Feature.enabled?(:database_reindexing, type: :ops, default_enabled: :yaml) + end + + def self.invoke(database = nil) + Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name| + next if database && database.to_s != connection_name.to_s + + Gitlab::Database::SharedModel.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false) + + # Hack: Before we do actual reindexing work, create async indexes + Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops) + + automatic_reindexing + end + rescue StandardError => e + Gitlab::AppLogger.error(e) + raise + end + # Performs automatic reindexing for a limited number of indexes per call # 1. Consume from the explicit reindexing queue # 2. Apply bloat heuristic to find most bloated indexes and reindex those diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb index 13298f67ca9..3e4a83aa2e7 100644 --- a/lib/gitlab/database/reindexing/coordinator.rb +++ b/lib/gitlab/database/reindexing/coordinator.rb @@ -53,6 +53,10 @@ module Gitlab def lease_timeout TIMEOUT_PER_ACTION end + + def lease_key + [super, index.connection_db_config.name].join('/') + end end end end diff --git a/lib/gitlab/database/schema_cache_with_renamed_table.rb b/lib/gitlab/database/schema_cache_with_renamed_table.rb index 28123edd708..74900dc0d26 100644 --- a/lib/gitlab/database/schema_cache_with_renamed_table.rb +++ b/lib/gitlab/database/schema_cache_with_renamed_table.rb @@ -42,7 +42,7 @@ module Gitlab def renamed_tables_cache @renamed_tables ||= begin Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, new_name| - ActiveRecord::Base.connection.view_exists?(old_name) + connection.view_exists?(old_name) end end end diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb index 3d929c62933..9ddc5391689 100644 --- a/lib/gitlab/database/schema_helpers.rb +++ b/lib/gitlab/database/schema_helpers.rb @@ -25,6 +25,7 @@ module Gitlab CREATE TRIGGER #{name} #{fires} ON #{table_name} FOR EACH ROW + #{yield if block_given?} EXECUTE FUNCTION #{function_name}() SQL end diff --git a/lib/gitlab/database/shared_model.rb b/lib/gitlab/database/shared_model.rb index f31dbc01907..17d7886e8c8 100644 --- a/lib/gitlab/database/shared_model.rb +++ b/lib/gitlab/database/shared_model.rb @@ -39,6 +39,10 @@ module Gitlab Thread.current[:overriding_connection] = connection end end + + def connection_db_config + self.class.connection_db_config + end end end end diff --git a/lib/gitlab/database/type/json_pg_safe.rb b/lib/gitlab/database/type/json_pg_safe.rb new file mode 100644 index 00000000000..bbc207bd0d9 --- /dev/null +++ b/lib/gitlab/database/type/json_pg_safe.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Type + # Extends Rails' ActiveRecord::Type::Json data type to remove JSON + # encooded nullbytes `\u0000` to prevent PostgreSQL errors like + # `PG::UntranslatableCharacter: ERROR: unsupported Unicode escape + # sequence`. + # + # Example: + # + # class SomeModel < ApplicationRecord + # # some_model.a_field is of type `jsonb` + # attribute :a_field, Gitlab::Database::Type::JsonPgSafe.new + # end + class JsonPgSafe < ActiveRecord::Type::Json + def serialize(value) + super&.gsub('\u0000', '') + end + end + end + end +end diff --git a/lib/gitlab/diff/custom_diff.rb b/lib/gitlab/diff/custom_diff.rb new file mode 100644 index 00000000000..3928ece9281 --- /dev/null +++ b/lib/gitlab/diff/custom_diff.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true +module Gitlab + module Diff + module CustomDiff + class << self + def preprocess_before_diff(path, old_blob, new_blob) + return unless path.ends_with? '.ipynb' + + transformed_diff(old_blob&.data, new_blob&.data)&.tap do + transformed_for_diff(new_blob, old_blob) + Gitlab::AppLogger.info({ message: 'IPYNB_DIFF_GENERATED' }) + end + rescue IpynbDiff::InvalidNotebookError => e + Gitlab::ErrorTracking.log_exception(e) + nil + end + + def transformed_diff(before, after) + transformed_diff = IpynbDiff.diff(before, after, + diff_opts: { context: 5, include_diff_info: true }, + transform_options: { cell_decorator: :percent }, + raise_if_invalid_notebook: true) + strip_diff_frontmatter(transformed_diff) + end + + def transformed_blob_language(blob) + 'md' if transformed_for_diff?(blob) + end + + def transformed_blob_data(blob) + if transformed_for_diff?(blob) + IpynbDiff.transform(blob.data, + raise_errors: true, + options: { include_metadata: false, cell_decorator: :percent }) + end + end + + def strip_diff_frontmatter(diff_content) + diff_content.scan(/.*\n/)[2..]&.join('') if diff_content.present? + end + + def blobs_with_transformed_diffs + @blobs_with_transformed_diffs ||= {} + end + + def transformed_for_diff?(blob) + blobs_with_transformed_diffs[blob] + end + + def transformed_for_diff(*blobs) + blobs.each do |b| + blobs_with_transformed_diffs[b] = true if b + end + end + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 83f242ff902..d9860d9fb86 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -44,7 +44,11 @@ module Gitlab new_blob_lazy old_blob_lazy - preprocess_before_diff(diff) if Feature.enabled?(:jupyter_clean_diffs, repository.project, default_enabled: true) + diff.diff = Gitlab::Diff::CustomDiff.preprocess_before_diff(diff.new_path, old_blob_lazy, new_blob_lazy) || diff.diff if use_custom_diff? + end + + def use_custom_diff? + strong_memoize(:_custom_diff_enabled) { Feature.enabled?(:jupyter_clean_diffs, repository.project, default_enabled: true) } end def position(position_marker, position_type: :text) @@ -450,33 +454,6 @@ module Gitlab find_renderable_viewer_class(classes) end - def preprocess_before_diff(diff) - return unless diff.new_path.ends_with? '.ipynb' - - from = old_blob_lazy&.data - to = new_blob_lazy&.data - - transformed_diff = IpynbDiff.diff(from, to, - diff_opts: { context: 5, include_diff_info: true }, - transform_options: { cell_decorator: :percent }, - raise_if_invalid_notebook: true) - new_diff = strip_diff_frontmatter(transformed_diff) - - if new_diff - diff.diff = new_diff - new_blob_lazy.transformed_for_diff = true if new_blob_lazy - old_blob_lazy.transformed_for_diff = true if old_blob_lazy - end - - Gitlab::AppLogger.info({ message: new_diff ? 'IPYNB_DIFF_GENERATED' : 'IPYNB_DIFF_NIL' }) - rescue IpynbDiff::InvalidNotebookError => e - Gitlab::ErrorTracking.log_exception(e) - end - - def strip_diff_frontmatter(diff_content) - diff_content.scan(/.*\n/)[2..-1]&.join('') if diff_content.present? - end - def alternate_viewer_class return unless viewer.instance_of?(DiffViewer::Renamed) diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb index 692186fc323..b459e3f6619 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb @@ -13,7 +13,7 @@ module Gitlab super(merge_request_diff, project: merge_request_diff.project, - diff_options: merged_diff_options(diff_options), + diff_options: diff_options, diff_refs: merge_request_diff.diff_refs, fallback_diff_refs: merge_request_diff.fallback_diff_refs) end @@ -68,13 +68,6 @@ module Gitlab diff_stats_cache.read || super end end - - def merged_diff_options(diff_options) - project = @merge_request_diff.project - max_diff_options = ::Commit.max_diff_options(project: project).merge(project: project) - - diff_options.present? ? diff_options.merge(max_diff_options) : max_diff_options - end end end end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index aedcfe3cb40..47f3324752d 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -153,13 +153,10 @@ module Gitlab blob.load_all_data! - return blob.present.highlight_transformed.lines if Feature.enabled?(:jupyter_clean_diffs, @project, default_enabled: true) - blob.present.highlight.lines end def blobs_too_large? - return false unless Feature.enabled?(:limited_diff_highlighting, project, default_enabled: :yaml) return true if Gitlab::Highlight.too_large?(diff_file.old_blob&.size) Gitlab::Highlight.too_large?(diff_file.new_blob&.size) diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index f70618195d0..802da50cfc6 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -6,8 +6,8 @@ module Gitlab attr_accessor :old_line, :new_line, :offset def initialize(old_line, new_line, offset: 0) - @old_line = old_line[offset..-1] - @new_line = new_line[offset..-1] + @old_line = old_line[offset..] + @new_line = new_line[offset..] @offset = offset end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 4b490ae0d26..7b31dd9926b 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -60,7 +60,7 @@ module Gitlab current_user: author, params: { title: mail.subject, - description: message_including_reply + description: message_including_reply_or_only_quotes }, spam_params: nil ).execute diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb index a717509e24d..43cbc49b9d2 100644 --- a/lib/gitlab/email/handler/reply_processing.rb +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -35,6 +35,10 @@ module Gitlab @message_with_reply ||= process_message(trim_reply: false) end + def message_including_reply_or_only_quotes + @message_including_reply_or_only_quotes ||= process_message(trim_reply: false, allow_only_quotes: true) + end + def message_with_appended_reply @message_with_appended_reply ||= process_message(append_reply: true) end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 8d73aa842be..71b1d4ed8f9 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -32,11 +32,11 @@ module Gitlab def execute raise ProjectNotFound if project.nil? - create_issue! + create_issue_or_note if from_address add_email_participant - send_thank_you_email + send_thank_you_email unless reply_email? end end @@ -82,6 +82,14 @@ module Gitlab project.present? && slug == project.full_path_slug end + def create_issue_or_note + if reply_email? + create_note_from_reply_email + else + create_issue! + end + end + def create_issue! @issue = ::Issues::CreateService.new( project: project, @@ -97,18 +105,42 @@ module Gitlab raise InvalidIssueError unless @issue.persisted? + begin + ::Issue::Email.create!(issue: @issue, email_message_id: mail.message_id) + rescue StandardError => e + Gitlab::ErrorTracking.log_exception(e) + end + if service_desk_setting&.issue_template_missing? - create_template_not_found_note(@issue) + create_template_not_found_note + end + end + + def issue_from_reply_to + strong_memoize(:issue_from_reply_to) do + next unless mail.in_reply_to + + Issue::Email.find_by_email_message_id(mail.in_reply_to)&.issue end end + def reply_email? + issue_from_reply_to.present? + end + + def create_note_from_reply_email + @issue = issue_from_reply_to + + create_note(message_including_reply) + end + def send_thank_you_email Notify.service_desk_thank_you_email(@issue.id).deliver_later Gitlab::Metrics::BackgroundTransaction.current&.add_event(:service_desk_thank_you_email) end def message_including_template - description = process_message(trim_reply: false, allow_only_quotes: true) + description = message_including_reply_or_only_quotes template_content = service_desk_setting&.issue_template_content if template_content.present? @@ -124,7 +156,7 @@ module Gitlab end end - def create_template_not_found_note(issue) + def create_template_not_found_note issue_template_key = service_desk_setting&.issue_template_key warning_note = <<-MD.strip_heredoc @@ -132,15 +164,15 @@ module Gitlab Please check service desk settings and update the file to be used. MD - note_params = { - noteable: issue, - note: warning_note - } + create_note(warning_note) + end + def create_note(note) ::Notes::CreateService.new( project, User.support_bot, - note_params + noteable: @issue, + note: note ).execute end @@ -157,6 +189,8 @@ module Gitlab end def add_email_participant + return if reply_email? && !Feature.enabled?(:issue_email_participants, @issue.project) + @issue.issue_email_participants.create(email: from_address) end end diff --git a/lib/gitlab/email/hook/smime_signature_interceptor.rb b/lib/gitlab/email/hook/smime_signature_interceptor.rb index 0b092b3e41e..ed1bf3e12bf 100644 --- a/lib/gitlab/email/hook/smime_signature_interceptor.rb +++ b/lib/gitlab/email/hook/smime_signature_interceptor.rb @@ -45,7 +45,6 @@ module Gitlab end def overwrite_headers(message, signed_email) - message.content_disposition = signed_email.content_disposition message.content_transfer_encoding = signed_email.content_transfer_encoding message.content_type = signed_email.content_type end diff --git a/lib/gitlab/email/message/in_product_marketing/admin_verify.rb b/lib/gitlab/email/message/in_product_marketing/admin_verify.rb index 19d9cf99cdb..888f84cde23 100644 --- a/lib/gitlab/email/message/in_product_marketing/admin_verify.rb +++ b/lib/gitlab/email/message/in_product_marketing/admin_verify.rb @@ -38,7 +38,7 @@ module Gitlab end def invite_members? - invite_members_for_task_experiment_enabled? + user.can?(:admin_group_member, group) end end end diff --git a/lib/gitlab/email/message/in_product_marketing/base.rb b/lib/gitlab/email/message/in_product_marketing/base.rb index 7cd54390b9f..9b50d86de58 100644 --- a/lib/gitlab/email/message/in_product_marketing/base.rb +++ b/lib/gitlab/email/message/in_product_marketing/base.rb @@ -88,29 +88,6 @@ module Gitlab end end - def address - s_('InProductMarketing|%{strong_start}GitLab Inc.%{strong_end} 268 Bush Street, #350, San Francisco, CA 94104, USA').html_safe % strong_options - end - - def footer_links - links = [ - [s_('InProductMarketing|Blog'), 'https://about.gitlab.com/blog'], - [s_('InProductMarketing|Twitter'), 'https://twitter.com/gitlab'], - [s_('InProductMarketing|Facebook'), 'https://www.facebook.com/gitlab'], - [s_('InProductMarketing|YouTube'), 'https://www.youtube.com/channel/UCnMGQ8QHMAnVIsI3xJrihhg'] - ] - case format - when :html - links.map do |text, link| - ActionController::Base.helpers.link_to(text, link) - end - else - '| ' + links.map do |text, link| - [text, link].join(' ') - end.join("\n| ") - end - end - def logo_path ["mailers/in_product_marketing", "#{track}-#{series}.png"].join('/') end @@ -166,16 +143,6 @@ module Gitlab link(s_('InProductMarketing|update your preferences'), preference_link) end - def invite_members_for_task_experiment_enabled? - return unless user.can?(:admin_group_member, group) - - experiment(:invite_members_for_task, namespace: group) do |e| - e.candidate { true } - e.record! - e.run - end - end - def validate_series! raise ArgumentError, "Only #{total_series} series available for this track." unless @series.between?(0, total_series - 1) end diff --git a/lib/gitlab/email/message/in_product_marketing/create.rb b/lib/gitlab/email/message/in_product_marketing/create.rb index 2c396775374..6b01c83b8e7 100644 --- a/lib/gitlab/email/message/in_product_marketing/create.rb +++ b/lib/gitlab/email/message/in_product_marketing/create.rb @@ -62,7 +62,7 @@ module Gitlab end def invite_members? - invite_members_for_task_experiment_enabled? + user.can?(:admin_group_member, group) end private diff --git a/lib/gitlab/email/message/in_product_marketing/helper.rb b/lib/gitlab/email/message/in_product_marketing/helper.rb index bffa90ed4ec..329cace9e9d 100644 --- a/lib/gitlab/email/message/in_product_marketing/helper.rb +++ b/lib/gitlab/email/message/in_product_marketing/helper.rb @@ -8,6 +8,29 @@ module Gitlab include ActionView::Context include ActionView::Helpers::TagHelper + def footer_links + links = [ + [s_('InProductMarketing|Blog'), 'https://about.gitlab.com/blog'], + [s_('InProductMarketing|Twitter'), 'https://twitter.com/gitlab'], + [s_('InProductMarketing|Facebook'), 'https://www.facebook.com/gitlab'], + [s_('InProductMarketing|YouTube'), 'https://www.youtube.com/channel/UCnMGQ8QHMAnVIsI3xJrihhg'] + ] + case format + when :html + links.map do |text, link| + ActionController::Base.helpers.link_to(text, link) + end + else + '| ' + links.map do |text, link| + [text, link].join(' ') + end.join("\n| ") + end + end + + def address + s_('InProductMarketing|%{strong_start}GitLab Inc.%{strong_end} 268 Bush Street, #350, San Francisco, CA 94104, USA').html_safe % strong_options + end + private def list(array) diff --git a/lib/gitlab/email/message/in_product_marketing/verify.rb b/lib/gitlab/email/message/in_product_marketing/verify.rb index daf0c969f2b..d2a78b53e1f 100644 --- a/lib/gitlab/email/message/in_product_marketing/verify.rb +++ b/lib/gitlab/email/message/in_product_marketing/verify.rb @@ -66,7 +66,7 @@ module Gitlab end def invite_members? - invite_members_for_task_experiment_enabled? + user.can?(:admin_group_member, group) end private diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 526f1188065..5b2bbfbe66b 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -73,7 +73,7 @@ module Gitlab def key_from_to_header mail.to.find do |address| - key = Gitlab::IncomingEmail.key_from_address(address) + key = email_class.key_from_address(address) break key if key end end @@ -100,7 +100,7 @@ module Gitlab def find_key_from_references ensure_references_array(mail.references).find do |mail_id| - key = Gitlab::IncomingEmail.key_from_fallback_message_id(mail_id) + key = email_class.key_from_fallback_message_id(mail_id) break key if key end end @@ -119,21 +119,21 @@ module Gitlab def find_key_from_delivered_to_header delivered_to.find do |header| - key = Gitlab::IncomingEmail.key_from_address(header.value) + key = email_class.key_from_address(header.value) break key if key end end def find_key_from_envelope_to_header envelope_to.find do |header| - key = Gitlab::IncomingEmail.key_from_address(header.value) + key = email_class.key_from_address(header.value) break key if key end end def find_key_from_x_envelope_to_header x_envelope_to.find do |header| - key = Gitlab::IncomingEmail.key_from_address(header.value) + key = email_class.key_from_address(header.value) break key if key end end @@ -158,6 +158,10 @@ module Gitlab autoreply && autoreply == 'yes' end + + def email_class + Gitlab::IncomingEmail + end end end end diff --git a/lib/gitlab/email/service_desk_receiver.rb b/lib/gitlab/email/service_desk_receiver.rb index 133c4ee4b45..6c6eb3b0a65 100644 --- a/lib/gitlab/email/service_desk_receiver.rb +++ b/lib/gitlab/email/service_desk_receiver.rb @@ -6,22 +6,13 @@ module Gitlab private def find_handler - return unless service_desk_key + return unless mail_key - Gitlab::Email::Handler::ServiceDeskHandler.new(mail, nil, service_desk_key: service_desk_key) + Gitlab::Email::Handler::ServiceDeskHandler.new(mail, nil, service_desk_key: mail_key) end - def service_desk_key - strong_memoize(:service_desk_key) do - find_service_desk_key - end - end - - def find_service_desk_key - mail.to.find do |address| - key = ::Gitlab::ServiceDeskEmail.key_from_address(address) - break key if key - end + def email_class + ::Gitlab::ServiceDeskEmail end end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index 519b1d94bf5..3c5d223b106 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -6,7 +6,7 @@ module Gitlab # When updating emoji assets increase the version below # and update the version number in `app/assets/javascripts/emoji/index.js` - EMOJI_VERSION = 1 + EMOJI_VERSION = 2 # Return a Pathname to emoji's current versioned folder # diff --git a/lib/gitlab/empty_search_results.rb b/lib/gitlab/empty_search_results.rb new file mode 100644 index 00000000000..71a78cb297b --- /dev/null +++ b/lib/gitlab/empty_search_results.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + # This class has the same interface as SearchResults except + # it is empty and does not do any work. + # + # We use this when responding to abusive search requests. + class EmptySearchResults + def initialize(*) + end + + def objects(*) + Kaminari.paginate_array([]) + end + + def formatted_count(*) + '0' + end + + def highlight_map(*) + {} + end + + def aggregations(*) + [] + end + end +end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 38ac5d9af74..6a637306225 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -140,12 +140,9 @@ module Gitlab end def inject_context_for_exception(event, ex) - case ex - when ActiveRecord::StatementInvalid - event.extra[:sql] = PgQuery.normalize(ex.sql.to_s) - else - inject_context_for_exception(event, ex.cause) if ex.cause.present? - end + sql = Gitlab::ExceptionLogFormatter.find_sql(ex) + + event.extra[:sql] = sql if sql end end end diff --git a/lib/gitlab/etag_caching/router/graphql.rb b/lib/gitlab/etag_caching/router/graphql.rb index 2b8639b9411..1f56670ee7f 100644 --- a/lib/gitlab/etag_caching/router/graphql.rb +++ b/lib/gitlab/etag_caching/router/graphql.rb @@ -17,6 +17,11 @@ module Gitlab %r(\Apipelines/sha/\w{7,40}\z), 'ci_editor', 'pipeline_authoring' + ], + [ + %r(\Aon_demand_scan/counts/), + 'on_demand_scans', + 'dynamic_application_security_testing' ] ].map(&method(:build_route)).freeze diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index d0d790a7c72..44c6984c09b 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -12,14 +12,18 @@ module Gitlab Gitlab::Redis::SharedState.with { |redis| redis.get(redis_shared_state_key(key)) } end - def touch(key, only_if_missing: false) - etag = generate_etag + def touch(*keys, only_if_missing: false) + etags = keys.map { generate_etag } Gitlab::Redis::SharedState.with do |redis| - redis.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) + redis.pipelined do + keys.each_with_index do |key, i| + redis.set(redis_shared_state_key(key), etags[i], ex: EXPIRY_TIME, nx: only_if_missing) + end + end end - etag + keys.size > 1 ? etags : etags.first end private diff --git a/lib/gitlab/exception_log_formatter.rb b/lib/gitlab/exception_log_formatter.rb index 9898651c9e3..315574fed31 100644 --- a/lib/gitlab/exception_log_formatter.rb +++ b/lib/gitlab/exception_log_formatter.rb @@ -2,18 +2,41 @@ module Gitlab module ExceptionLogFormatter - def self.format!(exception, payload) - return unless exception + class << self + def format!(exception, payload) + return unless exception - # Elasticsearch/Fluentd don't handle nested structures well. - # Use periods to flatten the fields. - payload.merge!( - 'exception.class' => exception.class.name, - 'exception.message' => exception.message - ) + # Elasticsearch/Fluentd don't handle nested structures well. + # Use periods to flatten the fields. + payload.merge!( + 'exception.class' => exception.class.name, + 'exception.message' => exception.message + ) - if exception.backtrace - payload['exception.backtrace'] = Rails.backtrace_cleaner.clean(exception.backtrace) + if exception.backtrace + payload['exception.backtrace'] = Rails.backtrace_cleaner.clean(exception.backtrace) + end + + if sql = find_sql(exception) + payload['exception.sql'] = sql + end + end + + def find_sql(exception) + if exception.is_a?(ActiveRecord::StatementInvalid) + # StatementInvalid may be caused by a statement timeout or a bad query + normalize_query(exception.sql.to_s) + elsif exception.cause.present? + find_sql(exception.cause) + end + end + + private + + def normalize_query(sql) + PgQuery.normalize(sql) + rescue PgQuery::ParseError + sql end end end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index c2009628c56..4cc653bec43 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -37,9 +37,6 @@ module Gitlab remove_known_trial_form_fields_noneditable: { tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFieldsNoneditable', rollout_strategy: :user - }, - invite_members_new_dropdown: { - tracking_category: 'Growth::Expansion::Experiment::InviteMembersNewDropdown' } }.freeze diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index 7cc29cde45c..303d952381f 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -16,7 +16,7 @@ module Gitlab included do before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled? - helper_method :experiment_enabled?, :experiment_tracking_category_and_group, :record_experiment_group, :tracking_label + helper_method :experiment_enabled?, :experiment_tracking_category_and_group, :record_experiment_group end def set_experimentation_subject_id_cookie diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index 4d82acd9d87..5d0a638f97a 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -57,7 +57,7 @@ module Gitlab def unfold_reference(reference, match, target_parent) before = @text[0...match.begin(0)] - after = @text[match.end(0)..-1] + after = @text[match.end(0)..] referable = find_referable(reference) return reference unless referable diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index b0d194f309a..f72217dedde 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -24,7 +24,7 @@ module Gitlab LFS_POINTER_MIN_SIZE = 120.bytes LFS_POINTER_MAX_SIZE = 200.bytes - attr_accessor :size, :mode, :id, :commit_id, :loaded_size, :binary, :transformed_for_diff + attr_accessor :size, :mode, :id, :commit_id, :loaded_size, :binary attr_writer :name, :path, :data def self.gitlab_blob_truncated_true @@ -127,7 +127,6 @@ module Gitlab # Retain the actual size before it is encoded @loaded_size = @data.bytesize if @data @loaded_all_data = @loaded_size == size - @transformed_for_diff = false record_metric_blob_size record_metric_truncated(truncated?) diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 631624c068c..24b67424f28 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -11,13 +11,13 @@ module Gitlab delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits - def self.default_limits(project: nil) - { max_files: ::Commit.diff_safe_max_files(project: project), max_lines: ::Commit.diff_safe_max_lines(project: project) } + def self.default_limits + { max_files: ::Commit.diff_safe_max_files, max_lines: ::Commit.diff_safe_max_lines } end def self.limits(options = {}) limits = {} - defaults = default_limits(project: options[:project]) + defaults = default_limits limits[:max_files] = options.fetch(:max_files, defaults[:max_files]) limits[:max_lines] = options.fetch(:max_lines, defaults[:max_lines]) limits[:max_bytes] = limits[:max_files] * 5.kilobytes # Average 5 KB per file diff --git a/lib/gitlab/git/diff_stats_collection.rb b/lib/gitlab/git/diff_stats_collection.rb index e30ec836a49..6a689f55478 100644 --- a/lib/gitlab/git/diff_stats_collection.rb +++ b/lib/gitlab/git/diff_stats_collection.rb @@ -22,8 +22,8 @@ module Gitlab @collection.map(&:path) end - def real_size(project: nil) - max_files = ::Commit.max_diff_options(project: project)[:max_files] + def real_size + max_files = ::Commit.diff_max_files if paths.size > max_files "#{max_files}+" else diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 5afdcc0bd4c..240a701aba9 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -160,6 +160,8 @@ module Gitlab wrapped_gitaly_errors do gitaly_repository_client.remove end + rescue NoRepository + nil end def replicate(source_repository) diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 7c688044e9c..c2b4182f609 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -204,19 +204,6 @@ module Gitlab Gitlab::Git::Commit.new(@repository, gitaly_commit) end - def between(from, to) - return list_commits(["^" + from, to], reverse: true) if Feature.enabled?(:between_commits_via_list_commits) - - request = Gitaly::CommitsBetweenRequest.new( - repository: @gitaly_repo, - from: from, - to: to - ) - - response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request, timeout: GitalyClient.medium_timeout) - consume_commits_response(response) - end - def diff_stats(left_commit_sha, right_commit_sha) request = Gitaly::DiffStatsRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index efa816c5eb0..d2495b32800 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -21,6 +21,7 @@ module Gitlab SEARCH_MAX_REQUESTS_PER_MINUTE = 30 DEFAULT_PER_PAGE = 100 LOWER_PER_PAGE = 50 + CLIENT_CONNECTION_ERROR = ::Faraday::ConnectionFailed # used/set in sawyer agent which octokit uses # A single page of data and the corresponding page number. Page = Struct.new(:objects, :number) @@ -148,14 +149,14 @@ module Gitlab # whether we are running in parallel mode or not. For more information see # `#rate_or_wait_for_rate_limit`. def with_rate_limit - return yield unless rate_limiting_enabled? + return with_retry { yield } unless rate_limiting_enabled? request_count_counter.increment raise_or_wait_for_rate_limit unless requests_remaining? begin - yield + with_retry { yield } rescue ::Octokit::TooManyRequests raise_or_wait_for_rate_limit @@ -166,7 +167,7 @@ module Gitlab end def search_repos_by_name(name, options = {}) - octokit.search_repositories(search_query(str: name, type: :name), options) + with_retry { octokit.search_repositories(search_query(str: name, type: :name), options) } end def search_query(str:, type:, include_collaborations: true, include_orgs: true) @@ -270,6 +271,25 @@ module Gitlab .map { |org| "org:#{org.login}" } .join(' ') end + + def with_retry + Retriable.retriable(on: CLIENT_CONNECTION_ERROR, on_retry: on_retry) do + yield + end + end + + def on_retry + proc do |exception, try, elapsed_time, next_interval| + Gitlab::Import::Logger.info( + message: "GitHub connection retry triggered", + 'error.class': exception.class, + 'error.message': exception.message, + try_count: try, + elapsed_time_s: elapsed_time, + wait_to_retry_s: next_interval + ) + end + end end end end diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index 0aa0896aa57..8a8d23401c1 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -31,6 +31,10 @@ module Gitlab else import_with_legacy_diff_note end + rescue ::DiffNote::NoteDiffFileCreationError => e + Logger.warn(message: e.message, 'error.class': e.class.name) + + import_with_legacy_diff_note rescue ActiveRecord::InvalidForeignKey => e # It's possible the project and the issue have been deleted since # scheduling this job. In this case we'll just skip creating the note diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 2cc3a82dd9b..673f56b5753 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -29,6 +29,7 @@ module Gitlab project_id: project.id, author_id: author_id, note: note_body, + discussion_id: note.discussion_id, system: false, created_at: note.created_at, updated_at: note.updated_at diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb index f72e595e8e9..a71590c02f8 100644 --- a/lib/gitlab/github_import/parallel_importer.rb +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -35,7 +35,9 @@ module Gitlab def execute Gitlab::Import::SetAsyncJid.set_jid(project.import_state) + # We need to track this job's status for use by Gitlab::GithubImport::RefreshImportJidWorker. Stage::ImportRepositoryWorker + .with_status .perform_async(project.id) true diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index fecff0644c2..04f53accfeb 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -4,6 +4,7 @@ module Gitlab module GithubImport module Representation class DiffNote + include Gitlab::Utils::StrongMemoize include ToHash include ExposeAttribute @@ -127,15 +128,17 @@ module Gitlab end def discussion_id - if in_reply_to_id.present? - current_discussion_id - else - Discussion.discussion_id( - Struct - .new(:noteable_id, :noteable_type) - .new(merge_request.id, NOTEABLE_TYPE) - ).tap do |discussion_id| - cache_discussion_id(discussion_id) + strong_memoize(:discussion_id) do + if in_reply_to_id.present? + current_discussion_id + else + Discussion.discussion_id( + Struct + .new(:noteable_id, :noteable_type) + .new(merge_request.id, NOTEABLE_TYPE) + ).tap do |discussion_id| + cache_discussion_id(discussion_id) + end end end end diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index bcdb1a5459b..bbf20b7e9e6 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -63,6 +63,14 @@ module Gitlab @attributes = attributes end + def discussion_id + Discussion.discussion_id( + Struct + .new(:noteable_id, :noteable_type) + .new(noteable_id, noteable_type) + ) + end + alias_method :issuable_type, :noteable_type def github_identifiers diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 9ad902efb3a..bb3ba1129fc 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -56,7 +56,6 @@ module Gitlab push_frontend_feature_flag(:security_auto_fix, default_enabled: false) push_frontend_feature_flag(:improved_emoji_picker, default_enabled: :yaml) push_frontend_feature_flag(:new_header_search, default_enabled: :yaml) - push_frontend_feature_flag(:configure_iac_scanning_via_mr, current_user, default_enabled: :yaml) push_frontend_feature_flag(:bootstrap_confirmation_modals, default_enabled: :yaml) end diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index 3d9b06855ff..f55afd90ac0 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -48,7 +48,7 @@ module Gitlab raw_keys.each_with_object({}) do |raw_key, grouped_subkeys| primary_subkey_id = raw_key.primary_subkey.keyid - grouped_subkeys[primary_subkey_id] = raw_key.subkeys[1..-1].map do |s| + grouped_subkeys[primary_subkey_id] = raw_key.subkeys[1..].map do |s| { keyid: s.keyid, fingerprint: s.fingerprint } end end diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 9a6317e2b76..59882e8d4f8 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -25,7 +25,7 @@ module Gitlab def lazy_signature BatchLoader.for(@commit.sha).batch do |shas, loader| - GpgSignature.by_commit_sha(shas).each do |signature| + CommitSignatures::GpgSignature.by_commit_sha(shas).each do |signature| loader.call(signature.commit_sha, signature) end end @@ -62,9 +62,9 @@ module Gitlab def create_cached_signature! using_keychain do |gpg_key| attributes = attributes(gpg_key) - break GpgSignature.new(attributes) if Gitlab::Database.read_only? + break CommitSignatures::GpgSignature.new(attributes) if Gitlab::Database.read_only? - GpgSignature.safe_create!(attributes) + CommitSignatures::GpgSignature.safe_create!(attributes) end end diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb index d892d27a917..0eb105143ea 100644 --- a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -9,9 +9,9 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def run - GpgSignature + CommitSignatures::GpgSignature .select(:id, :commit_sha, :project_id) - .where('gpg_key_id IS NULL OR verification_status <> ?', GpgSignature.verification_statuses[:verified]) + .where('gpg_key_id IS NULL OR verification_status <> ?', CommitSignatures::GpgSignature.verification_statuses[:verified]) .where(gpg_key_primary_keyid: @gpg_key.keyids) .find_each { |sig| sig.gpg_commit&.update_signature!(sig) } end diff --git a/lib/gitlab/graphql/tracers/logger_tracer.rb b/lib/gitlab/graphql/tracers/logger_tracer.rb index c7ba56824db..3302b2bae3f 100644 --- a/lib/gitlab/graphql/tracers/logger_tracer.rb +++ b/lib/gitlab/graphql/tracers/logger_tracer.rb @@ -11,19 +11,20 @@ module Gitlab end def trace(key, data) - result = yield - + yield + rescue StandardError => e + data[:exception] = e + raise e + ensure case key when "execute_query" log_execute_query(**data) end - - result end private - def log_execute_query(query: nil, duration_s: 0) + def log_execute_query(query: nil, duration_s: 0, exception: nil) # execute_query should always have :query, but we're just being defensive return unless query @@ -39,6 +40,8 @@ module Gitlab query_string: query.query_string } + Gitlab::ExceptionLogFormatter.format!(exception, info) + info.merge!(::Gitlab::ApplicationContext.current) info.merge!(analysis_info) if analysis_info diff --git a/lib/gitlab/graphql/tracers/timer_tracer.rb b/lib/gitlab/graphql/tracers/timer_tracer.rb index 326620a22bc..8e058621110 100644 --- a/lib/gitlab/graphql/tracers/timer_tracer.rb +++ b/lib/gitlab/graphql/tracers/timer_tracer.rb @@ -17,13 +17,9 @@ module Gitlab def trace(key, data) start_time = Gitlab::Metrics::System.monotonic_time - result = yield - - duration_s = Gitlab::Metrics::System.monotonic_time - start_time - - data[:duration_s] = duration_s - - result + yield + ensure + data[:duration_s] = Gitlab::Metrics::System.monotonic_time - start_time end end end diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index db807a3c557..0e787a77a25 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -7,6 +7,7 @@ module Gitlab %i[ assignee_id author_id + blocking_discussions_resolved created_at description head_pipeline_id @@ -57,7 +58,8 @@ module Gitlab human_time_estimate: merge_request.human_time_estimate, assignee_ids: merge_request.assignee_ids, assignee_id: merge_request.assignee_ids.first, # This key is deprecated - state: merge_request.state # This key is deprecated + state: merge_request.state, # This key is deprecated + blocking_discussions_resolved: merge_request.mergeable_discussions_state? } merge_request.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 251bc34d462..12203cab8c8 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -43,27 +43,27 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 52, + 'da_DK' => 51, 'de' => 15, 'en' => 100, 'eo' => 0, - 'es' => 40, + 'es' => 39, 'fil_PH' => 0, - 'fr' => 11, + 'fr' => 12, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 2, - 'ja' => 36, + 'ja' => 35, 'ko' => 11, - 'nb_NO' => 34, + 'nb_NO' => 33, 'nl_NL' => 0, 'pl_PL' => 5, 'pt_BR' => 49, - 'ro_RO' => 24, - 'ru' => 26, + 'ro_RO' => 23, + 'ru' => 25, 'tr_TR' => 15, - 'uk' => 39, - 'zh_CN' => 97, + 'uk' => 45, + 'zh_CN' => 95, 'zh_HK' => 2, 'zh_TW' => 3 }.freeze diff --git a/lib/gitlab/import/import_failure_service.rb b/lib/gitlab/import/import_failure_service.rb index 142c00f7a6b..bebd64b29a9 100644 --- a/lib/gitlab/import/import_failure_service.rb +++ b/lib/gitlab/import/import_failure_service.rb @@ -15,11 +15,21 @@ module Gitlab exception: exception, import_state: import_state, project_id: project_id, - error_source: error_source - ).execute(fail_import: fail_import, metrics: metrics) + error_source: error_source, + fail_import: fail_import, + metrics: metrics + ).execute end - def initialize(exception:, import_state: nil, project_id: nil, error_source: nil) + def initialize( + exception:, + import_state: nil, + project_id: nil, + error_source: nil, + fail_import: false, + metrics: false + ) + if import_state.blank? && project_id.blank? raise ArgumentError, 'import_state OR project_id must be provided' end @@ -34,9 +44,11 @@ module Gitlab @exception = exception @error_source = error_source + @fail_import = fail_import + @metrics = metrics end - def execute(fail_import:, metrics:) + def execute track_exception persist_failure @@ -46,7 +58,7 @@ module Gitlab private - attr_reader :exception, :import_state, :project, :error_source + attr_reader :exception, :import_state, :project, :error_source, :fail_import, :metrics def track_exception attributes = { @@ -65,12 +77,15 @@ module Gitlab Gitlab::ErrorTracking.track_exception(exception, attributes) end + # Failures with `retry_count: 0` are considered "hard_failures" and those + # are exposed on the REST API projects/:id/import def persist_failure project.import_failures.create( source: error_source, exception_class: exception.class.to_s, exception_message: exception.message.truncate(255), - correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id + correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id, + retry_count: fail_import ? 0 : nil ) end diff --git a/lib/gitlab/import/set_async_jid.rb b/lib/gitlab/import/set_async_jid.rb index 527d84477fe..054fcdb433f 100644 --- a/lib/gitlab/import/set_async_jid.rb +++ b/lib/gitlab/import/set_async_jid.rb @@ -13,7 +13,7 @@ module Gitlab def self.set_jid(import_state) jid = generate_jid(import_state) - Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION) + Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION, value: 2) import_state.update_column(:jid, jid) end diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb index f3c392b8c20..cbc8ee9e18b 100644 --- a/lib/gitlab/import_export/group/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb @@ -106,12 +106,7 @@ module Gitlab def update_params! params = @importable_attributes.except(*relations.keys.map(&:to_s)) params = params.merge(present_override_params) - - # Cleaning all imported and overridden params - params = Gitlab::ImportExport::AttributeCleaner.clean( - relation_hash: params, - relation_class: importable_class, - excluded_keys: excluded_keys_for_relation(importable_class_sym)) + params = filter_attributes(params) @importable.assign_attributes(params) @@ -122,6 +117,25 @@ module Gitlab end end + def filter_attributes(params) + if use_attributes_permitter? && attributes_permitter.permitted_attributes_defined?(importable_class_sym) + attributes_permitter.permit(importable_class_sym, params) + else + Gitlab::ImportExport::AttributeCleaner.clean( + relation_hash: params, + relation_class: importable_class, + excluded_keys: excluded_keys_for_relation(importable_class_sym)) + end + end + + def attributes_permitter + @attributes_permitter ||= Gitlab::ImportExport::AttributesPermitter.new + end + + def use_attributes_permitter? + Feature.enabled?(:permitted_attributes_for_import_export, default_enabled: :yaml) + end + def present_override_params # we filter out the empty strings from the overrides # keeping the default values configured diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 9ab8fa68d0e..fb8d6e7d89b 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -40,6 +40,13 @@ module Gitlab end end + def serialize_root(exportable_path = @exportable_path) + attributes = exportable.as_json( + relations_schema.merge(include: nil, preloads: nil)) + + json_writer.write_attributes(exportable_path, attributes) + end + def serialize_relation(definition) raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash) raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one? @@ -60,12 +67,6 @@ module Gitlab attr_reader :json_writer, :relations_schema, :exportable - def serialize_root - attributes = exportable.as_json( - relations_schema.merge(include: nil, preloads: nil)) - json_writer.write_attributes(@exportable_path, attributes) - end - def serialize_many_relations(key, records, options) enumerator = Enumerator.new do |items| key_preloads = preloads&.dig(key) diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index d815dd284ba..ef146359da9 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -126,7 +126,6 @@ included_attributes: - :project_id project_badges: - :created_at - - :group_id - :image_url - :link_url - :name @@ -414,7 +413,6 @@ included_attributes: - :b_mode - :too_large - :binary - - :diff metrics: - :created_at - :updated_at @@ -572,7 +570,6 @@ included_attributes: - :updated_at actions: - :event - - :image_v432x230 design: &design_definition - :iid - :project_id @@ -632,6 +629,74 @@ included_attributes: - :expires_at - :ldap - :override + project: + - :approvals_before_merge + - :archived + - :auto_cancel_pending_pipelines + - :autoclose_referenced_issues + - :build_allow_git_fetch + - :build_coverage_regex + - :build_timeout + - :ci_config_path + - :delete_error + - :description + - :disable_overriding_approvers_per_merge_request + - :external_authorization_classification_label + - :has_external_issue_tracker + - :has_external_wiki + - :issues_template + - :jobs_cache_index + - :last_repository_check_failed + - :merge_requests_author_approval + - :merge_requests_disable_committers_approval + - :merge_requests_ff_only_enabled + - :merge_requests_rebase_enabled + - :merge_requests_template + - :only_allow_merge_if_all_discussions_are_resolved + - :only_allow_merge_if_pipeline_succeeds + - :pages_https_only + - :pending_delete + - :printing_merge_request_link_enabled + - :public_builds + - :remove_source_branch_after_merge + - :request_access_enabled + - :require_password_to_approve + - :reset_approvals_on_push + - :resolve_outdated_diff_discussions + - :service_desk_enabled + - :shared_runners_enabled + - :suggestion_commit_message + - :visibility_level + - :hooks + - :issues_access_level + - :forking_access_level + - :merge_requests_access_level + - :wiki_access_level + - :snippets_access_level + - :builds_access_level + - :repository_access_level + - :pages_access_level + - :metrics_dashboard_access_level + - :analytics_access_level + - :operations_access_level + - :security_and_compliance_access_level + - :container_registry_access_level + - :allow_merge_on_skipped_pipeline + - :auto_devops_deploy_strategy + - :auto_devops_enabled + - :container_registry_enabled + - :issues_enabled + - :jobs_enabled + - :merge_method + - :merge_requests_enabled + - :snippets_enabled + - :squash_option + - :topics + - :visibility + - :wiki_enabled + - :build_git_strategy + - :build_enabled + - :security_and_compliance_enabled # Do not include the following attributes for the models specified. excluded_attributes: @@ -747,6 +812,7 @@ excluded_attributes: - :service_desk_reply_to - :upvotes_count - :work_item_type_id + - :email_message_id merge_request: &merge_request_excluded_definition - :milestone_id - :sprint_id @@ -889,8 +955,6 @@ excluded_attributes: system_note_metadata: - :description_version_id - :note_id - pipeline_schedules: - - :active methods: notes: - :type @@ -1021,3 +1085,6 @@ ee: - :auto_fix_dast - :auto_fix_dependency_scanning - :auto_fix_sast + project: + - :requirements_enabled + - :requirements_access_level diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index d84db92fe69..c391f86b47b 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -131,7 +131,9 @@ module Gitlab end def setup_diff - @relation_hash['diff'] = @relation_hash.delete('utf8_diff') + diff = @relation_hash.delete('utf8_diff') + + parsed_relation_hash['diff'] = diff end def setup_pipeline diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb index e6c9ba0773c..84ead5119d5 100644 --- a/lib/gitlab/lograge/custom_options.rb +++ b/lib/gitlab/lograge/custom_options.rb @@ -32,6 +32,10 @@ module Gitlab ::Gitlab::ExceptionLogFormatter.format!(exception, payload) + if Feature.enabled?(:feature_flag_state_logs, type: :ops) + payload[:feature_flag_states] = Feature.logged_states.map { |key, state| "#{key}:#{state ? 1 : 0}" } + end + payload end end diff --git a/lib/gitlab/merge_requests/merge_commit_message.rb b/lib/gitlab/merge_requests/commit_message_generator.rb index 2a6a7859b33..0e9ec6f5cb3 100644 --- a/lib/gitlab/merge_requests/merge_commit_message.rb +++ b/lib/gitlab/merge_requests/commit_message_generator.rb @@ -1,31 +1,21 @@ # frozen_string_literal: true module Gitlab module MergeRequests - class MergeCommitMessage + class CommitMessageGenerator def initialize(merge_request:) @merge_request = merge_request end - def message + def merge_message return unless @merge_request.target_project.merge_commit_template.present? - message = @merge_request.target_project.merge_commit_template - message = message.delete("\r") + replace_placeholders(@merge_request.target_project.merge_commit_template) + end - # Remove placeholders that correspond to empty values and are the last word in the line - # along with all whitespace characters preceding them. - # This allows us to recreate previous default merge commit message behaviour - we skipped new line character - # before empty description and before closed issues when none were present. - PLACEHOLDERS.each do |key, value| - unless value.call(merge_request).present? - message = message.gsub(BLANK_PLACEHOLDERS_REGEXES[key], '') - end - end + def squash_message + return unless @merge_request.target_project.squash_commit_template.present? - Gitlab::StringPlaceholderReplacer - .replace_string_placeholders(message, PLACEHOLDERS_REGEX) do |key| - PLACEHOLDERS[key].call(merge_request) - end + replace_placeholders(@merge_request.target_project.squash_commit_template) end private @@ -45,7 +35,9 @@ module Gitlab "Closes #{closes_issues_references.to_sentence}" end, 'description' => ->(merge_request) { merge_request.description.presence || '' }, - 'reference' => ->(merge_request) { merge_request.to_reference(full: true) } + 'reference' => ->(merge_request) { merge_request.to_reference(full: true) }, + 'first_commit' => -> (merge_request) { merge_request.first_commit&.safe_message&.strip.presence || '' }, + 'first_multiline_commit' => -> (merge_request) { merge_request.first_multiline_commit&.safe_message&.strip.presence || merge_request.title } }.freeze PLACEHOLDERS_REGEX = Regexp.union(PLACEHOLDERS.keys.map do |key| @@ -55,6 +47,26 @@ module Gitlab BLANK_PLACEHOLDERS_REGEXES = (PLACEHOLDERS.map do |key, value| [key, Regexp.new("[\n\r]+%{#{Regexp.escape(key)}}$")] end).to_h.freeze + + def replace_placeholders(message) + # convert CRLF to LF + message = message.delete("\r") + + # Remove placeholders that correspond to empty values and are the last word in the line + # along with all whitespace characters preceding them. + # This allows us to recreate previous default merge commit message behaviour - we skipped new line character + # before empty description and before closed issues when none were present. + PLACEHOLDERS.each do |key, value| + unless value.call(merge_request).present? + message = message.gsub(BLANK_PLACEHOLDERS_REGEXES[key], '') + end + end + + Gitlab::StringPlaceholderReplacer + .replace_string_placeholders(message, PLACEHOLDERS_REGEX) do |key| + PLACEHOLDERS[key].call(merge_request) + end + end end end end diff --git a/lib/gitlab/merge_requests/mergeability/redis_interface.rb b/lib/gitlab/merge_requests/mergeability/redis_interface.rb index 081ccfca360..b0e739f91ff 100644 --- a/lib/gitlab/merge_requests/mergeability/redis_interface.rb +++ b/lib/gitlab/merge_requests/mergeability/redis_interface.rb @@ -7,13 +7,13 @@ module Gitlab VERSION = 1 def save_check(merge_check:, result_hash:) - Gitlab::Redis::SharedState.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.set(merge_check.cache_key + ":#{VERSION}", result_hash.to_json, ex: EXPIRATION) end end def retrieve_check(merge_check:) - Gitlab::Redis::SharedState.with do |redis| + Gitlab::Redis::Cache.with do |redis| Gitlab::Json.parse(redis.get(merge_check.cache_key + ":#{VERSION}")) end end diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb index ff8b8bf2237..47c862c0232 100644 --- a/lib/gitlab/metrics/exporter/base_exporter.rb +++ b/lib/gitlab/metrics/exporter/base_exporter.rb @@ -11,12 +11,14 @@ module Gitlab attr_accessor :readiness_checks - def enabled? - settings.enabled + def initialize(settings, **options) + super(**options) + + @settings = settings end - def settings - raise NotImplementedError + def enabled? + settings.enabled end def log_filename @@ -25,6 +27,8 @@ module Gitlab private + attr_reader :settings + def start_working logger = WEBrick::Log.new(log_filename) logger.time_format = "[%Y-%m-%dT%H:%M:%S.%L%z]" diff --git a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb index 4d38d9e67bf..eea71fda6a0 100644 --- a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb +++ b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb @@ -4,10 +4,6 @@ module Gitlab module Metrics module Exporter class SidekiqExporter < BaseExporter - def settings - Settings.monitoring.sidekiq_exporter - end - def log_filename if settings['log_enabled'] File.join(Rails.root, 'log', 'sidekiq_exporter.log') @@ -15,29 +11,6 @@ module Gitlab File::NULL end end - - private - - # Sidekiq Exporter does not work properly in sidekiq-cluster - # mode. It tries to start the service on the same port for - # each of the cluster workers, this results in failure - # due to duplicate binding. - # - # For now we ignore this error, as metrics are still "kind of" - # valid as they are rendered from shared directory. - # - # Issue: https://gitlab.com/gitlab-org/gitlab/issues/5714 - def start_working - super - rescue Errno::EADDRINUSE => e - Sidekiq.logger.error( - class: self.class.to_s, - message: 'Cannot start sidekiq_exporter', - 'exception.message' => e.message - ) - - false - end end end end diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb index c5fa1e545d7..d41484aaaa7 100644 --- a/lib/gitlab/metrics/exporter/web_exporter.rb +++ b/lib/gitlab/metrics/exporter/web_exporter.rb @@ -27,7 +27,7 @@ module Gitlab # This exporter is always run on master process def initialize - super + super(Settings.monitoring.web_exporter) # DEPRECATED: # these `readiness_checks` are deprecated @@ -39,10 +39,6 @@ module Gitlab ] end - def settings - Gitlab.config.monitoring.web_exporter - end - def log_filename File.join(Rails.root, 'log', 'web_exporter.log') end diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb index 5d7f434b660..965d85e20e5 100644 --- a/lib/gitlab/metrics/samplers/database_sampler.rb +++ b/lib/gitlab/metrics/samplers/database_sampler.rb @@ -38,16 +38,42 @@ module Gitlab end def host_stats - return [] unless ActiveRecord::Base.connected? + connection_class_stats + replica_host_stats + end + + def connection_class_stats + Gitlab::Database.database_base_models.each_value.with_object([]) do |base_model, stats| + next unless base_model.connected? + + stats << { labels: labels_for_class(base_model), stats: base_model.connection_pool.stat } + end + end + + def replica_host_stats + Gitlab::Database::LoadBalancing.each_load_balancer.with_object([]) do |load_balancer, stats| + next if load_balancer.primary_only? - [{ labels: labels_for_class(ActiveRecord::Base), stats: ActiveRecord::Base.connection_pool.stat }] + load_balancer.host_list.hosts.each do |host| + stats << { labels: labels_for_replica_host(load_balancer, host), stats: host.connection.pool.stat } + end + end end def labels_for_class(klass) { host: klass.connection_db_config.host, port: klass.connection_db_config.configuration_hash[:port], - class: klass.to_s + class: klass.to_s, + db_config_name: klass.connection_db_config.name + } + end + + def labels_for_replica_host(load_balancer, host) + { + host: host.host, + port: host.port, + class: load_balancer.configuration.primary_connection_specification_name, + db_config_name: Gitlab::Database.db_config_name(host.connection) } end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index df0582149a9..715dd86d93c 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -5,6 +5,8 @@ module Gitlab module Subscribers # Class for tracking the total query duration of a transaction. class ActiveRecord < ActiveSupport::Subscriber + extend Gitlab::Utils::StrongMemoize + attach_to :active_record IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze @@ -107,7 +109,7 @@ module Gitlab # Per database metrics db_config_name = db_config_name(event.payload) - duration_key = compose_metric_key(:duration_s, db_role, db_config_name) + duration_key = compose_metric_key(:duration_s, nil, db_config_name) ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration end @@ -144,7 +146,7 @@ module Gitlab # when we are also logging the db_role. Otherwise it will be hard to # tell if the log key is referring to a db_role OR a db_config_name. if db_role.present? && db_config_name.present? - log_key = compose_metric_key(counter, db_role, db_config_name) + log_key = compose_metric_key(counter, nil, db_config_name) Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1 end end @@ -172,26 +174,34 @@ module Gitlab end def self.load_balancing_metric_counter_keys - load_balancing_metric_keys(DB_LOAD_BALANCING_COUNTERS) + strong_memoize(:load_balancing_metric_counter_keys) do + load_balancing_metric_keys(DB_LOAD_BALANCING_COUNTERS) + end end def self.load_balancing_metric_duration_keys - load_balancing_metric_keys(DB_LOAD_BALANCING_DURATIONS) + strong_memoize(:load_balancing_metric_duration_keys) do + load_balancing_metric_keys(DB_LOAD_BALANCING_DURATIONS) + end end def self.load_balancing_metric_keys(metrics) - [].tap do |counters| + counters = [] + + metrics.each do |metric| DB_LOAD_BALANCING_ROLES.each do |role| - metrics.each do |metric| - counters << compose_metric_key(metric, role) - next unless ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] + counters << compose_metric_key(metric, role) + end - ::Gitlab::Database.db_config_names.each do |config_name| - counters << compose_metric_key(metric, role, config_name) - end + if ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] + ::Gitlab::Database.db_config_names.each do |config_name| + counters << compose_metric_key(metric, nil, config_name) # main + counters << compose_metric_key(metric, nil, config_name + ::Gitlab::Database::LoadBalancing::LoadBalancer::REPLICA_SUFFIX) # main_replica end end end + + counters end def compose_metric_key(metric, db_role = nil, db_config_name = nil) diff --git a/lib/gitlab/pagination/keyset/iterator.rb b/lib/gitlab/pagination/keyset/iterator.rb index bcd17fd0d34..33587f9d6f4 100644 --- a/lib/gitlab/pagination/keyset/iterator.rb +++ b/lib/gitlab/pagination/keyset/iterator.rb @@ -9,7 +9,7 @@ module Gitlab raise(UnsupportedScopeOrder) unless success @cursor = cursor - @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) + @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(@scope) @use_union_optimization = in_operator_optimization_options ? false : use_union_optimization @in_operator_optimization_options = in_operator_optimization_options end diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb index 80726fc8efd..1a00692bdbe 100644 --- a/lib/gitlab/pagination/keyset/order.rb +++ b/lib/gitlab/pagination/keyset/order.rb @@ -155,7 +155,7 @@ module Gitlab def apply_cursor_conditions(scope, values = {}, options = { use_union_optimization: false, in_operator_optimization_options: nil }) values ||= {} transformed_values = values.with_indifferent_access - scope = apply_custom_projections(scope.dup) + scope = apply_custom_projections(scope) where_values = build_where_values(transformed_values) diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb index 7b5013f137b..4f8a6ffb2cc 100644 --- a/lib/gitlab/pagination/offset_pagination.rb +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -29,7 +29,7 @@ module Gitlab return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit, type: :ops) limited_total_count = pagination_data.total_count_with_limit - if limited_total_count > max_limit + if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT # The call to `total_count_with_limit` memoizes `@arel` because of a call to `references_eager_loaded_tables?` # We need to call `reset` because `without_count` relies on `@arel` being unmemoized pagination_data.reset.without_count @@ -38,14 +38,6 @@ module Gitlab end end - def max_limit - if Feature.enabled?(:lower_relation_max_count_limit, type: :ops) - Kaminari::ActiveRecordRelationMethods::MAX_COUNT_NEW_LOWER_LIMIT - else - Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT - end - end - def needs_pagination?(relation) return true unless relation.respond_to?(:current_page) return true if params[:page].present? && relation.current_page != params[:page].to_i diff --git a/lib/gitlab/patch/legacy_database_config.rb b/lib/gitlab/patch/legacy_database_config.rb index a7d4fdf7490..6040f737c75 100644 --- a/lib/gitlab/patch/legacy_database_config.rb +++ b/lib/gitlab/patch/legacy_database_config.rb @@ -35,6 +35,40 @@ module Gitlab attr_reader :uses_legacy_database_config end + def load_database_yaml + return super unless Gitlab.ee? + + super.deep_merge(load_geo_database_yaml) + end + + # This method is taken from Rails to load a database YAML file without + # evaluating ERB. This allows us to create the rake tasks for the Geo + # tracking database without filling in the configuration values or + # loading the environment. To be removed when we start configure Geo + # tracking database in database.yml instead of custom database_geo.yml + # + # https://github.com/rails/rails/blob/v6.1.4/railties/lib/rails/application/configuration.rb#L255 + def load_geo_database_yaml + path = Rails.root.join("config/database_geo.yml") + return {} unless File.exist?(path) + + require "rails/application/dummy_erb_compiler" + + yaml = DummyERB.new(Pathname.new(path).read).result + config = YAML.load(yaml) || {} # rubocop:disable Security/YAMLLoad + + config.to_h do |env, configs| + # This check is taken from Rails where the transformation + # of a flat database.yml is done into `primary:` + # https://github.com/rails/rails/blob/v6.1.4/activerecord/lib/active_record/database_configurations.rb#L169 + if configs.is_a?(Hash) && !configs.all? { |_, v| v.is_a?(Hash) } + configs = { "geo" => configs } + end + + [env, configs] + end + end + def database_configuration @uses_legacy_database_config = false # rubocop:disable Gitlab/ModuleWithInstanceVariables @@ -48,6 +82,16 @@ module Gitlab @uses_legacy_database_config = true # rubocop:disable Gitlab/ModuleWithInstanceVariables end + if Gitlab.ee? && File.exist?(Rails.root.join("config/database_geo.yml")) + migrations_paths = ["ee/db/geo/migrate"] + migrations_paths << "ee/db/geo/post_migrate" unless ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] + + configs["geo"] = + Rails.application.config_for(:database_geo) + .merge(migrations_paths: migrations_paths, schema_migrations_path: "ee/db/geo/schema_migrations") + .stringify_keys + end + [env, configs] end end diff --git a/lib/gitlab/patch/sidekiq_client.rb b/lib/gitlab/patch/sidekiq_client.rb deleted file mode 100644 index 2de13560cce..00000000000 --- a/lib/gitlab/patch/sidekiq_client.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Patch - module SidekiqClient - private - - # This is a copy of https://github.com/mperham/sidekiq/blob/v6.2.2/lib/sidekiq/client.rb#L187-L194 - # but using `conn.pipelined` instead of `conn.multi`. The multi call isn't needed here because in - # the case of scheduled jobs, only one Redis call is made. For other jobs, we don't really need - # the commands to be atomic. - def raw_push(payloads) - @redis_pool.with do |conn| # rubocop:disable Gitlab/ModuleWithInstanceVariables - conn.pipelined do - atomic_push(conn, payloads) - end - end - true - end - end - end -end diff --git a/lib/gitlab/patch/sidekiq_cron_poller.rb b/lib/gitlab/patch/sidekiq_poller.rb index 56ca24c68f5..d4264cec1ab 100644 --- a/lib/gitlab/patch/sidekiq_cron_poller.rb +++ b/lib/gitlab/patch/sidekiq_poller.rb @@ -2,7 +2,7 @@ module Gitlab module Patch - module SidekiqCronPoller + module SidekiqPoller def enqueue Rails.application.reloader.wrap do ::Gitlab::WithRequestStore.with_request_store do diff --git a/lib/gitlab/process_management.rb b/lib/gitlab/process_management.rb new file mode 100644 index 00000000000..25a198e4a6a --- /dev/null +++ b/lib/gitlab/process_management.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module Gitlab + module ProcessManagement + # Traps the given signals and yields the block whenever these signals are + # received. + # + # The block is passed the name of the signal. + # + # Example: + # + # trap_signals(%i(HUP TERM)) do |signal| + # ... + # end + def self.trap_signals(signals) + signals.each do |signal| + trap(signal) do + yield signal + end + end + end + + # Traps the given signals with the given command. + # + # Example: + # + # modify_signals(%i(HUP TERM), 'DEFAULT') + def self.modify_signals(signals, command) + signals.each { |signal| trap(signal, command) } + end + + def self.signal(pid, signal) + Process.kill(signal, pid) + true + rescue Errno::ESRCH + false + end + + def self.signal_processes(pids, signal) + pids.each { |pid| signal(pid, signal) } + end + + # Waits for the given process to complete using a separate thread. + def self.wait_async(pid) + Thread.new do + Process.wait(pid) rescue Errno::ECHILD + end + end + + # Returns true if all the processes are alive. + def self.all_alive?(pids) + pids.each do |pid| + return false unless process_alive?(pid) + end + + true + end + + def self.any_alive?(pids) + pids_alive(pids).any? + end + + def self.pids_alive(pids) + pids.select { |pid| process_alive?(pid) } + end + + def self.process_alive?(pid) + return false if pid.nil? + + # Signal 0 tests whether the process exists and we have access to send signals + # but is otherwise a noop (doesn't actually send a signal to the process) + signal(pid, 0) + end + + def self.process_died?(pid) + !process_alive?(pid) + end + + def self.write_pid(path) + File.open(path, 'w') do |handle| + handle.write(Process.pid.to_s) + end + end + end +end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index a55ead519e2..5fd422db722 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -206,7 +206,7 @@ module Gitlab end desc _('Add Zoom meeting') - explanation _('Adds a Zoom meeting') + explanation _('Adds a Zoom meeting.') params '<Zoom URL>' types Issue condition do @@ -223,7 +223,7 @@ module Gitlab end desc _('Remove Zoom meeting') - explanation _('Remove Zoom meeting') + explanation _('Remove Zoom meeting.') execution_message _('Zoom meeting removed') types Issue condition do @@ -236,7 +236,7 @@ module Gitlab end desc _('Add email participant(s)') - explanation _('Adds email participant(s)') + explanation _('Adds email participant(s).') params 'email1@example.com email2@example.com (up to 6 emails)' types Issue condition do @@ -285,6 +285,34 @@ module Gitlab end end + desc _('Add customer relation contacts') + explanation _('Add customer relation contact(s).') + params 'contact@example.com person@example.org' + types Issue + condition do + current_user.can?(:set_issue_crm_contacts, quick_action_target) + end + execution_message do + _('One or more contacts were successfully added.') + end + command :add_contacts do |contact_emails| + @updates[:add_contacts] = contact_emails.split(' ') + end + + desc _('Remove customer relation contacts') + explanation _('Remove customer relation contact(s).') + params 'contact@example.com person@example.org' + types Issue + condition do + current_user.can?(:set_issue_crm_contacts, quick_action_target) + end + execution_message do + _('One or more contacts were successfully removed.') + end + command :remove_contacts do |contact_emails| + @updates[:remove_contacts] = contact_emails.split(' ') + end + private def zoom_link_service diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb index dbc77c9f9d7..94ae29af3d0 100644 --- a/lib/gitlab/rack_attack/request.rb +++ b/lib/gitlab/rack_attack/request.rb @@ -139,14 +139,12 @@ module Gitlab def throttle_unauthenticated_files_api? files_api_path? && - Feature.enabled?(:files_api_throttling, default_enabled: :yaml) && Gitlab::Throttle.settings.throttle_unauthenticated_files_api_enabled && unauthenticated? end def throttle_authenticated_files_api? files_api_path? && - Feature.enabled?(:files_api_throttling, default_enabled: :yaml) && Gitlab::Throttle.settings.throttle_authenticated_files_api_enabled end diff --git a/lib/gitlab/rate_limit_helpers.rb b/lib/gitlab/rate_limit_helpers.rb deleted file mode 100644 index 653410a40a5..00000000000 --- a/lib/gitlab/rate_limit_helpers.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module RateLimitHelpers - ARCHIVE_RATE_LIMIT_REACHED_MESSAGE = 'This archive has been requested too many times. Try again later.' - ARCHIVE_RATE_ANONYMOUS_THRESHOLD = 100 # Allow 100 requests/min for anonymous users - ARCHIVE_RATE_THROTTLE_KEY = :project_repositories_archive - - def archive_rate_limit_reached?(user, project) - return false unless Feature.enabled?(:archive_rate_limit) - - key = ARCHIVE_RATE_THROTTLE_KEY - - if rate_limiter.throttled?(key, scope: [project, user], threshold: archive_rate_threshold_by_user(user)) - rate_limiter.log_request(request, "#{key}_request_limit".to_sym, user) - - return true - end - - false - end - - def archive_rate_threshold_by_user(user) - if user - nil # Use the defaults - else - ARCHIVE_RATE_ANONYMOUS_THRESHOLD - end - end - - def rate_limiter - ::Gitlab::ApplicationRateLimiter - end - end -end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index f930a0040bc..500b62bf0e8 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -21,6 +21,8 @@ module Gitlab FAILED_TO_READ_ERROR_MESSAGE = 'Failed to read from the redis primary_store.' FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis primary_store.' + SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i(info).freeze + READ_COMMANDS = %i( get mget @@ -39,41 +41,42 @@ module Gitlab flushdb ).freeze - def initialize(primary_store, secondary_store, instance_name = nil) + def initialize(primary_store, secondary_store, instance_name) @primary_store = primary_store @secondary_store = secondary_store @instance_name = instance_name validate_stores! end - + # rubocop:disable GitlabSecurity/PublicSend READ_COMMANDS.each do |name| define_method(name) do |*args, &block| - if multi_store_enabled? + if use_primary_and_secondary_stores? read_command(name, *args, &block) else - secondary_store.send(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + default_store.send(name, *args, &block) end end end WRITE_COMMANDS.each do |name| define_method(name) do |*args, &block| - if multi_store_enabled? + if use_primary_and_secondary_stores? write_command(name, *args, &block) else - secondary_store.send(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + default_store.send(name, *args, &block) end end end def method_missing(...) - return @instance.send(...) if @instance # rubocop:disable GitlabSecurity/PublicSend + return @instance.send(...) if @instance log_method_missing(...) - secondary_store.send(...) # rubocop:disable GitlabSecurity/PublicSend + default_store.send(...) end + # rubocop:enable GitlabSecurity/PublicSend def respond_to_missing?(command_name, include_private = false) true @@ -83,23 +86,37 @@ module Gitlab # https://github.com/redis-store/redis-rack/blob/a833086ba494083b6a384a1a4e58b36573a9165d/lib/redis/rack/connection.rb#L15 # Done similarly in https://github.com/lsegal/yard/blob/main/lib/yard/templates/template.rb#L122 def is_a?(klass) - return true if klass == secondary_store.class + return true if klass == default_store.class super(klass) end alias_method :kind_of?, :is_a? def to_s - if multi_store_enabled? - primary_store.to_s - else - secondary_store.to_s - end + use_primary_and_secondary_stores? ? primary_store.to_s : default_store.to_s + end + + def use_primary_and_secondary_stores? + feature_flags_available? && + Feature.enabled?("use_primary_and_secondary_stores_for_#{instance_name.underscore}", default_enabled: :yaml) && + !same_redis_store? + end + + def use_primary_store_as_default? + feature_flags_available? && + Feature.enabled?("use_primary_store_as_default_for_#{instance_name.underscore}", default_enabled: :yaml) && + !same_redis_store? end private + def default_store + use_primary_store_as_default? ? primary_store : secondary_store + end + def log_method_missing(command_name, *_args) + return if SKIP_LOG_METHOD_MISSING_FOR_COMMANDS.include?(command_name) + log_error(MethodMissingError.new, command_name) increment_method_missing_count(command_name) end @@ -155,10 +172,6 @@ module Gitlab send_command(secondary_store, command_name, *args, &block) end - def multi_store_enabled? - Feature.enabled?(:use_multi_store, default_enabled: :yaml) && !same_redis_store? - end - def same_redis_store? strong_memoize(:same_redis_store) do # <Redis client v4.4.0 for redis:///path_to/redis/redis.socket/5>" @@ -194,12 +207,13 @@ module Gitlab def increment_method_missing_count(command_name) @method_missing_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_method_missing_total, 'Client side Redis MultiStore method missing') - @method_missing_counter.increment(command: command_name, innamece_name: instance_name) + @method_missing_counter.increment(command: command_name, instance_name: instance_name) end def validate_stores! raise ArgumentError, 'primary_store is required' unless primary_store raise ArgumentError, 'secondary_store is required' unless secondary_store + raise ArgumentError, 'instance_name is required' unless instance_name raise ArgumentError, 'invalid primary_store' unless primary_store.is_a?(::Redis) raise ArgumentError, 'invalid secondary_store' unless secondary_store.is_a?(::Redis) end diff --git a/lib/gitlab/redis/sessions.rb b/lib/gitlab/redis/sessions.rb index 3bf1eb6211d..c547828d907 100644 --- a/lib/gitlab/redis/sessions.rb +++ b/lib/gitlab/redis/sessions.rb @@ -3,9 +3,45 @@ module Gitlab module Redis class Sessions < ::Gitlab::Redis::Wrapper - # The data we store on Sessions used to be stored on SharedState. - def self.config_fallback - SharedState + SESSION_NAMESPACE = 'session:gitlab' + USER_SESSIONS_NAMESPACE = 'session:user:gitlab' + USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab' + IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab2' + OTP_SESSIONS_NAMESPACE = 'session:otp' + + class << self + # The data we store on Sessions used to be stored on SharedState. + def config_fallback + SharedState + end + + private + + def redis + # Don't use multistore if redis.sessions configuration is not provided + return super if config_fallback? + + primary_store = ::Redis.new(params) + secondary_store = ::Redis.new(config_fallback.params) + + MultiStore.new(primary_store, secondary_store, store_name) + end + end + + def store(extras = {}) + # Don't use multistore if redis.sessions configuration is not provided + return super if self.class.config_fallback? + + primary_store = create_redis_store(redis_store_options, extras) + secondary_store = create_redis_store(self.class.config_fallback.params, extras) + + MultiStore.new(primary_store, secondary_store, self.class.store_name) + end + + private + + def create_redis_store(options, extras) + ::Redis::Store.new(options.merge(extras)) end end end diff --git a/lib/gitlab/redis/sessions_store_helper.rb b/lib/gitlab/redis/sessions_store_helper.rb new file mode 100644 index 00000000000..c80442847f1 --- /dev/null +++ b/lib/gitlab/redis/sessions_store_helper.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + module SessionsStoreHelper + extend ActiveSupport::Concern + + module StoreMethods + def redis_store_class + use_redis_session_store? ? Gitlab::Redis::Sessions : Gitlab::Redis::SharedState + end + + private + + def use_redis_session_store? + Gitlab::Utils.to_boolean(ENV['GITLAB_USE_REDIS_SESSIONS_STORE'], default: true) + end + end + + include StoreMethods + + included do + extend StoreMethods + end + end + end +end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 1250eabb041..fb3a143121b 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -3,10 +3,6 @@ module Gitlab module Redis class SharedState < ::Gitlab::Redis::Wrapper - SESSION_NAMESPACE = 'session:gitlab' - USER_SESSIONS_NAMESPACE = 'session:user:gitlab' - USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab' - IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab2' end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 985c8dc619c..75dbccb965d 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -28,7 +28,7 @@ module Gitlab end def pool - @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } + @pool ||= ConnectionPool.new(size: pool_size) { redis } end def pool_size @@ -67,6 +67,10 @@ module Gitlab File.expand_path('../../..', __dir__) end + def config_fallback? + config_file_name == config_fallback&.config_file_name + end + def config_file_name [ # Instance specific config sources: @@ -100,6 +104,12 @@ module Gitlab "::Gitlab::Instrumentation::Redis::#{store_name}".constantize end + + private + + def redis + ::Redis.new(params) + end end def initialize(rails_env = nil) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 904fc744c6b..8e139ae0709 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -16,8 +16,13 @@ module Gitlab @conan_revision_regex ||= %r{\A0\z}.freeze end + def conan_recipe_user_channel_regex + %r{\A(_|#{conan_name_regex})\z}.freeze + end + def conan_recipe_component_regex - @conan_recipe_component_regex ||= %r{\A[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{1,49}\z}.freeze + # https://docs.conan.io/en/latest/reference/conanfile/attributes.html#name + @conan_recipe_component_regex ||= %r{\A#{conan_name_regex}\z}.freeze end def composer_package_version_regex @@ -211,6 +216,12 @@ module Gitlab def generic_package_file_name_regex generic_package_name_regex end + + private + + def conan_name_regex + @conan_name_regex ||= %r{[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{1,49}}.freeze + end end extend self @@ -413,7 +424,11 @@ module Gitlab end def issue - @issue ||= /(?<issue>\d+\b)/ + @issue ||= /(?<issue>\d+)(?<format>\+)?(?=\W|\z)/ + end + + def merge_request + @merge_request ||= /(?<merge_request>\d+)(?<format>\+)?/ end def base64_regex @@ -430,3 +445,5 @@ module Gitlab end end end + +Gitlab::Regex.prepend_mod diff --git a/lib/gitlab/relative_positioning/item_context.rb b/lib/gitlab/relative_positioning/item_context.rb index 1e738aef9b0..98e52e8e767 100644 --- a/lib/gitlab/relative_positioning/item_context.rb +++ b/lib/gitlab/relative_positioning/item_context.rb @@ -66,19 +66,11 @@ module Gitlab end def lhs_neighbour - scoped_items - .where('relative_position < ?', relative_position) - .reorder(relative_position: :desc) - .first - .then { |x| neighbour(x) } + neighbour(object.next_object_by_relative_position(ignoring: ignoring, order: :desc)) end def rhs_neighbour - scoped_items - .where('relative_position > ?', relative_position) - .reorder(relative_position: :asc) - .first - .then { |x| neighbour(x) } + neighbour(object.next_object_by_relative_position(ignoring: ignoring, order: :asc)) end def neighbour(item) @@ -87,12 +79,6 @@ module Gitlab self.class.new(item, range, ignoring: ignoring) end - def scoped_items - r = model_class.relative_positioning_query_base(object) - r = object.exclude_self(r, excluded: ignoring) if ignoring.present? - r - end - def calculate_relative_position(calculation) # When calculating across projects, this is much more efficient than # MAX(relative_position) without the GROUP BY, due to index usage: @@ -186,6 +172,10 @@ module Gitlab Gap.new(gap.first, gap.second || default_end) end + def scoped_items + object.relative_positioning_scoped_items(ignoring: ignoring) + end + def relative_position object.relative_position end diff --git a/lib/gitlab/repository_archive_rate_limiter.rb b/lib/gitlab/repository_archive_rate_limiter.rb new file mode 100644 index 00000000000..31a3dc34bf6 --- /dev/null +++ b/lib/gitlab/repository_archive_rate_limiter.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module RepositoryArchiveRateLimiter + def check_archive_rate_limit!(current_user, project, &block) + return unless Feature.enabled?(:archive_rate_limit) + + threshold = current_user ? nil : 100 + + check_rate_limit!(:project_repositories_archive, scope: [project, current_user], threshold: threshold, &block) + end + end +end diff --git a/lib/gitlab/saas.rb b/lib/gitlab/saas.rb index 1e00bd4cbfc..577e33fd700 100644 --- a/lib/gitlab/saas.rb +++ b/lib/gitlab/saas.rb @@ -13,6 +13,10 @@ module Gitlab 'https://staging.gitlab.com' end + def self.canary_toggle_com_url + 'https://next.gitlab.com' + end + def self.subdomain_regex %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z}.freeze end diff --git a/lib/gitlab/search/abuse_detection.rb b/lib/gitlab/search/abuse_detection.rb new file mode 100644 index 00000000000..7b5377bce88 --- /dev/null +++ b/lib/gitlab/search/abuse_detection.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module Search + class AbuseDetection + include ActiveModel::Validations + include AbuseValidators + + ABUSIVE_TERM_SIZE = 100 + ALLOWED_CHARS_REGEX = %r{\A[[:alnum:]_\-\/\.!]+\z}.freeze + MINIMUM_SEARCH_CHARS = 2 + + ALLOWED_SCOPES = %w( + blobs + code + commits + epics + issues + merge_requests + milestones + notes + projects + snippet_titles + users + wiki_blobs + ).freeze + + READABLE_PARAMS = %i( + group_id + project_id + project_ref + query_string + repository_ref + scope + ).freeze + + STOP_WORDS = %w( + a an and are as at be but by for if in into is it no not of on or such that the their then there these they this to was will with + ).freeze + + validates :project_id, :group_id, + numericality: { only_integer: true, message: "abusive ID detected" }, allow_blank: true + + validates :scope, inclusion: { in: ALLOWED_SCOPES, message: 'abusive scope detected' }, allow_blank: true + + validates :repository_ref, :project_ref, + format: { with: ALLOWED_CHARS_REGEX, message: "abusive characters detected" }, allow_blank: true + + validates :query_string, + exclusion: { in: STOP_WORDS, message: 'stopword only abusive search detected' }, allow_blank: true + + validates :query_string, + length: { minimum: MINIMUM_SEARCH_CHARS, message: 'abusive tiny search detected' }, unless: :skip_tiny_search_validation?, allow_blank: true + + validates :query_string, + no_abusive_term_length: { maximum: ABUSIVE_TERM_SIZE, maximum_for_url: ABUSIVE_TERM_SIZE * 2 } + + validates :query_string, :repository_ref, :project_ref, no_abusive_coercion_from_string: true + + attr_reader(*READABLE_PARAMS) + + def initialize(params) + READABLE_PARAMS.each { |p| instance_variable_set("@#{p}", params[p]) } + end + + private + + def skip_tiny_search_validation? + wildcard_search? || stop_word_search? + end + + def wildcard_search? + query_string == '*' + end + + def stop_word_search? + STOP_WORDS.include? query_string + end + end + end +end diff --git a/lib/gitlab/search/abuse_validators/no_abusive_coercion_from_string_validator.rb b/lib/gitlab/search/abuse_validators/no_abusive_coercion_from_string_validator.rb new file mode 100644 index 00000000000..06464980afd --- /dev/null +++ b/lib/gitlab/search/abuse_validators/no_abusive_coercion_from_string_validator.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Search + module AbuseValidators + class NoAbusiveCoercionFromStringValidator < ActiveModel::EachValidator + def validate_each(instance, attribute, value) + if value.present? && !value.is_a?(String) + instance.errors.add attribute, "abusive coercion from string detected" + end + end + end + end + end +end diff --git a/lib/gitlab/search/abuse_validators/no_abusive_term_length_validator.rb b/lib/gitlab/search/abuse_validators/no_abusive_term_length_validator.rb new file mode 100644 index 00000000000..8a94520d8fd --- /dev/null +++ b/lib/gitlab/search/abuse_validators/no_abusive_term_length_validator.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Search + module AbuseValidators + class NoAbusiveTermLengthValidator < ActiveModel::EachValidator + def validate_each(instance, attribute, value) + return unless value.is_a?(String) + + if value.split.any? { |term| term_too_long?(term) } + instance.errors.add attribute, 'abusive term length detected' + end + end + + private + + def term_too_long?(term) + char_limit = url_detected?(term) ? maximum_for_url : maximum + term.length >= char_limit + end + + def url_detected?(uri_str) + URI::DEFAULT_PARSER.regexp[:ABS_URI].match? uri_str + end + + def maximum_for_url + options.fetch(:maximum_for_url, maximum) + end + + def maximum + options.fetch(:maximum) + end + end + end + end +end diff --git a/lib/gitlab/search/params.rb b/lib/gitlab/search/params.rb new file mode 100644 index 00000000000..e6a1305a82a --- /dev/null +++ b/lib/gitlab/search/params.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +module Gitlab + module Search + class Params + include ActiveModel::Validations + + SEARCH_CHAR_LIMIT = 4096 + SEARCH_TERM_LIMIT = 64 + + # Generic validation + validates :query_string, length: { maximum: SEARCH_CHAR_LIMIT } + validate :not_too_many_terms + + attr_reader :raw_params, :query_string, :abuse_detection + alias_method :search, :query_string + alias_method :term, :query_string + + def initialize(params, detect_abuse: true) + @raw_params = params.is_a?(Hash) ? params.with_indifferent_access : params.dup + @query_string = strip_surrounding_whitespace(@raw_params[:search] || @raw_params[:term]) + @detect_abuse = detect_abuse + @abuse_detection = AbuseDetection.new(self) if @detect_abuse + + validate + end + + def [](key) + if respond_to? key + # We have this logic here to support reading custom attributes + # like @query_string + # + # This takes precedence over values in @raw_params + public_send(key) # rubocop:disable GitlabSecurity/PublicSend + else + raw_params[key] + end + end + + def abusive? + detect_abuse? && abuse_detection.errors.any? + end + + def valid_query_length? + return true unless errors.has_key? :query_string + + errors[:query_string].none? { |msg| msg.include? SEARCH_CHAR_LIMIT.to_s } + end + + def valid_terms_count? + return true unless errors.has_key? :query_string + + errors[:query_string].none? { |msg| msg.include? SEARCH_TERM_LIMIT.to_s } + end + + def validate + if detect_abuse? + abuse_detection.validate + end + + super + end + + def valid? + if detect_abuse? + abuse_detection.valid? && super + else + super + end + end + + private + + def detect_abuse? + @detect_abuse + end + + def not_too_many_terms + if query_string.split.count { |word| word.length >= 3 } > SEARCH_TERM_LIMIT + errors.add :query_string, "has too many search terms (maximum is #{SEARCH_TERM_LIMIT})" + end + end + + def strip_surrounding_whitespace(obj) + obj.to_s.strip + end + end + end +end diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb index c0420126ada..97ee7c7817d 100644 --- a/lib/gitlab/search/query.rb +++ b/lib/gitlab/search/query.rb @@ -44,7 +44,7 @@ module Gitlab next unless match - input = match.split(':')[1..-1].join + input = match.split(':')[1..].join next if input.empty? filter[:negated] = match.start_with?("-") diff --git a/lib/gitlab/security/scan_configuration.rb b/lib/gitlab/security/scan_configuration.rb new file mode 100644 index 00000000000..eaccbb3be7e --- /dev/null +++ b/lib/gitlab/security/scan_configuration.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Security + class ScanConfiguration + include ::Gitlab::Utils::StrongMemoize + include Gitlab::Routing.url_helpers + + attr_reader :type + + def initialize(project:, type:, configured: false) + @project = project + @type = type + @configured = configured + end + + def available? + # SAST and Secret Detection are always available, but this isn't + # reflected by our license model yet. + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/333113 + %i[sast secret_detection].include?(type) + end + + def configured? + configured + end + + def configuration_path + configurable_scans[type] + end + + private + + attr_reader :project, :configured + + def configurable_scans + strong_memoize(:configurable_scans) do + { + sast: project_security_configuration_sast_path(project) + } + end + end + end + end +end + +Gitlab::Security::ScanConfiguration.prepend_mod_with('Gitlab::Security::ScanConfiguration') diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb index 52d88f074b7..aa25eb5a571 100644 --- a/lib/gitlab/sherlock/line_profiler.rb +++ b/lib/gitlab/sherlock/line_profiler.rb @@ -70,7 +70,7 @@ module Gitlab next if total_duration <= MINIMUM_DURATION - stats[1..-1].each_with_index do |data, index| + stats[1..].each_with_index do |data, index| next unless source_lines[index] duration = microsec_to_millisec(data[0]) diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index 07ddac209f8..3eef41a2ca2 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -8,6 +8,7 @@ module Gitlab EE_QUEUE_CONFIG_PATH = 'ee/app/workers/all_queues.yml' JH_QUEUE_CONFIG_PATH = 'jh/app/workers/all_queues.yml' SIDEKIQ_QUEUES_PATH = 'config/sidekiq_queues.yml' + JH_SIDEKIQ_QUEUES_PATH = 'jh/config/sidekiq_queues.yml' QUEUE_CONFIG_PATHS = [ FOSS_QUEUE_CONFIG_PATH, @@ -100,18 +101,24 @@ module Gitlab def queues_for_sidekiq_queues_yml namespaces_with_equal_weights = workers + .reject { |worker| worker.jh? } .group_by(&:queue_namespace) .map(&:last) .select { |workers| workers.map(&:get_weight).uniq.count == 1 } .map(&:first) namespaces = namespaces_with_equal_weights.map(&:queue_namespace).to_set - remaining_queues = workers.reject { |worker| namespaces.include?(worker.queue_namespace) } + remaining_queues = workers.reject { |worker| worker.jh? }.reject { |worker| namespaces.include?(worker.queue_namespace) } (namespaces_with_equal_weights.map(&:namespace_and_weight) + remaining_queues.map(&:queue_and_weight)).sort end + # Override in JH repo + def jh_queues_for_sidekiq_queues_yml + [] + end + # YAML.load_file is OK here as we control the file contents def sidekiq_queues_yml_outdated? config_queues = YAML.load_file(SIDEKIQ_QUEUES_PATH)[:queues] @@ -154,3 +161,5 @@ module Gitlab end end end + +Gitlab::SidekiqConfig.prepend_mod diff --git a/lib/gitlab/sidekiq_enq.rb b/lib/gitlab/sidekiq_enq.rb deleted file mode 100644 index de0c00fe561..00000000000 --- a/lib/gitlab/sidekiq_enq.rb +++ /dev/null @@ -1,111 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class SidekiqEnq - LUA_ZPOPBYSCORE = <<~EOS - local key, now = KEYS[1], ARGV[1] - local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1) - if jobs[1] then - redis.call("zrem", key, jobs[1]) - return jobs[1] - end - EOS - - LUA_ZPOPBYSCORE_SHA = Digest::SHA1.hexdigest(LUA_ZPOPBYSCORE) - - def enqueue_jobs(now = Time.now.to_f.to_s, sorted_sets = Sidekiq::Scheduled::SETS) - Rails.application.reloader.wrap do - ::Gitlab::WithRequestStore.with_request_store do - if Feature.enabled?(:atomic_sidekiq_scheduler, default_enabled: :yaml) - atomic_find_jobs_and_enqueue(now, sorted_sets) - else - find_jobs_and_enqueue(now, sorted_sets) - end - - ensure - ::Gitlab::Database::LoadBalancing.release_hosts - end - end - end - - private - - # This is a copy of https://github.com/mperham/sidekiq/blob/32c55e31659a1e6bd42f98334cca5eef2863de8d/lib/sidekiq/scheduled.rb#L11-L34 - # - # It effectively reverts - # https://github.com/mperham/sidekiq/commit/9b75467b33759888753191413eddbc15c37a219e - # because we observe that the extra ZREMs caused by this change can lead to high - # CPU usage on Redis at peak times: - # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1179 - # - def find_jobs_and_enqueue(now, sorted_sets) - # A job's "score" in Redis is the time at which it should be processed. - # Just check Redis for the set of jobs with a timestamp before now. - Sidekiq.redis do |conn| - sorted_sets.each do |sorted_set| - start_time = ::Gitlab::Metrics::System.monotonic_time - jobs = redundant_jobs = 0 - - Sidekiq.logger.info(message: 'Enqueuing scheduled jobs', status: 'start', sorted_set: sorted_set) - - # Get the next item in the queue if it's score (time to execute) is <= now. - # We need to go through the list one at a time to reduce the risk of something - # going wrong between the time jobs are popped from the scheduled queue and when - # they are pushed onto a work queue and losing the jobs. - while job = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 1]).first - # Pop item off the queue and add it to the work queue. If the job can't be popped from - # the queue, it's because another process already popped it so we can move on to the - # next one. - if conn.zrem(sorted_set, job) - jobs += 1 - Sidekiq::Client.push(Sidekiq.load_json(job)) - else - redundant_jobs += 1 - end - end - - end_time = ::Gitlab::Metrics::System.monotonic_time - Sidekiq.logger.info(message: 'Enqueuing scheduled jobs', - status: 'done', - sorted_set: sorted_set, - jobs_count: jobs, - redundant_jobs_count: redundant_jobs, - duration_s: end_time - start_time) - end - end - end - - def atomic_find_jobs_and_enqueue(now, sorted_sets) - Sidekiq.redis do |conn| - sorted_sets.each do |sorted_set| - start_time = ::Gitlab::Metrics::System.monotonic_time - jobs = 0 - - Sidekiq.logger.info(message: 'Atomically enqueuing scheduled jobs', status: 'start', sorted_set: sorted_set) - - while job = redis_eval_lua(conn, LUA_ZPOPBYSCORE, LUA_ZPOPBYSCORE_SHA, keys: [sorted_set], argv: [now]) - jobs += 1 - Sidekiq::Client.push(Sidekiq.load_json(job)) - end - - end_time = ::Gitlab::Metrics::System.monotonic_time - Sidekiq.logger.info(message: 'Atomically enqueuing scheduled jobs', - status: 'done', - sorted_set: sorted_set, - jobs_count: jobs, - duration_s: end_time - start_time) - end - end - end - - def redis_eval_lua(conn, script, sha, keys: nil, argv: nil) - conn.evalsha(sha, keys: keys, argv: argv) - rescue ::Redis::CommandError => e - if e.message.start_with?('NOSCRIPT') - conn.eval(script, keys: keys, argv: argv) - else - raise - end - end - end -end diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index fbf2718d718..120d18f63f2 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -29,13 +29,16 @@ module Gitlab # for most jobs. DEFAULT_EXPIRATION = 30.minutes.to_i + DEFAULT_VALUE = 1 + DEFAULT_VALUE_MESSAGE = 'Keys using the default value for SidekiqStatus detected' + # Starts tracking of the given job. # # jid - The Sidekiq job ID # expire - The expiration time of the Redis key. - def self.set(jid, expire = DEFAULT_EXPIRATION) + def self.set(jid, expire = DEFAULT_EXPIRATION, value: DEFAULT_VALUE) Sidekiq.redis do |redis| - redis.set(key_for(jid), 1, ex: expire) + redis.set(key_for(jid), value, ex: expire) end end @@ -88,13 +91,20 @@ module Gitlab # true = job is still running or enqueued # false = job completed def self.job_status(job_ids) + return [] if job_ids.empty? + keys = job_ids.map { |jid| key_for(jid) } + results = Sidekiq.redis { |redis| redis.mget(*keys) } - Sidekiq.redis do |redis| - redis.pipelined do - keys.each { |key| redis.exists(key) } - end + if Feature.enabled?(:log_implicit_sidekiq_status_calls, default_enabled: :yaml) + to_log = keys.zip(results).select do |_key, result| + result == DEFAULT_VALUE.to_s + end.map(&:first) + + Sidekiq.logger.info(message: DEFAULT_VALUE_MESSAGE, keys: to_log) if to_log.any? end + + results.map { |result| !result.nil? } end # Returns the JIDs that are completed diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb index bfd5038557d..cee7270f2fb 100644 --- a/lib/gitlab/sidekiq_status/client_middleware.rb +++ b/lib/gitlab/sidekiq_status/client_middleware.rb @@ -5,8 +5,9 @@ module Gitlab class ClientMiddleware def call(_, job, _, _) status_expiration = job['status_expiration'] || Gitlab::SidekiqStatus::DEFAULT_EXPIRATION + value = job['status_expiration'] ? 2 : Gitlab::SidekiqStatus::DEFAULT_VALUE - Gitlab::SidekiqStatus.set(job['jid'], status_expiration) + Gitlab::SidekiqStatus.set(job['jid'], status_expiration, value: value) yield end end diff --git a/lib/gitlab/spamcheck/client.rb b/lib/gitlab/spamcheck/client.rb index 925ca44dfc9..40b01552244 100644 --- a/lib/gitlab/spamcheck/client.rb +++ b/lib/gitlab/spamcheck/client.rb @@ -21,14 +21,16 @@ module Gitlab update: ::Spamcheck::Action::UPDATE }.freeze + URL_SCHEME_REGEX = %r{^grpc://|^tls://}.freeze + def initialize @endpoint_url = Gitlab::CurrentSettings.current_application_settings.spam_check_endpoint_url - # remove the `grpc://` as it's only useful to ensure we're expecting to - # connect with Spamcheck - @endpoint_url = @endpoint_url.gsub(%r(^grpc:\/\/), '') + @creds = client_creds(@endpoint_url) - @creds = stub_creds + # remove the `grpc://` or 'tls://' as it's only useful to ensure we're expecting to + # connect with Spamcheck + @endpoint_url = @endpoint_url.sub(URL_SCHEME_REGEX, '') end def issue_spam?(spam_issue:, user:, context: {}) @@ -96,11 +98,11 @@ module Gitlab nanos: ar_timestamp.to_time.nsec) end - def stub_creds - if Rails.env.development? || Rails.env.test? - :this_channel_is_insecure + def client_creds(url) + if URI(url).scheme == 'tls' + GRPC::Core::ChannelCredentials.new(::Gitlab::X509::Certificate.ca_certs_bundle) else - GRPC::Core::ChannelCredentials.new ::Gitlab::X509::Certificate.ca_certs_bundle + :this_channel_is_insecure end end diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb index 5ddc88edf50..292a9d07e6a 100644 --- a/lib/gitlab/string_range_marker.rb +++ b/lib/gitlab/string_range_marker.rb @@ -99,7 +99,7 @@ module Gitlab start = prev = positions[0] range = MarkerRange.new(start, prev, mode: mode) - positions[1..-1].each do |pos| + positions[1..].each do |pos| if pos == prev + 1 range = MarkerRange.new(start, pos, mode: mode) prev = pos diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb index 4f6d25097e2..d987247fdc4 100644 --- a/lib/gitlab/subscription_portal.rb +++ b/lib/gitlab/subscription_portal.rb @@ -18,6 +18,10 @@ module Gitlab "#{self.subscriptions_url}/payment_forms/cc_validation" end + def self.registration_validation_form_url + "#{self.subscriptions_url}/payment_forms/cc_registration_validation" + end + def self.subscriptions_comparison_url 'https://about.gitlab.com/pricing/gitlab-com/feature-comparison' end @@ -62,6 +66,10 @@ module Gitlab "#{self.subscriptions_url}/gitlab/namespaces/#{group_id}/renew" end + def self.edit_account_url + "#{self.subscriptions_url}/customers/edit" + end + def self.subscription_portal_admin_email ENV.fetch('SUBSCRIPTION_PORTAL_ADMIN_EMAIL', 'gl_com_api@gitlab.com') end @@ -69,9 +77,15 @@ module Gitlab def self.subscription_portal_admin_token ENV.fetch('SUBSCRIPTION_PORTAL_ADMIN_TOKEN', 'customer_admin_token') end + + def self.renewal_service_email + 'renewals-support@gitlab.com' + end end end Gitlab::SubscriptionPortal.prepend_mod Gitlab::SubscriptionPortal::SUBSCRIPTIONS_URL = Gitlab::SubscriptionPortal.subscriptions_url.freeze Gitlab::SubscriptionPortal::PAYMENT_FORM_URL = Gitlab::SubscriptionPortal.payment_form_url.freeze +Gitlab::SubscriptionPortal::RENEWAL_SERVICE_EMAIL = Gitlab::SubscriptionPortal.renewal_service_email.freeze +Gitlab::SubscriptionPortal::REGISTRATION_VALIDATION_FORM_URL = Gitlab::SubscriptionPortal.registration_validation_form_url.freeze diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 227962fc0f7..6a98fa12903 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -190,6 +190,9 @@ module Gitlab end def checkout_version(version, target_dir) + # Explicitly setting the git protocol version to v2 allows older Git binaries + # to do have a shallow clone obtain objects by object ID. + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} config protocol.version 2]) run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --quiet origin #{version}]) run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout -f --quiet FETCH_HEAD --]) end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index ec032cf2d3c..a58b4beb0df 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -2,11 +2,9 @@ module Gitlab module Tracking - SNOWPLOW_NAMESPACE = 'gl' - class << self def enabled? - snowplow_micro_enabled? || Gitlab::CurrentSettings.snowplow_enabled? + snowplow.enabled? end def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists @@ -25,6 +23,10 @@ module Gitlab snowplow.hostname end + def snowplow_micro_enabled? + Rails.env.development? && Gitlab::Utils.to_boolean(ENV['SNOWPLOW_MICRO_ENABLE']) + end + private def snowplow @@ -34,10 +36,6 @@ module Gitlab Gitlab::Tracking::Destinations::Snowplow.new end end - - def snowplow_micro_enabled? - Rails.env.development? && Gitlab::Utils.to_boolean(ENV['SNOWPLOW_MICRO_ENABLE']) - end end end end diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb index 5596e9acd30..ddcd4693738 100644 --- a/lib/gitlab/tracking/destinations/snowplow.rb +++ b/lib/gitlab/tracking/destinations/snowplow.rb @@ -8,6 +8,8 @@ module Gitlab class Snowplow < Base extend ::Gitlab::Utils::Override + SNOWPLOW_NAMESPACE = 'gl' + override :event def event(category, action, label: nil, property: nil, value: nil, context: nil) return unless enabled? @@ -19,7 +21,7 @@ module Gitlab def options(group) additional_features = Feature.enabled?(:additional_snowplow_tracking, group, type: :ops) { - namespace: Gitlab::Tracking::SNOWPLOW_NAMESPACE, + namespace: SNOWPLOW_NAMESPACE, hostname: hostname, cookie_domain: cookie_domain, app_id: app_id, @@ -28,16 +30,16 @@ module Gitlab }.transform_keys! { |key| key.to_s.camelize(:lower).to_sym } end + def enabled? + Gitlab::CurrentSettings.snowplow_enabled? + end + def hostname Gitlab::CurrentSettings.snowplow_collector_hostname end private - def enabled? - Gitlab::Tracking.enabled? - end - def app_id Gitlab::CurrentSettings.snowplow_app_id end @@ -54,7 +56,7 @@ module Gitlab @tracker ||= SnowplowTracker::Tracker.new( emitter, SnowplowTracker::Subject.new, - Gitlab::Tracking::SNOWPLOW_NAMESPACE, + SNOWPLOW_NAMESPACE, app_id ) end diff --git a/lib/gitlab/tracking/destinations/snowplow_micro.rb b/lib/gitlab/tracking/destinations/snowplow_micro.rb index b818d349a6d..3553efba1e1 100644 --- a/lib/gitlab/tracking/destinations/snowplow_micro.rb +++ b/lib/gitlab/tracking/destinations/snowplow_micro.rb @@ -15,7 +15,12 @@ module Gitlab protocol: uri.scheme, port: uri.port, force_secure_tracker: false - ) + ).transform_keys! { |key| key.to_s.camelize(:lower).to_sym } + end + + override :enabled? + def enabled? + true end override :hostname @@ -23,8 +28,6 @@ module Gitlab "#{uri.host}:#{uri.port}" end - private - def uri strong_memoize(:snowplow_uri) do uri = URI(ENV['SNOWPLOW_MICRO_URI'] || DEFAULT_URI) @@ -33,6 +36,8 @@ module Gitlab end end + private + override :cookie_domain def cookie_domain '.gitlab.com' diff --git a/lib/gitlab/usage/metrics/instrumentations/snowplow_configured_to_gitlab_collector_metric.rb b/lib/gitlab/usage/metrics/instrumentations/snowplow_configured_to_gitlab_collector_metric.rb new file mode 100644 index 00000000000..3a92525303b --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/snowplow_configured_to_gitlab_collector_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class SnowplowConfiguredToGitlabCollectorMetric < GenericMetric + GITLAB_SNOWPLOW_COLLECTOR_HOSTNAME = 'snowplow.trx.gitlab.net' + + def value + Gitlab::CurrentSettings.snowplow_collector_hostname == GITLAB_SNOWPLOW_COLLECTOR_HOSTNAME + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/snowplow_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/snowplow_enabled_metric.rb new file mode 100644 index 00000000000..5d504c70e73 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/snowplow_enabled_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class SnowplowEnabledMetric < GenericMetric + def value + Gitlab::CurrentSettings.snowplow_enabled? + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb index 6dcbe5f5fe5..d1a7bb65cc3 100644 --- a/lib/gitlab/usage/metrics/names_suggestions/generator.rb +++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb @@ -18,10 +18,6 @@ module Gitlab private - def instrumentation_metrics - ::Gitlab::UsageDataMetrics.suggested_names - end - def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) Gitlab::Usage::Metrics::NameSuggestion.for(:count, column: column, relation: relation) end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 20e526aeefa..917c273d3f6 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -10,6 +10,12 @@ # alt_usage_data { Gitlab::VERSION } # redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) # redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } + +# NOTE: +# Implementing metrics direct in `usage_data.rb` is deprecated, +# please add new instrumentation class and use add_metric method. +# For more information, see https://docs.gitlab.com/ee/development/service_ping/metrics_instrumentation.html + module Gitlab class UsageData DEPRECATED_VALUE = -1000 @@ -45,10 +51,7 @@ module Gitlab clear_memoized with_finished_at(:recording_ce_finished_at) do - usage_data = usage_data_metrics - usage_data = usage_data.with_indifferent_access.deep_merge(instrumentation_metrics.with_indifferent_access) if Feature.enabled?(:usage_data_instrumentation) - - usage_data + usage_data_metrics end end @@ -225,7 +228,9 @@ module Gitlab operating_system: alt_usage_data(fallback: nil) { operating_system }, gitaly_apdex: alt_usage_data { gitaly_apdex }, collected_data_categories: add_metric('CollectedDataCategoriesMetric', time_frame: 'none'), - service_ping_features_enabled: add_metric('ServicePingFeaturesMetric', time_frame: 'none') + service_ping_features_enabled: add_metric('ServicePingFeaturesMetric', time_frame: 'none'), + snowplow_enabled: add_metric('SnowplowEnabledMetric', time_frame: 'none'), + snowplow_configured_to_gitlab_collector: add_metric('SnowplowConfiguredToGitlabCollectorMetric', time_frame: 'none') } } end @@ -401,7 +406,8 @@ module Gitlab results[:projects_jira_cloud_active] = jira_integration_data_hash[:projects_jira_cloud_active] results - rescue ActiveRecord::StatementInvalid + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK } end # rubocop: enable CodeReuse/ActiveRecord @@ -663,8 +669,6 @@ module Gitlab end def redis_hll_counters - return {} unless Feature.enabled?(:redis_hll_tracking, type: :ops, default_enabled: :yaml) - { redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data } end @@ -726,7 +730,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord # rubocop: disable UsageData/LargeTable start = ::Event.where(time_period).select(:id).order(created_at: :asc).first&.id - finish = ::Event.where(time_period).select(:id).order(created_at: :asc).first&.id + finish = ::Event.where(time_period).select(:id).order(created_at: :desc).first&.id estimate_batch_distinct_count(::Event.where(time_period), :author_id, start: start, finish: finish) # rubocop: enable UsageData/LargeTable # rubocop: enable CodeReuse/ActiveRecord @@ -753,10 +757,6 @@ module Gitlab .deep_merge(aggregated_metrics_data) end - def instrumentation_metrics - Gitlab::UsageDataMetrics.uncached_data # rubocop:disable UsageData/LargeTable - end - def metric_time_period(time_period) time_period.present? ? '28d' : 'none' end diff --git a/lib/gitlab/usage_data_counters/known_events/analytics.yml b/lib/gitlab/usage_data_counters/known_events/analytics.yml index 261bdeb9bfa..5a1e7f03278 100644 --- a/lib/gitlab/usage_data_counters/known_events/analytics.yml +++ b/lib/gitlab/usage_data_counters/known_events/analytics.yml @@ -66,3 +66,15 @@ category: analytics redis_slot: analytics aggregation: weekly +- name: p_analytics_ci_cd_pipelines + category: analytics + redis_slot: analytics + aggregation: weekly +- name: p_analytics_ci_cd_deployment_frequency + category: analytics + redis_slot: analytics + aggregation: weekly +- name: p_analytics_ci_cd_lead_time + category: analytics + redis_slot: analytics + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index 40922433635..d90960b344c 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -119,6 +119,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_security_dast_api_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_security_container_scanning category: ci_templates redis_slot: ci_templates @@ -551,6 +555,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_implicit_security_dast_api_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_implicit_security_container_scanning category: ci_templates redis_slot: ci_templates diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index feebc7f395a..bb98a0b262a 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -355,6 +355,19 @@ category: testing aggregation: weekly feature_flag: users_expanding_widgets_usage_data +- name: users_expanding_testing_license_compliance_report + redis_slot: testing + category: testing + aggregation: weekly + feature_flag: users_expanding_widgets_usage_data +- name: users_visiting_testing_license_compliance_full_report + redis_slot: testing + category: testing + aggregation: weekly +- name: users_visiting_testing_manage_license_compliance + redis_slot: testing + category: testing + aggregation: weekly # Container Security - Network Policies - name: clusters_using_network_policies_ui redis_slot: network_policies diff --git a/lib/gitlab/usage_data_counters/known_events/importer_events.yml b/lib/gitlab/usage_data_counters/known_events/importer_events.yml index 79bbac229bc..c84d756a013 100644 --- a/lib/gitlab/usage_data_counters/known_events/importer_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/importer_events.yml @@ -4,14 +4,11 @@ category: importer redis_slot: import aggregation: weekly - feature_flag: track_importer_activity - name: github_import_project_success category: importer redis_slot: import aggregation: weekly - feature_flag: track_importer_activity - name: github_import_project_failure category: importer redis_slot: import aggregation: weekly - feature_flag: track_importer_activity diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index dff2c4f8d03..d831ac02dd1 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -279,3 +279,11 @@ category: quickactions redis_slot: quickactions aggregation: weekly +- name: i_quickactions_add_contacts + category: quickactions + redis_slot: quickactions + aggregation: weekly +- name: i_quickactions_remove_contacts + category: quickactions + redis_slot: quickactions + aggregation: weekly diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb index be5a571fb82..1661a1b6987 100644 --- a/lib/gitlab/usage_data_non_sql_metrics.rb +++ b/lib/gitlab/usage_data_non_sql_metrics.rb @@ -5,13 +5,6 @@ module Gitlab SQL_METRIC_DEFAULT = -3 class << self - def uncached_data - # instrumentation_metrics is already included with feature flag enabled - return super if Feature.enabled?(:usage_data_instrumentation) - - super.with_indifferent_access.deep_merge(instrumentation_metrics.with_indifferent_access) - end - def add_metric(metric, time_frame: 'none', options: {}) metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize @@ -50,12 +43,6 @@ module Gitlab projects_jira_cloud_active: 0 } end - - private - - def instrumentation_metrics - ::Gitlab::Usage::Metric.all.map(&:with_instrumentation).reduce({}, :deep_merge) - end end end end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index f543b29e43f..c6490ba7374 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -5,13 +5,6 @@ module Gitlab # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41091 class UsageDataQueries < UsageData class << self - def uncached_data - # instrumentation_metrics is already included with feature flag enabled - return super if Feature.enabled?(:usage_data_instrumentation) - - super.with_indifferent_access.deep_merge(instrumentation_metrics.with_indifferent_access) - end - def add_metric(metric, time_frame: 'none', options: {}) metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize @@ -71,12 +64,6 @@ module Gitlab def epics_deepest_relationship_level { epics_deepest_relationship_level: 0 } end - - private - - def instrumentation_metrics - ::Gitlab::Usage::Metric.all.map(&:with_instrumentation).reduce({}, :deep_merge) - end end end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index cb34ed69a9c..96cff024371 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -120,18 +120,14 @@ module Gitlab Random.rand(Float::MAX.to_i).to_s(36) end - # See: http://stackoverflow.com/questions/2108727/which-in-ruby-checking-if-program-exists-in-path-from-ruby - # Cross-platform way of finding an executable in the $PATH. + # Behaves like `which` on Linux machines: given PATH, try to resolve the given + # executable name to an absolute path, or return nil. # # which('ruby') #=> /usr/bin/ruby - def which(cmd, env = ENV) - exts = env['PATHEXT'] ? env['PATHEXT'].split(';') : [''] - - env['PATH'].split(File::PATH_SEPARATOR).each do |path| - exts.each do |ext| - exe = File.join(path, "#{cmd}#{ext}") - return exe if File.executable?(exe) && !File.directory?(exe) - end + def which(filename) + ENV['PATH']&.split(File::PATH_SEPARATOR)&.each do |path| + full_path = File.join(path, filename) + return full_path if File.executable?(full_path) end nil diff --git a/lib/gitlab/utils/nokogiri.rb b/lib/gitlab/utils/nokogiri.rb index 4b37bb7e5ea..5113553c584 100644 --- a/lib/gitlab/utils/nokogiri.rb +++ b/lib/gitlab/utils/nokogiri.rb @@ -16,7 +16,7 @@ module Gitlab # we remove the leading `//` and add `descendant-or-self::` # in order to ensure we're searching from this node and all # descendants. - xpath.map { |t| "descendant-or-self::#{t[2..-1]}" }.join('|') + xpath.map { |t| "descendant-or-self::#{t[2..]}" }.join('|') end end end diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb index 483bfe12c68..255fa0169bf 100644 --- a/lib/gitlab/utils/strong_memoize.rb +++ b/lib/gitlab/utils/strong_memoize.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require_dependency 'gitlab/utils' - module Gitlab module Utils module StrongMemoize diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 77f04929661..e347168f419 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -43,13 +43,8 @@ module Gitlab HISTOGRAM_FALLBACK = { '-1' => -1 }.freeze DISTRIBUTED_HLL_FALLBACK = -2 MAX_BUCKET_SIZE = 100 - INSTRUMENTATION_CLASS_FALLBACK = -100 def add_metric(metric, time_frame: 'none', options: {}) - # Results of this method should be overwritten by instrumentation class values - # -100 indicates the metric was not properly merged. - return INSTRUMENTATION_CLASS_FALLBACK if Feature.enabled?(:usage_data_instrumentation) - metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize metric_class.new(time_frame: time_frame, options: options).value @@ -61,7 +56,8 @@ module Gitlab else relation.count end - rescue ActiveRecord::StatementInvalid + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) FALLBACK end @@ -71,7 +67,8 @@ module Gitlab else relation.distinct_count_by(column) end - rescue ActiveRecord::StatementInvalid + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) FALLBACK end @@ -83,7 +80,8 @@ module Gitlab yield buckets if block_given? buckets.estimated_distinct_count - rescue ActiveRecord::StatementInvalid + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) FALLBACK # catch all rescue should be removed as a part of feature flag rollout issue # https://gitlab.com/gitlab-org/gitlab/-/issues/285485 @@ -94,7 +92,8 @@ module Gitlab def sum(relation, column, batch_size: nil, start: nil, finish: nil) Gitlab::Database::BatchCount.batch_sum(relation, column, batch_size: batch_size, start: start, finish: finish) - rescue ActiveRecord::StatementInvalid + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) FALLBACK end @@ -160,7 +159,8 @@ module Gitlab query: query.to_sql, message: e.message ) - + # Raises error for dev env + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) HISTOGRAM_FALLBACK end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/x509/commit.rb b/lib/gitlab/x509/commit.rb index 91951a3e505..c7f4b7cbdf5 100644 --- a/lib/gitlab/x509/commit.rb +++ b/lib/gitlab/x509/commit.rb @@ -25,7 +25,7 @@ module Gitlab def lazy_signature BatchLoader.for(@commit.sha).batch do |shas, loader| - X509CommitSignature.by_commit_sha(shas).each do |signature| + CommitSignatures::X509CommitSignature.by_commit_sha(shas).each do |signature| loader.call(signature.commit_sha, signature) end end @@ -49,9 +49,9 @@ module Gitlab def create_cached_signature! return if attributes.nil? - return X509CommitSignature.new(attributes) if Gitlab::Database.read_only? + return CommitSignatures::X509CommitSignature.new(attributes) if Gitlab::Database.read_only? - X509CommitSignature.safe_create!(attributes) + CommitSignatures::X509CommitSignature.safe_create!(attributes) end end end |