diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-05-19 15:44:42 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-05-19 15:44:42 +0000 |
| commit | 4555e1b21c365ed8303ffb7a3325d773c9b8bf31 (patch) | |
| tree | 5423a1c7516cffe36384133ade12572cf709398d /lib/gitlab | |
| parent | e570267f2f6b326480d284e0164a6464ba4081bc (diff) | |
| download | gitlab-ce-4555e1b21c365ed8303ffb7a3325d773c9b8bf31.tar.gz | |
Add latest changes from gitlab-org/gitlab@13-12-stable-eev13.12.0-rc42
Diffstat (limited to 'lib/gitlab')
507 files changed, 5906 insertions, 2367 deletions
diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 830980f0997..6afcd745d4e 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -159,4 +159,4 @@ module Gitlab end end -Gitlab::Access.prepend_if_ee('EE::Gitlab::Access') +Gitlab::Access.prepend_mod_with('Gitlab::Access') diff --git a/lib/gitlab/alert_management/payload.rb b/lib/gitlab/alert_management/payload.rb index a1063001330..1b67b91e839 100644 --- a/lib/gitlab/alert_management/payload.rb +++ b/lib/gitlab/alert_management/payload.rb @@ -49,4 +49,4 @@ module Gitlab end end -Gitlab::AlertManagement::Payload.prepend_if_ee('EE::Gitlab::AlertManagement::Payload') +Gitlab::AlertManagement::Payload.prepend_mod_with('Gitlab::AlertManagement::Payload') diff --git a/lib/gitlab/alert_management/payload/base.rb b/lib/gitlab/alert_management/payload/base.rb index 786c5bf675b..5e535ded439 100644 --- a/lib/gitlab/alert_management/payload/base.rb +++ b/lib/gitlab/alert_management/payload/base.rb @@ -130,7 +130,7 @@ module Gitlab strong_memoize(:environment) do next unless environment_name - EnvironmentsFinder + ::Environments::EnvironmentsFinder .new(project, nil, { name: environment_name }) .execute .first @@ -193,7 +193,7 @@ module Gitlab def parse_time(value) Time.parse(value).utc - rescue ArgumentError + rescue ArgumentError, TypeError end def parse_integer(value) diff --git a/lib/gitlab/alert_management/payload/generic.rb b/lib/gitlab/alert_management/payload/generic.rb index e2db9b62dd5..15238b5e50f 100644 --- a/lib/gitlab/alert_management/payload/generic.rb +++ b/lib/gitlab/alert_management/payload/generic.rb @@ -27,4 +27,4 @@ module Gitlab end end -Gitlab::AlertManagement::Payload::Generic.prepend_if_ee('EE::Gitlab::AlertManagement::Payload::Generic') +Gitlab::AlertManagement::Payload::Generic.prepend_mod_with('Gitlab::AlertManagement::Payload::Generic') diff --git a/lib/gitlab/analytics/cycle_analytics/average.rb b/lib/gitlab/analytics/cycle_analytics/average.rb index a449b71b165..7140d31d536 100644 --- a/lib/gitlab/analytics/cycle_analytics/average.rb +++ b/lib/gitlab/analytics/cycle_analytics/average.rb @@ -7,9 +7,10 @@ module Gitlab include Gitlab::Utils::StrongMemoize include StageQueryHelpers - def initialize(stage:, query:) + def initialize(stage:, query:, params: {}) @stage = stage @query = query + @params = params end def seconds @@ -22,7 +23,7 @@ module Gitlab private - attr_reader :stage + attr_reader :stage, :params # rubocop: disable CodeReuse/ActiveRecord def select_average diff --git a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb index 4dec71b35e8..c7987d63153 100644 --- a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb +++ b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb @@ -5,6 +5,7 @@ module Gitlab module CycleAnalytics class BaseQueryBuilder include Gitlab::CycleAnalytics::MetricsTables + include StageQueryHelpers delegate :subject_class, to: :stage @@ -13,17 +14,19 @@ module Gitlab Issue.to_s => IssuesFinder }.freeze + DEFAULT_END_EVENT_FILTER = :finished + def initialize(stage:, params: {}) @stage = stage @params = build_finder_params(params) + @params[:state] = :opened if in_progress? end # rubocop: disable CodeReuse/ActiveRecord def build query = finder.execute query = stage.start_event.apply_query_customization(query) - query = stage.end_event.apply_query_customization(query) - query.where(duration_condition) + apply_end_event_query_customization(query) end # rubocop: enable CodeReuse/ActiveRecord @@ -46,6 +49,7 @@ module Gitlab def build_finder_params(params) {}.tap do |finder_params| finder_params[:current_user] = params[:current_user] + finder_params[:end_event_filter] = params[:end_event_filter] || DEFAULT_END_EVENT_FILTER add_parent_model_params!(finder_params) add_time_range_params!(finder_params, params[:from], params[:to]) @@ -62,9 +66,20 @@ module Gitlab finder_params[:created_after] = from || 30.days.ago finder_params[:created_before] = to if to end + + # rubocop: disable CodeReuse/ActiveRecord + def apply_end_event_query_customization(query) + if in_progress? + stage.end_event.apply_negated_query_customization(query) + else + query = stage.end_event.apply_query_customization(query) + query.where(duration_condition) + end + end + # rubocop: enable CodeReuse/ActiveRecord end end end end -Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder') +Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder') diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb index 10a008a76d5..56179533ffb 100644 --- a/lib/gitlab/analytics/cycle_analytics/data_collector.rb +++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb @@ -12,6 +12,8 @@ module Gitlab class DataCollector include Gitlab::Utils::StrongMemoize + MAX_COUNT = 1001 + delegate :serialized_records, to: :records_fetcher def initialize(stage:, params: {}) @@ -27,13 +29,19 @@ module Gitlab def median strong_memoize(:median) do - Median.new(stage: stage, query: query) + Median.new(stage: stage, query: query, params: params) end end def average strong_memoize(:average) do - Average.new(stage: stage, query: query) + Average.new(stage: stage, query: query, params: params) + end + end + + def count + strong_memoize(:count) do + limit_count end end @@ -44,9 +52,16 @@ module Gitlab def query BaseQueryBuilder.new(stage: stage, params: params).build end + + # Limiting the maximum number of records so the COUNT(*) query stays efficient for large groups. + # COUNT = 1001, show 1000+ on the UI + # COUNT < 1001, show the actual number on the UI + def limit_count + query.limit(MAX_COUNT).count + end end end end end -Gitlab::Analytics::CycleAnalytics::DataCollector.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::DataCollector') +Gitlab::Analytics::CycleAnalytics::DataCollector.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::DataCollector') diff --git a/lib/gitlab/analytics/cycle_analytics/median.rb b/lib/gitlab/analytics/cycle_analytics/median.rb index 6c0450ac9e5..5775d0324c6 100644 --- a/lib/gitlab/analytics/cycle_analytics/median.rb +++ b/lib/gitlab/analytics/cycle_analytics/median.rb @@ -6,9 +6,10 @@ module Gitlab class Median include StageQueryHelpers - def initialize(stage:, query:) + def initialize(stage:, query:, params: {}) @stage = stage @query = query + @params = params end # rubocop: disable CodeReuse/ActiveRecord @@ -26,7 +27,7 @@ module Gitlab private - attr_reader :stage + attr_reader :stage, :params def percentile_cont percentile_cont_ordering = Arel::Nodes::UnaryOperation.new(Arel::Nodes::SqlLiteral.new('ORDER BY'), duration) diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb index b4752ed9e5b..9a37a41ff81 100644 --- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb @@ -124,7 +124,7 @@ module Gitlab def time_columns [ stage.start_event.timestamp_projection.as('start_event_timestamp'), - stage.end_event.timestamp_projection.as('end_event_timestamp'), + end_event_timestamp_projection.as('end_event_timestamp'), round_duration_to_seconds.as('total_time') ] end @@ -133,4 +133,4 @@ module Gitlab end end -Gitlab::Analytics::CycleAnalytics::RecordsFetcher.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::RecordsFetcher') +Gitlab::Analytics::CycleAnalytics::RecordsFetcher.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::RecordsFetcher') diff --git a/lib/gitlab/analytics/cycle_analytics/sorting.rb b/lib/gitlab/analytics/cycle_analytics/sorting.rb index 828879d466d..c399bac423b 100644 --- a/lib/gitlab/analytics/cycle_analytics/sorting.rb +++ b/lib/gitlab/analytics/cycle_analytics/sorting.rb @@ -4,23 +4,35 @@ module Gitlab module Analytics module CycleAnalytics class Sorting + include StageQueryHelpers + + def initialize(stage:, query:, params: {}) + @stage = stage + @query = query + @params = params + end + # rubocop: disable CodeReuse/ActiveRecord - SORTING_OPTIONS = { - end_event: { - asc: -> (query, stage) { query.reorder(stage.end_event.timestamp_projection.asc) }, - desc: -> (query, stage) { query.reorder(stage.end_event.timestamp_projection.desc) } - }.freeze, - duration: { - asc: -> (query, stage) { query.reorder(Arel::Nodes::Subtraction.new(stage.end_event.timestamp_projection, stage.start_event.timestamp_projection).asc) }, - desc: -> (query, stage) { query.reorder(Arel::Nodes::Subtraction.new(stage.end_event.timestamp_projection, stage.start_event.timestamp_projection).desc) } - }.freeze - }.freeze - # rubocop: enable CodeReuse/ActiveRecord, + def apply(sort, direction) + sorting_options = { + end_event: { + asc: -> { query.reorder(end_event_timestamp_projection.asc) }, + desc: -> { query.reorder(end_event_timestamp_projection.desc) } + }, + duration: { + asc: -> { query.reorder(duration.asc) }, + desc: -> { query.reorder(duration.desc) } + } + } - def self.apply(query, stage, sort, direction) - sort_lambda = SORTING_OPTIONS.dig(sort, direction) || SORTING_OPTIONS.dig(:end_event, :desc) - sort_lambda.call(query, stage) + sort_lambda = sorting_options.dig(sort, direction) || sorting_options.dig(:end_event, :desc) + sort_lambda.call end + # rubocop: enable CodeReuse/ActiveRecord + + private + + attr_reader :stage, :query, :params end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb index 02b1024b8b3..b7a11bc0418 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb @@ -85,4 +85,4 @@ module Gitlab end end -Gitlab::Analytics::CycleAnalytics::StageEvents.prepend_if_ee('::EE::Gitlab::Analytics::CycleAnalytics::StageEvents') +Gitlab::Analytics::CycleAnalytics::StageEvents.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::StageEvents') diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb index 4bb225b63f1..8e87245e62b 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb @@ -17,11 +17,6 @@ module Gitlab MergeRequest end - def timestamp_projection - Arel::Nodes::NamedFunction.new('COALESCE', column_list) - end - - override :column_list def column_list [ issue_metrics_table[:first_mentioned_in_commit_at], diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb index a159580b7bd..30b457b667c 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb @@ -17,8 +17,8 @@ module Gitlab Issue end - def timestamp_projection - issue_table[:created_at] + def column_list + [issue_table[:created_at]] end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb index 3e93e60e686..4ca3c19051e 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb @@ -17,13 +17,8 @@ module Gitlab Issue end - def timestamp_projection - mr_metrics_table[:first_deployed_to_production_at] - end - - override :column_list def column_list - [timestamp_projection] + [mr_metrics_table[:first_deployed_to_production_at]] end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb index a3b7fa16daf..aa509e8c4d2 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb @@ -17,8 +17,8 @@ module Gitlab Issue end - def timestamp_projection - issue_metrics_table[:first_mentioned_in_commit_at] + def column_list + [issue_metrics_table[:first_mentioned_in_commit_at]] end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb index 7c1f4436c93..284d8534b96 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb @@ -17,11 +17,6 @@ module Gitlab Issue end - def timestamp_projection - Arel::Nodes::NamedFunction.new('COALESCE', column_list) - end - - override :column_list def column_list [ issue_metrics_table[:first_associated_with_milestone_at], diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb index 013e068e479..31249ae2036 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb @@ -17,8 +17,8 @@ module Gitlab MergeRequest end - def timestamp_projection - mr_table[:created_at] + def column_list + [mr_table[:created_at]] end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb index 654d0befbc3..4c0e9b61e64 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb @@ -17,8 +17,8 @@ module Gitlab MergeRequest end - def timestamp_projection - mr_metrics_table[:first_deployed_to_production_at] + def column_list + [mr_metrics_table[:first_deployed_to_production_at]] end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb index a0b1c12756f..178fe03d7db 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb @@ -17,8 +17,8 @@ module Gitlab MergeRequest end - def timestamp_projection - mr_metrics_table[:latest_build_finished_at] + def column_list + [mr_metrics_table[:latest_build_finished_at]] end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb index da3b5cdfaa4..95e59cd29a6 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb @@ -17,8 +17,8 @@ module Gitlab MergeRequest end - def timestamp_projection - mr_metrics_table[:latest_build_started_at] + def column_list + [mr_metrics_table[:latest_build_started_at]] end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb index e67a6f7eea6..00ac2e7d56c 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb @@ -17,8 +17,8 @@ module Gitlab MergeRequest end - def timestamp_projection - mr_metrics_table[:merged_at] + def column_list + [mr_metrics_table[:merged_at]] end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb index fe477490648..fd30ab5277d 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb @@ -11,7 +11,12 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - override :column_list + # rubocop: disable CodeReuse/ActiveRecord + def apply_negated_query_customization(query) + super.joins(:metrics) + end + # rubocop: enable CodeReuse/ActiveRecord + def column_list [timestamp_projection] end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb index bddc326de71..9b4cbc9090c 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb @@ -17,11 +17,6 @@ module Gitlab Issue end - def timestamp_projection - Arel::Nodes::NamedFunction.new('COALESCE', column_list) - end - - override :column_list def column_list [ issue_metrics_table[:first_associated_with_milestone_at], diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb index cfc9300a710..530e53f9d10 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb @@ -34,14 +34,16 @@ module Gitlab # Each StageEvent must expose a timestamp or a timestamp like expression in order to build a range query. # Example: get me all the Issue records between start event end end event def timestamp_projection - raise NotImplementedError + columns = column_list + + columns.one? ? columns.first : Arel::Nodes::NamedFunction.new('COALESCE', columns) end # List of columns that are referenced in the `timestamp_projection` expression # Example timestamp projection: COALESCE(issue_metrics.created_at, issue_metrics.updated_at) # Expected column list: issue_metrics.created_at, issue_metrics.updated_at def column_list - [] + raise NotImplementedError end # Optionally a StageEvent may apply additional filtering or join other tables on the base query. @@ -49,6 +51,12 @@ module Gitlab query end + # rubocop: disable CodeReuse/ActiveRecord + def apply_negated_query_customization(query) + query.where(timestamp_projection.eq(nil)) + end + # rubocop: enable CodeReuse/ActiveRecord + def self.label_based? false end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb index 777a8278e6e..11fe1dde12f 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb @@ -18,22 +18,30 @@ module Gitlab def duration Arel::Nodes::Subtraction.new( - stage.end_event.timestamp_projection, + end_event_timestamp_projection, stage.start_event.timestamp_projection ) end + def end_event_timestamp_projection + if in_progress? + Arel::Nodes::NamedFunction.new('TO_TIMESTAMP', [Time.current.to_i]) + else + stage.end_event.timestamp_projection + end + end + # rubocop: disable CodeReuse/ActiveRecord def order_by(query, sort, direction, extra_columns_to_select = [:id]) - ordered_query = Gitlab::Analytics::CycleAnalytics::Sorting.apply(query, stage, sort, direction) + ordered_query = Gitlab::Analytics::CycleAnalytics::Sorting.new(stage: stage, query: query, params: params).apply(sort, direction) # When filtering for more than one label, postgres requires the columns in ORDER BY to be present in the GROUP BY clause if requires_grouping? - column_list = [ - *extra_columns_to_select, - *stage.end_event.column_list, - *stage.start_event.column_list - ] + column_list = [].tap do |array| + array.concat(extra_columns_to_select) + array.concat(stage.end_event.column_list) unless in_progress? + array.concat(stage.start_event.column_list) + end ordered_query = ordered_query.group(column_list) end @@ -45,6 +53,10 @@ module Gitlab def requires_grouping? Array(params[:label_name]).size > 1 end + + def in_progress? + params[:end_event_filter] == :in_progress + end end end end diff --git a/lib/gitlab/api_authentication/token_locator.rb b/lib/gitlab/api_authentication/token_locator.rb index 09039f3fc43..df342905d2e 100644 --- a/lib/gitlab/api_authentication/token_locator.rb +++ b/lib/gitlab/api_authentication/token_locator.rb @@ -10,7 +10,17 @@ module Gitlab attr_reader :location - validates :location, inclusion: { in: %i[http_basic_auth http_token] } + validates :location, inclusion: { + in: %i[ + http_basic_auth + http_token + http_bearer_token + http_deploy_token_header + http_job_token_header + http_private_token_header + token_param + ] + } def initialize(location) @location = location @@ -23,6 +33,16 @@ module Gitlab extract_from_http_basic_auth request when :http_token extract_from_http_token request + when :http_bearer_token + extract_from_http_bearer_token request + when :http_deploy_token_header + extract_from_http_deploy_token_header request + when :http_job_token_header + extract_from_http_job_token_header request + when :http_private_token_header + extract_from_http_private_token_header request + when :token_param + extract_from_token_param request end end @@ -41,6 +61,41 @@ module Gitlab UsernameAndPassword.new(nil, password) end + + def extract_from_http_bearer_token(request) + password = request.headers['Authorization'] + return unless password.present? + + UsernameAndPassword.new(nil, password.split(' ').last) + end + + def extract_from_http_deploy_token_header(request) + password = request.headers['Deploy-Token'] + return unless password.present? + + UsernameAndPassword.new(nil, password) + end + + def extract_from_http_job_token_header(request) + password = request.headers['Job-Token'] + return unless password.present? + + UsernameAndPassword.new(nil, password) + end + + def extract_from_http_private_token_header(request) + password = request.headers['Private-Token'] + return unless password.present? + + UsernameAndPassword.new(nil, password) + end + + def extract_from_token_param(request) + password = request.query_parameters['token'] + return unless password.present? + + UsernameAndPassword.new(nil, password) + end end end end diff --git a/lib/gitlab/api_authentication/token_resolver.rb b/lib/gitlab/api_authentication/token_resolver.rb index 9234837cdf7..dd9039e37f6 100644 --- a/lib/gitlab/api_authentication/token_resolver.rb +++ b/lib/gitlab/api_authentication/token_resolver.rb @@ -15,9 +15,14 @@ module Gitlab personal_access_token job_token deploy_token + personal_access_token_from_jwt + deploy_token_from_jwt + job_token_from_jwt ] } + UsernameAndPassword = ::Gitlab::APIAuthentication::TokenLocator::UsernameAndPassword + def initialize(token_type) @token_type = token_type validate! @@ -56,6 +61,15 @@ module Gitlab when :deploy_token_with_username resolve_deploy_token_with_username raw + + when :personal_access_token_from_jwt + resolve_personal_access_token_from_jwt raw + + when :deploy_token_from_jwt + resolve_deploy_token_from_jwt raw + + when :job_token_from_jwt + resolve_job_token_from_jwt raw end end @@ -116,6 +130,33 @@ module Gitlab end end + def resolve_personal_access_token_from_jwt(raw) + with_jwt_token(raw) do |jwt_token| + break unless jwt_token['token'].is_a?(Integer) + + pat = ::PersonalAccessToken.find(jwt_token['token']) + break unless pat + + pat + end + end + + def resolve_deploy_token_from_jwt(raw) + with_jwt_token(raw) do |jwt_token| + break unless jwt_token['token'].is_a?(String) + + resolve_deploy_token(UsernameAndPassword.new(nil, jwt_token['token'])) + end + end + + def resolve_job_token_from_jwt(raw) + with_jwt_token(raw) do |jwt_token| + break unless jwt_token['token'].is_a?(String) + + resolve_job_token(UsernameAndPassword.new(nil, jwt_token['token'])) + end + end + def with_personal_access_token(raw, &block) pat = ::PersonalAccessToken.find_by_token(raw.password) return unless pat @@ -136,6 +177,13 @@ module Gitlab yield(job) end + + def with_jwt_token(raw, &block) + jwt_token = ::Gitlab::JWTToken.decode(raw.password) + raise ::Gitlab::Auth::UnauthorizedError unless jwt_token + + yield(jwt_token) + end end end end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index ceda82cb6f6..601f2175cfc 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -135,4 +135,4 @@ module Gitlab end end -Gitlab::ApplicationContext.prepend_if_ee('EE::Gitlab::ApplicationContext') +Gitlab::ApplicationContext.prepend_mod_with('Gitlab::ApplicationContext') diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index f74edf2b767..f91a56a0cd2 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -34,6 +34,7 @@ module Gitlab group_import: { threshold: -> { application_settings.group_import_limit }, interval: 1.minute }, group_testing_hook: { threshold: 5, interval: 1.minute }, profile_add_new_email: { threshold: 5, interval: 1.minute }, + web_hook_calls: { interval: 1.minute }, profile_resend_email_confirmation: { threshold: 5, interval: 1.minute }, update_environment_canary_ingress: { threshold: 1, interval: 1.minute }, auto_rollback_deployment: { threshold: 1, interval: 3.minutes } diff --git a/lib/gitlab/artifacts/migration_helper.rb b/lib/gitlab/artifacts/migration_helper.rb deleted file mode 100644 index 4f047ab3ea8..00000000000 --- a/lib/gitlab/artifacts/migration_helper.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Artifacts - class MigrationHelper - def migrate_to_remote_storage(&block) - artifacts = ::Ci::JobArtifact.with_files_stored_locally - migrate(artifacts, ObjectStorage::Store::REMOTE, &block) - end - - def migrate_to_local_storage(&block) - artifacts = ::Ci::JobArtifact.with_files_stored_remotely - migrate(artifacts, ObjectStorage::Store::LOCAL, &block) - end - - private - - def batch_size - ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i - end - - def migrate(artifacts, store, &block) - artifacts.find_each(batch_size: batch_size) do |artifact| # rubocop:disable CodeReuse/ActiveRecord - artifact.file.migrate!(store) - - yield artifact if block - rescue => e - raise StandardError.new("Failed to transfer artifact of type #{artifact.file_type} and ID #{artifact.id} with error: #{e.message}") - end - end - end - end -end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 1f5cce249d8..c6997288b65 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -29,7 +29,7 @@ module Gitlab CI_JOB_USER = 'gitlab-ci-token' class << self - prepend_if_ee('EE::Gitlab::Auth') # rubocop: disable Cop/InjectEnterpriseEditionModule + prepend_mod_with('Gitlab::Auth') # rubocop: disable Cop/InjectEnterpriseEditionModule def omniauth_enabled? Gitlab.config.omniauth.enabled @@ -156,9 +156,9 @@ module Gitlab underscored_service = matched_login['service'].underscore - if Service.available_services_names.include?(underscored_service) + if Integration.available_services_names.include?(underscored_service) # We treat underscored_service as a trusted input because it is included - # in the Service.available_services_names allowlist. + # in the Integration.available_services_names allowlist. service = project.public_send("#{underscored_service}_service") # rubocop:disable GitlabSecurity/PublicSend if service && service.activated? && service.valid_token?(password) diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index 6f6ac79c16b..416e36c7ccb 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -160,7 +160,7 @@ module Gitlab case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes) when AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) + raise InsufficientScopeError, scopes when AccessTokenValidationService::EXPIRED raise ExpiredError when AccessTokenValidationService::REVOKED @@ -321,4 +321,4 @@ module Gitlab end end -Gitlab::Auth::AuthFinders.prepend_if_ee('::EE::Gitlab::Auth::AuthFinders') +Gitlab::Auth::AuthFinders.prepend_mod_with('Gitlab::Auth::AuthFinders') diff --git a/lib/gitlab/auth/database/authentication.rb b/lib/gitlab/auth/database/authentication.rb index c0dc2b0875f..bf35a9abe41 100644 --- a/lib/gitlab/auth/database/authentication.rb +++ b/lib/gitlab/auth/database/authentication.rb @@ -9,6 +9,7 @@ module Gitlab class Authentication < Gitlab::Auth::OAuth::Authentication def login(login, password) return false unless Gitlab::CurrentSettings.password_authentication_enabled_for_git? + return false if user.password_based_login_forbidden? return user if user&.valid_password?(password) end diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb index 66d20ee2b59..62a817d7c4d 100644 --- a/lib/gitlab/auth/ldap/access.rb +++ b/lib/gitlab/auth/ldap/access.rb @@ -117,4 +117,4 @@ module Gitlab end end -Gitlab::Auth::Ldap::Access.prepend_if_ee('::EE::Gitlab::Auth::Ldap::Access') +Gitlab::Auth::Ldap::Access.prepend_mod_with('Gitlab::Auth::Ldap::Access') diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index 7f85d3b1cd3..3853709698b 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -141,4 +141,4 @@ module Gitlab end end -Gitlab::Auth::Ldap::Adapter.prepend_if_ee('::EE::Gitlab::Auth::Ldap::Adapter') +Gitlab::Auth::Ldap::Adapter.prepend_mod_with('Gitlab::Auth::Ldap::Adapter') diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 97e4f921228..441f0d14b39 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -59,7 +59,7 @@ module Gitlab end def self.invalid_provider(provider) - raise InvalidProvider.new("Unknown provider (#{provider}). Available providers: #{providers}") + raise InvalidProvider, "Unknown provider (#{provider}). Available providers: #{providers}" end def self.encrypted_secrets @@ -288,7 +288,7 @@ module Gitlab def secrets @secrets ||= self.class.encrypted_secrets[@provider.delete_prefix('ldap').to_sym] - rescue => e + rescue StandardError => e Gitlab::AppLogger.error "LDAP encrypted secrets are invalid: #{e.inspect}" nil @@ -320,4 +320,4 @@ module Gitlab end end -Gitlab::Auth::Ldap::Config.prepend_if_ee('::EE::Gitlab::Auth::Ldap::Config') +Gitlab::Auth::Ldap::Config.prepend_mod_with('Gitlab::Auth::Ldap::Config') diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb index 102820d6bd5..79e1937478c 100644 --- a/lib/gitlab/auth/ldap/person.rb +++ b/lib/gitlab/auth/ldap/person.rb @@ -121,4 +121,4 @@ module Gitlab end end -Gitlab::Auth::Ldap::Person.prepend_if_ee('::EE::Gitlab::Auth::Ldap::Person') +Gitlab::Auth::Ldap::Person.prepend_mod_with('Gitlab::Auth::Ldap::Person') diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb index 814c17b7e44..d134350775d 100644 --- a/lib/gitlab/auth/ldap/user.rb +++ b/lib/gitlab/auth/ldap/user.rb @@ -49,4 +49,4 @@ module Gitlab end end -Gitlab::Auth::Ldap::User.prepend_if_ee('::EE::Gitlab::Auth::Ldap::User') +Gitlab::Auth::Ldap::User.prepend_mod_with('Gitlab::Auth::Ldap::User') diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index 46ff6b2ccab..2ec75669d24 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -81,7 +81,7 @@ module Gitlab # Get the first part of the email address (before @) # In addition in removes illegal characters def generate_username(email) - email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/, '').to_s + email.match(/^[^@]*/)[0].mb_chars.unicode_normalize(:nfkd).gsub(/[^\x00-\x7F]/, '').to_s end def generate_temporarily_email(username) @@ -92,4 +92,4 @@ module Gitlab end end -Gitlab::Auth::OAuth::AuthHash.prepend_if_ee('::EE::Gitlab::Auth::OAuth::AuthHash') +Gitlab::Auth::OAuth::AuthHash.prepend_mod_with('Gitlab::Auth::OAuth::AuthHash') diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index fe1bf730e76..523452d1074 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -115,6 +115,8 @@ module Gitlab log.info "Correct LDAP account has been found. identity to user: #{gl_user.username}." gl_user.identities.build(provider: ldap_person.provider, extern_uid: ldap_person.dn) end + + identity end def find_or_build_ldap_user @@ -292,4 +294,4 @@ module Gitlab end end -Gitlab::Auth::OAuth::User.prepend_if_ee('::EE::Gitlab::Auth::OAuth::User') +Gitlab::Auth::OAuth::User.prepend_mod_with('Gitlab::Auth::OAuth::User') diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb index 757a0e671c3..da874524826 100644 --- a/lib/gitlab/auth/result.rb +++ b/lib/gitlab/auth/result.rb @@ -25,4 +25,4 @@ module Gitlab end end -Gitlab::Auth::Result.prepend_if_ee('::EE::Gitlab::Auth::Result') +Gitlab::Auth::Result.prepend_mod_with('Gitlab::Auth::Result') diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index 67a53fa3205..3f13a264b0a 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -30,4 +30,4 @@ module Gitlab end end -Gitlab::Auth::Saml::Config.prepend_if_ee('::EE::Gitlab::Auth::Saml::Config') +Gitlab::Auth::Saml::Config.prepend_mod_with('Gitlab::Auth::Saml::Config') diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb index 37bc3f9bed0..205d5fe0015 100644 --- a/lib/gitlab/auth/saml/user.rb +++ b/lib/gitlab/auth/saml/user.rb @@ -62,4 +62,4 @@ module Gitlab end end -Gitlab::Auth::Saml::User.prepend_if_ee('::EE::Gitlab::Auth::Saml::User') +Gitlab::Auth::Saml::User.prepend_mod_with('Gitlab::Auth::Saml::User') diff --git a/lib/gitlab/authorized_keys.rb b/lib/gitlab/authorized_keys.rb index 50cd15b7a10..e7eba65bea8 100644 --- a/lib/gitlab/authorized_keys.rb +++ b/lib/gitlab/authorized_keys.rb @@ -161,7 +161,7 @@ module Gitlab end def strip(key) - key.split(/[ ]+/)[0, 2].join(' ') + key.split(/ +/)[0, 2].join(' ') end end end diff --git a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb new file mode 100644 index 00000000000..79e7a2f2279 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to set namespaces.traversal_ids in sub-batches, of all namespaces with + # a parent and not already set. + # rubocop:disable Style/Documentation + class BackfillNamespaceTraversalIdsChildren + class Namespace < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'namespaces' + + scope :base_query, -> { where.not(parent_id: nil) } + end + + PAUSE_SECONDS = 0.1 + + def perform(start_id, end_id, sub_batch_size) + batch_query = Namespace.base_query.where(id: start_id..end_id) + batch_query.each_batch(of: sub_batch_size) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(id), max(id)')).first + ranged_query = Namespace.unscoped.base_query.where(id: first..last) + + update_sql = <<~SQL + UPDATE namespaces + SET traversal_ids = calculated_ids.traversal_ids + FROM #{calculated_traversal_ids(ranged_query)} calculated_ids + WHERE namespaces.id = calculated_ids.id + AND namespaces.traversal_ids = '{}' + SQL + ActiveRecord::Base.connection.execute(update_sql) + + sleep PAUSE_SECONDS + end + + # We have to add all arguments when marking a job as succeeded as they + # are all used to track the job by `queue_background_migration_jobs_by_range_at_intervals` + mark_job_as_succeeded(start_id, end_id, sub_batch_size) + end + + private + + # Calculate the ancestor path for a given set of namespaces. + def calculated_traversal_ids(batch) + <<~SQL + ( + WITH RECURSIVE cte(source_id, namespace_id, parent_id, height) AS ( + ( + SELECT batch.id, batch.id, batch.parent_id, 1 + FROM (#{batch.to_sql}) AS batch + ) + UNION ALL + ( + SELECT cte.source_id, n.id, n.parent_id, cte.height+1 + FROM namespaces n, cte + WHERE n.id = cte.parent_id + ) + ) + SELECT flat_hierarchy.source_id as id, + array_agg(flat_hierarchy.namespace_id ORDER BY flat_hierarchy.height DESC) as traversal_ids + FROM (SELECT * FROM cte FOR UPDATE) flat_hierarchy + GROUP BY flat_hierarchy.source_id + ) + SQL + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'BackfillNamespaceTraversalIdsChildren', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb new file mode 100644 index 00000000000..1c0a83285a6 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to set namespaces.traversal_ids in sub-batches, of all namespaces + # without a parent and not already set. + # rubocop:disable Style/Documentation + class BackfillNamespaceTraversalIdsRoots + class Namespace < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'namespaces' + + scope :base_query, -> { where(parent_id: nil) } + end + + PAUSE_SECONDS = 0.1 + + def perform(start_id, end_id, sub_batch_size) + ranged_query = Namespace.base_query + .where(id: start_id..end_id) + .where("traversal_ids = '{}'") + + ranged_query.each_batch(of: sub_batch_size) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(id), max(id)')).first + + # The query need to be reconstructed because .each_batch modifies the default scope + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510 + Namespace.unscoped + .base_query + .where(id: first..last) + .where("traversal_ids = '{}'") + .update_all('traversal_ids = ARRAY[id]') + + sleep PAUSE_SECONDS + end + + mark_job_as_succeeded(start_id, end_id, sub_batch_size) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'BackfillNamespaceTraversalIdsRoots', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_snippet_repositories.rb b/lib/gitlab/background_migration/backfill_snippet_repositories.rb index 8befade8c3a..6f37f1846d2 100644 --- a/lib/gitlab/background_migration/backfill_snippet_repositories.rb +++ b/lib/gitlab/background_migration/backfill_snippet_repositories.rb @@ -36,7 +36,7 @@ module Gitlab create_repository_and_files(snippet) logger.info(message: 'Snippet Migration: repository created and migrated', snippet: snippet.id) - rescue => e + rescue StandardError => e set_file_path_error(e) set_signature_error(e) @@ -68,7 +68,7 @@ module Gitlab # Removing the db record def destroy_snippet_repository(snippet) snippet.snippet_repository&.delete - rescue => e + rescue StandardError => e logger.error(message: "Snippet Migration: error destroying snippet repository. Reason: #{e.message}", snippet: snippet.id) end @@ -78,7 +78,7 @@ module Gitlab snippet.repository.remove snippet.repository.expire_exists_cache - rescue => e + rescue StandardError => e logger.error(message: "Snippet Migration: error deleting repository. Reason: #{e.message}", snippet: snippet.id) end diff --git a/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb b/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb index 83d60d2db19..41f7f7f2f24 100644 --- a/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb +++ b/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly.prepend_if_ee('EE::Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly') +Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly.prepend_mod_with('Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly') diff --git a/lib/gitlab/background_migration/calculate_wiki_sizes.rb b/lib/gitlab/background_migration/calculate_wiki_sizes.rb index 76598f6e2a6..7b334b9c1d0 100644 --- a/lib/gitlab/background_migration/calculate_wiki_sizes.rb +++ b/lib/gitlab/background_migration/calculate_wiki_sizes.rb @@ -9,7 +9,7 @@ module Gitlab .where(id: start_id..stop_id) .includes(project: [:route, :group, namespace: [:owner]]).find_each do |statistics| statistics.refresh!(only: [:wiki_size]) - rescue => e + rescue StandardError => e Gitlab::AppLogger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" end end diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb index b89ea7dc250..529b8cdf8d4 100644 --- a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb +++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb @@ -2,8 +2,8 @@ module Gitlab module BackgroundMigration - # Background migration that updates the value of a - # column using the value of another column in the same table. + # Background migration that updates the value of one or more + # columns using the value of other columns in the same table. # # - The {start_id, end_id} arguments are at the start so that it can be used # with `queue_batched_background_migration` @@ -16,8 +16,6 @@ module Gitlab class CopyColumnUsingBackgroundMigrationJob include Gitlab::Database::DynamicModelHelpers - PAUSE_SECONDS = 0.1 - # start_id - The start ID of the range of rows to update. # end_id - The end ID of the range of rows to update. # batch_table - The name of the table that contains the columns. @@ -25,20 +23,26 @@ module Gitlab # sub_batch_size - We don't want updates to take more than ~100ms # This allows us to run multiple smaller batches during # the minimum 2.minute interval that we can schedule jobs - # copy_from - The column containing the data to copy. - # copy_to - The column to copy the data to. - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, copy_from, copy_to) - quoted_copy_from = connection.quote_column_name(copy_from) - quoted_copy_to = connection.quote_column_name(copy_to) + # pause_ms - The number of milliseconds to sleep between each subbatch execution. + # copy_from - List of columns containing the data to copy. + # copy_to - List of columns to copy the data to. Order must match the order in `copy_from`. + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms, copy_from, copy_to) + copy_from = Array.wrap(copy_from) + copy_to = Array.wrap(copy_to) + + raise ArgumentError, 'number of source and destination columns must match' unless copy_from.count == copy_to.count + + assignment_clauses = column_assignment_clauses(copy_from, copy_to) parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| batch_metrics.time_operation(:update_all) do - sub_batch.update_all("#{quoted_copy_to}=#{quoted_copy_from}") + sub_batch.update_all(assignment_clauses) end - sleep(PAUSE_SECONDS) + pause_ms = 0 if pause_ms < 0 + sleep(pause_ms * 0.001) end end @@ -55,6 +59,17 @@ module Gitlab def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) define_batchable_model(source_table).where(source_key_column => start_id..stop_id) end + + def column_assignment_clauses(copy_from, copy_to) + assignments = copy_from.zip(copy_to).map do |from_column, to_column| + from_column = connection.quote_column_name(from_column) + to_column = connection.quote_column_name(to_column) + + "#{to_column} = #{from_column}" + end + + assignments.join(', ') + end end end end diff --git a/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb b/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb new file mode 100644 index 00000000000..293530f6536 --- /dev/null +++ b/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# rubocop: disable Style/Documentation +class Gitlab::BackgroundMigration::DropInvalidVulnerabilities + # rubocop: disable Gitlab/NamespacedClass + class Vulnerability < ActiveRecord::Base + self.table_name = "vulnerabilities" + has_many :findings, class_name: 'VulnerabilitiesFinding', inverse_of: :vulnerability + end + + class VulnerabilitiesFinding < ActiveRecord::Base + self.table_name = "vulnerability_occurrences" + belongs_to :vulnerability, class_name: 'Vulnerability', inverse_of: :findings, foreign_key: 'vulnerability_id' + end + # rubocop: enable Gitlab/NamespacedClass + + # rubocop: disable CodeReuse/ActiveRecord + def perform(start_id, end_id) + Vulnerability + .where(id: start_id..end_id) + .left_joins(:findings) + .where(vulnerability_occurrences: { vulnerability_id: nil }) + .delete_all + + mark_job_as_succeeded(start_id, end_id) + end + # rubocop: enable CodeReuse/ActiveRecord + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'DropInvalidVulnerabilities', + arguments + ) + end +end diff --git a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb index c0099d44b5a..7b5c32e3d6d 100644 --- a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb +++ b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb @@ -24,7 +24,7 @@ module Gitlab certificate_valid_not_before: domain.x509&.not_before&.iso8601, certificate_valid_not_after: domain.x509&.not_after&.iso8601 ) - rescue => e + rescue StandardError => e Gitlab::AppLogger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" end end diff --git a/lib/gitlab/background_migration/fix_orphan_promoted_issues.rb b/lib/gitlab/background_migration/fix_orphan_promoted_issues.rb index d6ec56ae19e..c50bf430d92 100644 --- a/lib/gitlab/background_migration/fix_orphan_promoted_issues.rb +++ b/lib/gitlab/background_migration/fix_orphan_promoted_issues.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::FixOrphanPromotedIssues.prepend_if_ee('EE::Gitlab::BackgroundMigration::FixOrphanPromotedIssues') +Gitlab::BackgroundMigration::FixOrphanPromotedIssues.prepend_mod_with('Gitlab::BackgroundMigration::FixOrphanPromotedIssues') diff --git a/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb b/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb index 46921a070c3..47a68c61fcc 100644 --- a/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb +++ b/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents.prepend_if_ee('EE::Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents') +Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents.prepend_mod_with('Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents') diff --git a/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb b/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb index 85bcf8558f2..160e6d2fe8b 100644 --- a/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb +++ b/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::GenerateGitlabSubscriptions.prepend_if_ee('EE::Gitlab::BackgroundMigration::GenerateGitlabSubscriptions') +Gitlab::BackgroundMigration::GenerateGitlabSubscriptions.prepend_mod_with('Gitlab::BackgroundMigration::GenerateGitlabSubscriptions') diff --git a/lib/gitlab/background_migration/migrate_approver_to_approval_rules.rb b/lib/gitlab/background_migration/migrate_approver_to_approval_rules.rb index 27b984b4531..ba66721f65c 100644 --- a/lib/gitlab/background_migration/migrate_approver_to_approval_rules.rb +++ b/lib/gitlab/background_migration/migrate_approver_to_approval_rules.rb @@ -12,4 +12,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MigrateApproverToApprovalRules.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRules') +Gitlab::BackgroundMigration::MigrateApproverToApprovalRules.prepend_mod_with('Gitlab::BackgroundMigration::MigrateApproverToApprovalRules') diff --git a/lib/gitlab/background_migration/migrate_approver_to_approval_rules_check_progress.rb b/lib/gitlab/background_migration/migrate_approver_to_approval_rules_check_progress.rb index 053b7363286..4899c50b9cf 100644 --- a/lib/gitlab/background_migration/migrate_approver_to_approval_rules_check_progress.rb +++ b/lib/gitlab/background_migration/migrate_approver_to_approval_rules_check_progress.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress') +Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress.prepend_mod_with('Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress') diff --git a/lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb b/lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb index 130f97b09d7..2855566d7e8 100644 --- a/lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb +++ b/lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch') +Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch.prepend_mod_with('Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch') diff --git a/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb b/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb index de2d9909961..d85f980d3f1 100644 --- a/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb +++ b/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups') +Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups.prepend_mod_with('Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups') diff --git a/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb b/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb new file mode 100644 index 00000000000..68bbd3cfebb --- /dev/null +++ b/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to migrate the context of project taggings from `tags` to `topics` + class MigrateProjectTaggingsContextFromTagsToTopics + # Temporary AR table for taggings + class Tagging < ActiveRecord::Base + include EachBatch + + self.table_name = 'taggings' + end + + def perform(start_id, stop_id) + Tagging.where(taggable_type: 'Project', context: 'tags', id: start_id..stop_id).each_batch(of: 500) do |relation| + relation.update_all(context: 'topics') + end + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_security_scans.rb b/lib/gitlab/background_migration/migrate_security_scans.rb index 189a150cb87..0ae984f2dbc 100644 --- a/lib/gitlab/background_migration/migrate_security_scans.rb +++ b/lib/gitlab/background_migration/migrate_security_scans.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MigrateSecurityScans.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateSecurityScans') +Gitlab::BackgroundMigration::MigrateSecurityScans.prepend_mod_with('Gitlab::BackgroundMigration::MigrateSecurityScans') diff --git a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb index 9ecf53317d0..c01545e5dca 100644 --- a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb +++ b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb @@ -8,18 +8,15 @@ module Gitlab class MoveContainerRegistryEnabledToProjectFeature MAX_BATCH_SIZE = 300 - module Migratable - # Migration model namespace isolated from application code. - class ProjectFeature < ActiveRecord::Base - ENABLED = 20 - DISABLED = 0 - end - end + ENABLED = 20 + DISABLED = 0 def perform(from_id, to_id) (from_id..to_id).each_slice(MAX_BATCH_SIZE) do |batch| process_batch(batch.first, batch.last) end + + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('MoveContainerRegistryEnabledToProjectFeature', [from_id, to_id]) end private @@ -37,9 +34,9 @@ module Gitlab <<~SQL UPDATE project_features SET container_registry_access_level = (CASE p.container_registry_enabled - WHEN true THEN #{ProjectFeature::ENABLED} - WHEN false THEN #{ProjectFeature::DISABLED} - ELSE #{ProjectFeature::DISABLED} + WHEN true THEN #{ENABLED} + WHEN false THEN #{DISABLED} + ELSE #{DISABLED} END) FROM projects p WHERE project_id = p.id AND diff --git a/lib/gitlab/background_migration/move_epic_issues_after_epics.rb b/lib/gitlab/background_migration/move_epic_issues_after_epics.rb index dc982e703d1..174994c7862 100644 --- a/lib/gitlab/background_migration/move_epic_issues_after_epics.rb +++ b/lib/gitlab/background_migration/move_epic_issues_after_epics.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics.prepend_if_ee('EE::Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics') +Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics.prepend_mod_with('Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics') diff --git a/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb b/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb index c3c0db2495c..890a43800c9 100644 --- a/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb +++ b/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb @@ -11,4 +11,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests') +Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests.prepend_mod_with('Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests') diff --git a/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb b/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb index 2243c7531c0..ac7ed18ba14 100644 --- a/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb +++ b/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb @@ -11,4 +11,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects') +Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects.prepend_mod_with('Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects') diff --git a/lib/gitlab/background_migration/populate_namespace_statistics.rb b/lib/gitlab/background_migration/populate_namespace_statistics.rb index e352ae71de6..e873ad412f2 100644 --- a/lib/gitlab/background_migration/populate_namespace_statistics.rb +++ b/lib/gitlab/background_migration/populate_namespace_statistics.rb @@ -13,4 +13,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateNamespaceStatistics.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateNamespaceStatistics') +Gitlab::BackgroundMigration::PopulateNamespaceStatistics.prepend_mod_with('Gitlab::BackgroundMigration::PopulateNamespaceStatistics') diff --git a/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb b/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb index e8f436b183e..ed7ffce8018 100644 --- a/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb +++ b/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb @@ -33,7 +33,7 @@ module Gitlab def update_namespace_statistics(namespace) Namespaces::StatisticsRefresherService.new.execute(namespace) - rescue => e + rescue StandardError => e error_message("Error updating statistics for namespace #{namespace.id}: #{e.message}") end diff --git a/lib/gitlab/background_migration/populate_project_snippet_statistics.rb b/lib/gitlab/background_migration/populate_project_snippet_statistics.rb index 7659b63271f..37af320f044 100644 --- a/lib/gitlab/background_migration/populate_project_snippet_statistics.rb +++ b/lib/gitlab/background_migration/populate_project_snippet_statistics.rb @@ -11,12 +11,12 @@ module Gitlab namespace_snippets.group_by(&:project).each do |project, snippets| upsert_snippet_statistics(snippets) update_project_statistics(project) - rescue + rescue StandardError error_message("Error updating statistics for project #{project.id}") end update_namespace_statistics(namespace_snippets.first.project.root_namespace) - rescue => e + rescue StandardError => e error_message("Error updating statistics for namespace #{namespace_id}: #{e.message}") end end diff --git a/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb b/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb index eb72ef1de33..e95955c450d 100644 --- a/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb +++ b/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb @@ -9,4 +9,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn') +Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn.prepend_mod_with('Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn') diff --git a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb index 4aff9d1e2c1..175966b940d 100644 --- a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb +++ b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb @@ -15,4 +15,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings.prepend_if_ee('::EE::Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings') +Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings.prepend_mod_with('Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings') diff --git a/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb b/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb index fc79f7125e3..8241fea66db 100644 --- a/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb +++ b/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId') +Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId.prepend_mod_with('Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId') diff --git a/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb index 2e81b1615d8..9a9f23e29ea 100644 --- a/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb +++ b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb @@ -11,4 +11,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics') +Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics.prepend_mod_with('Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics') diff --git a/lib/gitlab/background_migration/prune_orphaned_geo_events.rb b/lib/gitlab/background_migration/prune_orphaned_geo_events.rb index 8b16db8be35..0efbe72775c 100644 --- a/lib/gitlab/background_migration/prune_orphaned_geo_events.rb +++ b/lib/gitlab/background_migration/prune_orphaned_geo_events.rb @@ -14,4 +14,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PruneOrphanedGeoEvents.prepend_if_ee('EE::Gitlab::BackgroundMigration::PruneOrphanedGeoEvents') +Gitlab::BackgroundMigration::PruneOrphanedGeoEvents.prepend_mod_with('Gitlab::BackgroundMigration::PruneOrphanedGeoEvents') diff --git a/lib/gitlab/background_migration/recalculate_project_authorizations.rb b/lib/gitlab/background_migration/recalculate_project_authorizations.rb index 3d2ce9fc10c..6a250a96c94 100644 --- a/lib/gitlab/background_migration/recalculate_project_authorizations.rb +++ b/lib/gitlab/background_migration/recalculate_project_authorizations.rb @@ -5,37 +5,7 @@ module Gitlab # rubocop:disable Style/Documentation class RecalculateProjectAuthorizations def perform(user_ids) - user_ids.each do |user_id| - user = User.find_by(id: user_id) - - next unless user - - service = Users::RefreshAuthorizedProjectsService.new( - user, - incorrect_auth_found_callback: - ->(project_id, access_level) do - logger.info(message: 'Removing ProjectAuthorizations', - user_id: user.id, - project_id: project_id, - access_level: access_level) - end, - missing_auth_found_callback: - ->(project_id, access_level) do - logger.info(message: 'Creating ProjectAuthorizations', - user_id: user.id, - project_id: project_id, - access_level: access_level) - end - ) - - service.execute - end - end - - private - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build + # no-op end end end diff --git a/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb b/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb index cc9b0329556..17ef6dec4c0 100644 --- a/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb +++ b/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveDuplicateCsFindings.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveDuplicateCsFindings') +Gitlab::BackgroundMigration::RemoveDuplicateCsFindings.prepend_mod_with('Gitlab::BackgroundMigration::RemoveDuplicateCsFindings') diff --git a/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb index cd305adc7cd..e5772fc7375 100644 --- a/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb +++ b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId') +Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId.prepend_mod_with('Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId') diff --git a/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb b/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb index 74c48b237cc..cb6a600a525 100644 --- a/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb +++ b/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveInaccessibleEpicTodos.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveInaccessibleEpicTodos') +Gitlab::BackgroundMigration::RemoveInaccessibleEpicTodos.prepend_mod_with('Gitlab::BackgroundMigration::RemoveInaccessibleEpicTodos') diff --git a/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb b/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb index 3920e8dc2de..540ffc6f548 100644 --- a/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb +++ b/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel') +Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel.prepend_mod_with('Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel') diff --git a/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb b/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb index f137e41c728..cecb385afa0 100644 --- a/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb +++ b/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceSeverityLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceSeverityLevel') +Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceSeverityLevel.prepend_mod_with('Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceSeverityLevel') diff --git a/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb b/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb index f6ea61f4502..4be61bfb689 100644 --- a/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb +++ b/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel') +Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel.prepend_mod_with('Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel') diff --git a/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb b/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb index 95540cd5f49..1ea483f929f 100644 --- a/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb +++ b/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel') +Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel.prepend_mod_with('Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel') diff --git a/lib/gitlab/background_migration/sync_blocking_issues_count.rb b/lib/gitlab/background_migration/sync_blocking_issues_count.rb index 6262320128c..49a632952fb 100644 --- a/lib/gitlab/background_migration/sync_blocking_issues_count.rb +++ b/lib/gitlab/background_migration/sync_blocking_issues_count.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::SyncBlockingIssuesCount.prepend_if_ee('EE::Gitlab::BackgroundMigration::SyncBlockingIssuesCount') +Gitlab::BackgroundMigration::SyncBlockingIssuesCount.prepend_mod_with('Gitlab::BackgroundMigration::SyncBlockingIssuesCount') diff --git a/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb b/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb index 651df36fcfd..054b918dade 100644 --- a/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb +++ b/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings') +Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings.prepend_mod_with('Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings') diff --git a/lib/gitlab/background_migration/update_timelogs_project_id.rb b/lib/gitlab/background_migration/update_timelogs_project_id.rb new file mode 100644 index 00000000000..24c9967b88e --- /dev/null +++ b/lib/gitlab/background_migration/update_timelogs_project_id.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class to populate project_id for timelogs + class UpdateTimelogsProjectId + BATCH_SIZE = 1000 + + def perform(start_id, stop_id) + (start_id..stop_id).step(BATCH_SIZE).each do |offset| + update_issue_timelogs(offset, offset + BATCH_SIZE) + update_merge_request_timelogs(offset, offset + BATCH_SIZE) + end + end + + def update_issue_timelogs(batch_start, batch_stop) + execute(<<~SQL) + UPDATE timelogs + SET project_id = issues.project_id + FROM issues + WHERE issues.id = timelogs.issue_id + AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop} + AND timelogs.project_id IS NULL; + SQL + end + + def update_merge_request_timelogs(batch_start, batch_stop) + execute(<<~SQL) + UPDATE timelogs + SET project_id = merge_requests.target_project_id + FROM merge_requests + WHERE merge_requests.id = timelogs.merge_request_id + AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop} + AND timelogs.project_id IS NULL; + SQL + end + + def execute(sql) + @connection ||= ::ActiveRecord::Base.connection + @connection.execute(sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb b/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb index bfe9f673b53..1cc03f061fb 100644 --- a/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb +++ b/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback') +Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback.prepend_mod_with('Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback') diff --git a/lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb b/lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb index a2940cba6fa..60adb6b7e3e 100644 --- a/lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb +++ b/lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed') +Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed.prepend_mod_with('Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed') diff --git a/lib/gitlab/background_migration/update_vulnerability_confidence.rb b/lib/gitlab/background_migration/update_vulnerability_confidence.rb index 6ffaa836f3c..40d29978dd4 100644 --- a/lib/gitlab/background_migration/update_vulnerability_confidence.rb +++ b/lib/gitlab/background_migration/update_vulnerability_confidence.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence') +Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence.prepend_mod_with('Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence') diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb index a2b50c41f4a..d76d06606ee 100644 --- a/lib/gitlab/background_migration/user_mentions/models/namespace.rb +++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb @@ -38,4 +38,4 @@ module Gitlab end end -Namespace.prepend_if_ee('::EE::Namespace') +Namespace.prepend_mod_with('Namespace') diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index ab7a08ffef9..44106897df8 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -13,7 +13,7 @@ module Gitlab repos_to_import = Dir.glob(import_path + '**/*.git') unless user = User.admins.order_id_asc.first - raise NoAdminError.new('No admin user found to import repositories') + raise NoAdminError, 'No admin user found to import repositories' end repos_to_import.each do |repo_path| @@ -92,7 +92,7 @@ module Gitlab end true - rescue => e + rescue StandardError => e log " * Failed to move repo: #{e.message}".color(:red) false diff --git a/lib/gitlab/blob_helper.rb b/lib/gitlab/blob_helper.rb index 57d632afd74..c5b183d113d 100644 --- a/lib/gitlab/blob_helper.rb +++ b/lib/gitlab/blob_helper.rb @@ -38,7 +38,7 @@ module Gitlab # If Charlock says its binary else - detect_encoding[:type] == :binary + find_encoding[:type] == :binary end end @@ -137,23 +137,25 @@ module Gitlab end def ruby_encoding - if hash = detect_encoding + if hash = find_encoding hash[:ruby_encoding] end end def encoding - if hash = detect_encoding + if hash = find_encoding hash[:encoding] end end - def detect_encoding - @detect_encoding ||= CharlockHolmes::EncodingDetector.new.detect(data) if data # rubocop:disable Gitlab/ModuleWithInstanceVariables - end - def empty? data.nil? || data == "" end + + private + + def find_encoding + @find_encoding ||= Gitlab::EncodingHelper.detect_encoding(data) if data # rubocop:disable Gitlab/ModuleWithInstanceVariables + end end end diff --git a/lib/gitlab/cache.rb b/lib/gitlab/cache.rb new file mode 100644 index 00000000000..90a0c38ff7b --- /dev/null +++ b/lib/gitlab/cache.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + class << self + # Utility method for performing a fetch but only + # once per request, storing the returned value in + # the request store, if active. + def fetch_once(key, **kwargs) + Gitlab::SafeRequestStore.fetch(key) do + Rails.cache.fetch(key, **kwargs) do + yield + end + end + end + end + end +end diff --git a/lib/gitlab/changelog/committer.rb b/lib/gitlab/changelog/committer.rb index 31661650eff..52c355478c5 100644 --- a/lib/gitlab/changelog/committer.rb +++ b/lib/gitlab/changelog/committer.rb @@ -55,7 +55,7 @@ module Gitlab result = service.execute - raise Error.new(result[:message]) if result[:status] != :success + raise Error, result[:message] if result[:status] != :success end end diff --git a/lib/gitlab/changelog/parser.rb b/lib/gitlab/changelog/parser.rb index a4c8da283cd..fac6fc19148 100644 --- a/lib/gitlab/changelog/parser.rb +++ b/lib/gitlab/changelog/parser.rb @@ -169,7 +169,7 @@ module Gitlab # We raise a custom error so it's easier to catch different changelog # related errors. In addition, this ensures the caller of this method # doesn't depend on a Parslet specific error class. - raise Error.new("Failed to parse the template: #{ex.message}") + raise Error, "Failed to parse the template: #{ex.message}" end end end diff --git a/lib/gitlab/chat/responder.rb b/lib/gitlab/chat/responder.rb index 6267fbc20e2..53a625e9d43 100644 --- a/lib/gitlab/chat/responder.rb +++ b/lib/gitlab/chat/responder.rb @@ -11,9 +11,9 @@ module Gitlab # # build - A `Ci::Build` that executed a chat command. def self.responder_for(build) - service = build.pipeline.chat_data&.chat_name&.service + integration = build.pipeline.chat_data&.chat_name&.integration - if (responder = service.try(:chat_responder)) + if (responder = integration.try(:chat_responder)) responder.new(build) end end diff --git a/lib/gitlab/checks/base_checker.rb b/lib/gitlab/checks/base_checker.rb index 0045d8a4113..68873610408 100644 --- a/lib/gitlab/checks/base_checker.rb +++ b/lib/gitlab/checks/base_checker.rb @@ -57,4 +57,4 @@ module Gitlab end end -Gitlab::Checks::BaseChecker.prepend_if_ee('EE::Gitlab::Checks::BaseChecker') +Gitlab::Checks::BaseChecker.prepend_mod_with('Gitlab::Checks::BaseChecker') diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 67c777f67a7..a2c3de3e775 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -54,4 +54,4 @@ module Gitlab end end -Gitlab::Checks::ChangeAccess.prepend_if_ee('EE::Gitlab::Checks::ChangeAccess') +Gitlab::Checks::ChangeAccess.prepend_mod_with('Gitlab::Checks::ChangeAccess') diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index b146fea66b9..a05181ab58e 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -76,4 +76,4 @@ module Gitlab end end -Gitlab::Checks::DiffCheck.prepend_if_ee('EE::Gitlab::Checks::DiffCheck') +Gitlab::Checks::DiffCheck.prepend_mod_with('Gitlab::Checks::DiffCheck') diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb index db7af0088d0..2635ad04770 100644 --- a/lib/gitlab/checks/matching_merge_request.rb +++ b/lib/gitlab/checks/matching_merge_request.rb @@ -21,4 +21,4 @@ module Gitlab end end -Gitlab::Checks::MatchingMergeRequest.prepend_if_ee('EE::Gitlab::Checks::MatchingMergeRequest') +Gitlab::Checks::MatchingMergeRequest.prepend_mod_with('Gitlab::Checks::MatchingMergeRequest') diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 1fac00337a3..97988d8aa13 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -30,6 +30,8 @@ module Gitlab Converter.new.convert(ansi, state) end + Result = Struct.new(:html, :state, :append, :truncated, :offset, :size, :total, keyword_init: true) # rubocop:disable Lint/StructNewOverride + class Converter def on_0(_) reset @@ -278,9 +280,7 @@ module Gitlab close_open_tags - # TODO: replace OpenStruct with a better type - # https://gitlab.com/gitlab-org/gitlab/issues/34305 - OpenStruct.new( + Ansi2html::Result.new( html: @out.force_encoding(Encoding.default_external), state: state, append: append, diff --git a/lib/gitlab/ci/build/cache.rb b/lib/gitlab/ci/build/cache.rb index 4fcb5168847..375e6b4a96f 100644 --- a/lib/gitlab/ci/build/cache.rb +++ b/lib/gitlab/ci/build/cache.rb @@ -7,39 +7,22 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize def initialize(cache, pipeline) - if multiple_cache_per_job? - cache = Array.wrap(cache) - @cache = cache.map do |cache| - Gitlab::Ci::Pipeline::Seed::Build::Cache - .new(pipeline, cache) - end - else - @cache = Gitlab::Ci::Pipeline::Seed::Build::Cache - .new(pipeline, cache) + cache = Array.wrap(cache) + @cache = cache.map do |cache| + Gitlab::Ci::Pipeline::Seed::Build::Cache + .new(pipeline, cache) end end def cache_attributes strong_memoize(:cache_attributes) do - if multiple_cache_per_job? - if @cache.empty? - {} - else - { options: { cache: @cache.map(&:attributes) } } - end + if @cache.empty? + {} else - @cache.build_attributes + { options: { cache: @cache.map(&:attributes) } } end end end - - private - - def multiple_cache_per_job? - strong_memoize(:multiple_cache_per_job) do - ::Gitlab::Ci::Features.multiple_cache_per_job? - end - end end end end diff --git a/lib/gitlab/ci/build/releaser.rb b/lib/gitlab/ci/build/releaser.rb index facb5f619bd..9720bb1123a 100644 --- a/lib/gitlab/ci/build/releaser.rb +++ b/lib/gitlab/ci/build/releaser.rb @@ -18,8 +18,9 @@ module Gitlab command = BASE_COMMAND.dup single_flags.each { |k, v| command.concat(" --#{k.to_s.dasherize} \"#{v}\"") } array_commands.each { |k, v| v.each { |elem| command.concat(" --#{k.to_s.singularize.dasherize} \"#{elem}\"") } } + asset_links.each { |link| command.concat(" --assets-link #{stringified_json(link)}") } - [command] + [command.freeze] end private @@ -31,6 +32,14 @@ module Gitlab def array_commands config.slice(*ARRAY_FLAGS) end + + def asset_links + config.dig(:assets, :links) || [] + end + + def stringified_json(object) + "#{object.to_json.to_json}" + end end end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 23b0c93a3ee..9c6428d701c 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -17,13 +17,14 @@ module Gitlab Config::Yaml::Tags::TagError ].freeze - attr_reader :root, :context, :ref + attr_reader :root, :context, :ref, :source - def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, ref: nil) + def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, ref: nil, source: nil) @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline) @context.set_deadline(TIMEOUT_SECONDS) @ref = ref + @source = source @config = expand_config(config) @@ -128,4 +129,4 @@ module Gitlab end end -Gitlab::Ci::Config.prepend_if_ee('EE::Gitlab::Ci::ConfigEE') +Gitlab::Ci::Config.prepend_mod_with('Gitlab::Ci::ConfigEE') diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index f9688c500d2..ab79add688b 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -4,88 +4,52 @@ module Gitlab module Ci class Config module Entry - ## - # Entry that represents a cache configuration - # - class Cache < ::Gitlab::Config::Entry::Simplifiable - strategy :Caches, if: -> (config) { Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml) } - strategy :Cache, if: -> (config) { Feature.disabled?(:multiple_cache_per_job, default_enabled: :yaml) } - - class Caches < ::Gitlab::Config::Entry::ComposableArray - include ::Gitlab::Config::Entry::Validatable - - MULTIPLE_CACHE_LIMIT = 4 - - validations do - validate do - unless config.is_a?(Hash) || config.is_a?(Array) - errors.add(:config, 'can only be a Hash or an Array') - end - - if config.is_a?(Array) && config.count > MULTIPLE_CACHE_LIMIT - errors.add(:config, "no more than #{MULTIPLE_CACHE_LIMIT} caches can be created") - end - end - end - - def initialize(*args) - super - - @key = nil - end - - def composable_class - Entry::Cache::Cache + class Cache < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[key untracked paths when policy].freeze + ALLOWED_POLICY = %w[pull-push push pull].freeze + DEFAULT_POLICY = 'pull-push' + ALLOWED_WHEN = %w[on_success on_failure always].freeze + DEFAULT_WHEN = 'on_success' + + validations do + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS + validates :policy, + inclusion: { in: ALLOWED_POLICY, message: 'should be pull-push, push, or pull' }, + allow_blank: true + + with_options allow_nil: true do + validates :when, + inclusion: { + in: ALLOWED_WHEN, + message: 'should be on_success, on_failure or always' + } end end - class Cache < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Configurable - include ::Gitlab::Config::Entry::Validatable - include ::Gitlab::Config::Entry::Attributable - - ALLOWED_KEYS = %i[key untracked paths when policy].freeze - ALLOWED_POLICY = %w[pull-push push pull].freeze - DEFAULT_POLICY = 'pull-push' - ALLOWED_WHEN = %w[on_success on_failure always].freeze - DEFAULT_WHEN = 'on_success' + entry :key, Entry::Key, + description: 'Cache key used to define a cache affinity.' - validations do - validates :config, type: Hash, allowed_keys: ALLOWED_KEYS - validates :policy, - inclusion: { in: ALLOWED_POLICY, message: 'should be pull-push, push, or pull' }, - allow_blank: true - - with_options allow_nil: true do - validates :when, - inclusion: { - in: ALLOWED_WHEN, - message: 'should be on_success, on_failure or always' - } - end - end + entry :untracked, ::Gitlab::Config::Entry::Boolean, + description: 'Cache all untracked files.' - entry :key, Entry::Key, - description: 'Cache key used to define a cache affinity.' + entry :paths, Entry::Paths, + description: 'Specify which paths should be cached across builds.' - entry :untracked, ::Gitlab::Config::Entry::Boolean, - description: 'Cache all untracked files.' + attributes :policy, :when - entry :paths, Entry::Paths, - description: 'Specify which paths should be cached across builds.' + def value + result = super - attributes :policy, :when + result[:key] = key_value + result[:policy] = policy || DEFAULT_POLICY + # Use self.when to avoid conflict with reserved word + result[:when] = self.when || DEFAULT_WHEN - def value - result = super - - result[:key] = key_value - result[:policy] = policy || DEFAULT_POLICY - # Use self.when to avoid conflict with reserved word - result[:when] = self.when || DEFAULT_WHEN - - result - end + result end class UnknownStrategy < ::Gitlab::Config::Entry::Node diff --git a/lib/gitlab/ci/config/entry/caches.rb b/lib/gitlab/ci/config/entry/caches.rb new file mode 100644 index 00000000000..75240599c9c --- /dev/null +++ b/lib/gitlab/ci/config/entry/caches.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents caches configuration + # + class Caches < ::Gitlab::Config::Entry::ComposableArray + include ::Gitlab::Config::Entry::Validatable + + MULTIPLE_CACHE_LIMIT = 4 + + validations do + validate do + unless config.is_a?(Hash) || config.is_a?(Array) + errors.add(:config, 'can only be a Hash or an Array') + end + + if config.is_a?(Array) && config.count > MULTIPLE_CACHE_LIMIT + errors.add(:config, "no more than #{MULTIPLE_CACHE_LIMIT} caches can be created") + end + end + end + + def initialize(*args) + super + + @key = nil + end + + def composable_class + Entry::Cache + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/default.rb b/lib/gitlab/ci/config/entry/default.rb index ab493ff7d78..eaaf9f69102 100644 --- a/lib/gitlab/ci/config/entry/default.rb +++ b/lib/gitlab/ci/config/entry/default.rb @@ -37,7 +37,7 @@ module Gitlab description: 'Script that will be executed after each job.', inherit: true - entry :cache, Entry::Cache, + entry :cache, Entry::Caches, description: 'Configure caching between build jobs.', inherit: true diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index a20b802be58..c8e8f0bc1fc 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -64,7 +64,7 @@ module Gitlab description: 'Commands that will be executed when finishing job.', inherit: true - entry :cache, Entry::Cache, + entry :cache, Entry::Caches, description: 'Cache definition for this job.', inherit: true @@ -200,4 +200,4 @@ module Gitlab end end -::Gitlab::Ci::Config::Entry::Job.prepend_if_ee('::EE::Gitlab::Ci::Config::Entry::Job') +::Gitlab::Ci::Config::Entry::Job.prepend_mod_with('Gitlab::Ci::Config::Entry::Job') diff --git a/lib/gitlab/ci/config/entry/need.rb b/lib/gitlab/ci/config/entry/need.rb index b3cf0f9e0fd..29dc48c7b42 100644 --- a/lib/gitlab/ci/config/entry/need.rb +++ b/lib/gitlab/ci/config/entry/need.rb @@ -118,4 +118,4 @@ module Gitlab end end -::Gitlab::Ci::Config::Entry::Need.prepend_if_ee('::EE::Gitlab::Ci::Config::Entry::Need') +::Gitlab::Ci::Config::Entry::Need.prepend_mod_with('Gitlab::Ci::Config::Entry::Need') diff --git a/lib/gitlab/ci/config/entry/needs.rb b/lib/gitlab/ci/config/entry/needs.rb index dd01cfeedff..11b202ddde9 100644 --- a/lib/gitlab/ci/config/entry/needs.rb +++ b/lib/gitlab/ci/config/entry/needs.rb @@ -56,4 +56,4 @@ module Gitlab end end -::Gitlab::Ci::Config::Entry::Needs.prepend_if_ee('::EE::Gitlab::Ci::Config::Entry::Needs') +::Gitlab::Ci::Config::Entry::Needs.prepend_mod_with('Gitlab::Ci::Config::Entry::Needs') diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index 54ef84b965a..e6290ef2479 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -61,7 +61,7 @@ module Gitlab description: 'Deprecated: stages for this pipeline.', reserved: true - entry :cache, Entry::Cache, + entry :cache, Entry::Caches, description: 'Configure caching between build jobs.', reserved: true diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 12e182b38fc..c8e4d9ed763 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -18,9 +18,8 @@ module Gitlab Feature.enabled?(:ci_pipeline_status_omit_commit_sha_in_cache_key, project, default_enabled: true) end - # Remove in https://gitlab.com/gitlab-org/gitlab/-/issues/224199 - def self.store_pipeline_messages?(project) - ::Feature.enabled?(:ci_store_pipeline_messages, project, default_enabled: true) + def self.merge_base_pipeline_for_metrics_comparison?(project) + Feature.enabled?(:merge_base_pipeline_for_metrics_comparison, project, default_enabled: :yaml) end def self.raise_job_rules_without_workflow_rules_warning? @@ -47,22 +46,17 @@ module Gitlab ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) end - def self.validate_build_dependencies?(project) - ::Feature.enabled?(:ci_validate_build_dependencies, project, default_enabled: :yaml) && - ::Feature.disabled?(:ci_validate_build_dependencies_override, project) - end - def self.display_quality_on_mr_diff?(project) - ::Feature.enabled?(:codequality_mr_diff, project, default_enabled: false) - end - - def self.multiple_cache_per_job? - ::Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml) + ::Feature.enabled?(:codequality_mr_diff, project, default_enabled: :yaml) end def self.gldropdown_tags_enabled? ::Feature.enabled?(:gldropdown_tags, default_enabled: :yaml) end + + def self.background_pipeline_retry_endpoint?(project) + ::Feature.enabled?(:background_pipeline_retry_endpoint, project) + end end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index a6ae249fa58..0b94debb24e 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -123,4 +123,4 @@ module Gitlab end end -Gitlab::Ci::Jwt.prepend_if_ee('::EE::Gitlab::Ci::Jwt') +Gitlab::Ci::Jwt.prepend_mod_with('Gitlab::Ci::Jwt') diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index 2baa8faf849..3469537a2e2 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -15,8 +15,8 @@ module Gitlab } end - def self.fabricate!(file_type, *args) - parsers.fetch(file_type.to_sym).new(*args) + def self.fabricate!(file_type, *args, **kwargs) + parsers.fetch(file_type.to_sym).new(*args, **kwargs) rescue KeyError raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'" end @@ -28,4 +28,4 @@ module Gitlab end end -Gitlab::Ci::Parsers.prepend_if_ee('::EE::Gitlab::Ci::Parsers') +Gitlab::Ci::Parsers.prepend_mod_with('Gitlab::Ci::Parsers') diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb index eb3adf713d4..d6b3af674a6 100644 --- a/lib/gitlab/ci/parsers/coverage/cobertura.rb +++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb @@ -121,7 +121,7 @@ module Gitlab # Using `Integer()` here to raise exception on invalid values [Integer(line["number"]), Integer(line["hits"])] end - rescue + rescue StandardError raise InvalidLineInformationError, "Line information had invalid values" end diff --git a/lib/gitlab/ci/parsers/terraform/tfplan.rb b/lib/gitlab/ci/parsers/terraform/tfplan.rb index abfbe18e23f..f9afa58f915 100644 --- a/lib/gitlab/ci/parsers/terraform/tfplan.rb +++ b/lib/gitlab/ci/parsers/terraform/tfplan.rb @@ -19,7 +19,7 @@ module Gitlab end rescue JSON::ParserError terraform_reports.add_plan(job_id, invalid_tfplan(:invalid_json_format, job_details)) - rescue + rescue StandardError details = job_details || {} plan_name = job_id || 'failed_tf_plan' terraform_reports.add_plan(plan_name, invalid_tfplan(:unknown_error, details)) diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb index 50cd703da4a..ca7fbde6713 100644 --- a/lib/gitlab/ci/parsers/test/junit.rb +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -31,7 +31,7 @@ module Gitlab def ensure_test_cases_limited!(total_parsed, limit) return unless limit > 0 && total_parsed > limit - raise JunitParserError.new("number of test cases exceeded the limit of #{limit}") + raise JunitParserError, "number of test cases exceeded the limit of #{limit}" end def all_cases(root, parent = nil, &blk) diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index a7680f6e593..3c150ca26bb 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -52,4 +52,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Config::Content.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Config::Content') +Gitlab::Ci::Pipeline::Chain::Config::Content.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Config::Content') diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 8f1c49563f2..49ec1250a5f 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -16,6 +16,7 @@ module Gitlab project: project, ref: @pipeline.ref, sha: @pipeline.sha, + source: @pipeline.source, user: current_user, parent_pipeline: parent_pipeline } @@ -31,7 +32,7 @@ module Gitlab @pipeline.merged_yaml = result.merged_yaml - rescue => ex + rescue StandardError => ex Gitlab::ErrorTracking.track_exception(ex, project_id: project.id, sha: @pipeline.sha diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index 9988b6f18ed..09158bf8bfd 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -19,6 +19,8 @@ module Gitlab # polluted with other unrelated errors (e.g. state machine) # https://gitlab.com/gitlab-org/gitlab/-/issues/220823 pipeline.errors.add(:base, message) + + pipeline.errors.full_messages end def warning(message) diff --git a/lib/gitlab/ci/pipeline/chain/limit/activity.rb b/lib/gitlab/ci/pipeline/chain/limit/activity.rb index 3c64278e305..ef9235477db 100644 --- a/lib/gitlab/ci/pipeline/chain/limit/activity.rb +++ b/lib/gitlab/ci/pipeline/chain/limit/activity.rb @@ -20,4 +20,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Limit::Activity.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Limit::Activity') +Gitlab::Ci::Pipeline::Chain::Limit::Activity.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Limit::Activity') diff --git a/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb b/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb index 2e8b437252f..3706dd0b9f6 100644 --- a/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb +++ b/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb @@ -20,4 +20,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Limit::JobActivity.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Limit::JobActivity') +Gitlab::Ci::Pipeline::Chain::Limit::JobActivity.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Limit::JobActivity') diff --git a/lib/gitlab/ci/pipeline/chain/limit/size.rb b/lib/gitlab/ci/pipeline/chain/limit/size.rb index 739648840e9..761bdb1c484 100644 --- a/lib/gitlab/ci/pipeline/chain/limit/size.rb +++ b/lib/gitlab/ci/pipeline/chain/limit/size.rb @@ -20,4 +20,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Limit::Size.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Limit::Size') +Gitlab::Ci::Pipeline::Chain::Limit::Size.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Limit::Size') diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb index df92e229f12..e4e4f4f484a 100644 --- a/lib/gitlab/ci/pipeline/chain/skip.rb +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -11,7 +11,14 @@ module Gitlab def perform! if skipped? - @pipeline.skip if @command.save_incompleted + if @command.save_incompleted + # Project iid must be called outside a transaction, so we ensure it is set here + # otherwise it may be set within the state transition transaction of the skip call + # which it will lock the InternalId row for the whole transaction + @pipeline.ensure_project_iid! + + @pipeline.skip + end end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index 55c125e03d5..1c1f7abb6f6 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -57,4 +57,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Validate::Abilities.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Validate::Abilities') +Gitlab::Ci::Pipeline::Chain::Validate::Abilities.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Validate::Abilities') diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 6149d2f04d7..539b44513f0 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -54,7 +54,7 @@ module Gitlab else raise InvalidResponseCode, "Unsupported response code received from Validation Service: #{response_code}" end - rescue => ex + rescue StandardError => ex Gitlab::ErrorTracking.track_exception(ex, project_id: project.id) true @@ -147,4 +147,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Validate::External.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Validate::External') +Gitlab::Ci::Pipeline::Chain::Validate::External.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Validate::External') diff --git a/lib/gitlab/ci/pipeline/chain/validate/security_orchestration_policy.rb b/lib/gitlab/ci/pipeline/chain/validate/security_orchestration_policy.rb new file mode 100644 index 00000000000..e3588aa3027 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/security_orchestration_policy.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class SecurityOrchestrationPolicy < Chain::Base + include Chain::Helpers + + def perform! + # no-op + end + + def break? + false + end + end + end + end + end + end +end + +Gitlab::Ci::Pipeline::Chain::Validate::SecurityOrchestrationPolicy.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Validate::SecurityOrchestrationPolicy') diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 6cb6fd3920d..84b88374a7f 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -13,6 +13,13 @@ module Gitlab ::Gitlab::Metrics.histogram(name, comment, labels, buckets) end + def self.pipeline_security_orchestration_policy_processing_duration_histogram + name = :gitlab_ci_pipeline_security_orchestration_policy_processing_duration_seconds + comment = 'Pipeline security orchestration policy processing duration' + + ::Gitlab::Metrics.histogram(name, comment) + end + def self.pipeline_size_histogram name = :gitlab_ci_pipeline_size_builds comment = 'Pipeline size' @@ -56,6 +63,21 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end + + def ci_minutes_exceeded_builds_counter + name = :ci_minutes_exceeded_builds_counter + comment = 'Count of builds dropped due to CI minutes exceeded' + + Gitlab::Metrics.counter(name, comment) + end + + def self.gitlab_ci_difference_live_vs_actual_minutes + name = :gitlab_ci_difference_live_vs_actual_minutes + comment = 'Comparison between CI minutes consumption from live tracking vs actual consumption' + labels = {} + buckets = [-120.0, -60.0, -30.0, -10.0, -5.0, -3.0, -1.0, 0.0, 1.0, 3.0, 5.0, 10.0, 30.0, 60.0, 120.0] + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end end end end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 7ecb9a1db16..46e4373ec85 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -10,7 +10,7 @@ module Gitlab QUEUE_ACTIVE_RUNNERS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze QUEUE_DEPTH_TOTAL_BUCKETS = [1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000, 7500, 10000, 15000, 20000].freeze - QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze + QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 15, 20, 30, 60].freeze METRICS_SHARD_TAG_PREFIX = 'metrics_shard::' DEFAULT_METRICS_SHARD = 'default' diff --git a/lib/gitlab/ci/reports/codequality_mr_diff.rb b/lib/gitlab/ci/reports/codequality_mr_diff.rb index e60a075e3f5..0595b6f966a 100644 --- a/lib/gitlab/ci/reports/codequality_mr_diff.rb +++ b/lib/gitlab/ci/reports/codequality_mr_diff.rb @@ -6,8 +6,8 @@ module Gitlab class CodequalityMrDiff attr_reader :files - def initialize(raw_report) - @raw_report = raw_report + def initialize(new_errors) + @new_errors = new_errors @files = {} build_report! end @@ -15,7 +15,7 @@ module Gitlab private def build_report! - codequality_files = @raw_report.all_degradations.each_with_object({}) do |degradation, codequality_files| + codequality_files = @new_errors.each_with_object({}) do |degradation, codequality_files| unless codequality_files[degradation.dig(:location, :path)].present? codequality_files[degradation.dig(:location, :path)] = [] end diff --git a/lib/gitlab/ci/reports/test_failure_history.rb b/lib/gitlab/ci/reports/test_failure_history.rb index 37d0da38065..c110dbf98be 100644 --- a/lib/gitlab/ci/reports/test_failure_history.rb +++ b/lib/gitlab/ci/reports/test_failure_history.rb @@ -13,7 +13,7 @@ module Gitlab def load! recent_failures_count.each do |key_hash, count| - failed_junit_tests[key_hash].set_recent_failures(count, project.default_branch_or_master) + failed_junit_tests[key_hash].set_recent_failures(count, project.default_branch_or_main) end end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 787dee3b267..cbd72f54ff4 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -20,6 +20,7 @@ module Gitlab scheduler_failure: 'scheduler failure', data_integrity_failure: 'data integrity failure', forward_deployment_failure: 'forward deployment failure', + pipeline_loop_detected: 'job would create infinitely looping pipelines', invalid_bridge_trigger: 'downstream pipeline trigger definition is invalid', downstream_bridge_project_not_found: 'downstream project could not be found', insufficient_bridge_permissions: 'no permissions to trigger downstream pipeline', @@ -28,7 +29,8 @@ module Gitlab secrets_provider_not_found: 'secrets provider can not be found', reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines', project_deleted: 'pipeline project was deleted', - user_blocked: 'pipeline user was blocked' + user_blocked: 'pipeline user was blocked', + ci_quota_exceeded: 'no more CI minutes available' }.freeze private_constant :REASONS @@ -68,4 +70,4 @@ module Gitlab end end -Gitlab::Ci::Status::Build::Failed.prepend_if_ee('::EE::Gitlab::Ci::Status::Build::Failed') +Gitlab::Ci::Status::Build::Failed.prepend_mod_with('Gitlab::Ci::Status::Build::Failed') diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index 4779c8d3d53..e7ed2081f6a 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -11,6 +11,8 @@ module Gitlab attr_reader :subject, :user + delegate :cache_key, to: :subject + def initialize(subject, user) @subject = subject @user = user diff --git a/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml deleted file mode 100644 index 7182b96594d..00000000000 --- a/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml +++ /dev/null @@ -1,52 +0,0 @@ -# -# You can use artifacts to pass data to jobs in later stages. -# For more information, see https://docs.gitlab.com/ee/ci/pipelines/job_artifacts.html -# - -stages: - - build - - test - - deploy - -build-job: - stage: build - script: - - echo "This job might build an important file, and pass it to later jobs." - - echo "This is the content of the important file" > important-file.txt - artifacts: - paths: - - important-file.txt - -test-job-with-artifacts: - stage: test - script: - - echo "This job uses the artifact from the job in the earlier stage." - - cat important-file.txt - - echo "It creates another file, and adds it to the artifacts." - - echo "This is a second important file" > important-file2.txt - artifacts: - paths: - - important-file2.txt - -test-job-with-no-artifacts: - stage: test - dependencies: [] # Use to skip downloading any artifacts - script: - - echo "This job does not get the artifacts from other jobs." - - cat important-file.txt || exit 0 - -deploy-job-with-all-artifacts: - stage: deploy - script: - - echo "By default, jobs download all available artifacts." - - cat important-file.txt - - cat important-file2.txt - -deploy-job-with-1-artifact: - stage: deploy - dependencies: - - build-job # Download artifacts from only this job - script: - - echo "You can configure a job to download artifacts from only certain jobs." - - cat important-file.txt - - cat important-file2.txt || exit 0 diff --git a/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml deleted file mode 100644 index 382bac09ed7..00000000000 --- a/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml +++ /dev/null @@ -1,36 +0,0 @@ -# -# You can define common tasks and run them before or after the main scripts in jobs. -# For more information, see: -# - https://docs.gitlab.com/ee/ci/yaml/README.html#before_script -# - https://docs.gitlab.com/ee/ci/yaml/README.html#after_script -# - -stages: - - test - -default: - before_script: - - echo "This script runs before the main script in every job, unless the job overrides it." - - echo "It may set up common dependencies, for example." - after_script: - - echo "This script runs after the main script in every job, unless the job overrides it." - - echo "It may do some common final clean up tasks" - -job-standard: - stage: test - script: - - echo "This job uses both of the globally defined before and after scripts." - -job-override-before: - stage: test - before_script: - - echo "Use a different before_script in this job." - script: - - echo "This job uses its own before_script, and the global after_script." - -job-override-after: - stage: test - after_script: - - echo "Use a different after_script in this job." - script: - - echo "This job uses its own after_script, and the global before_script." diff --git a/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml deleted file mode 100644 index 5f27def74c9..00000000000 --- a/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml +++ /dev/null @@ -1,53 +0,0 @@ -# -# A manual job is a type of job that is not executed automatically and must be explicitly started by a user. -# To make a job manual, add when: manual to its configuration. -# For more information, see https://docs.gitlab.com/ee/ci/yaml/README.html#whenmanual -# - -stages: - - build - - test - - deploy - -build-job: - stage: build - script: - - echo "This job is not a manual job" - -manual-build: - stage: build - script: - - echo "This manual job passes after you trigger it." - when: manual - -manual-build-allowed-to-fail: - stage: build - script: - - echo "This manual job fails after you trigger it." - - echo "It is allowed to fail, so the pipeline does not fail. - when: manual - allow_failure: true # Default behavior - -test-job: - stage: test - script: - - echo "This is a normal test job" - - echo "It runs when the when the build stage completes." - - echo "It does not need to wait for the manual jobs in the build stage to run." - -manual-test-not-allowed-to-fail: - stage: test - script: - - echo "This manual job fails after you trigger it." - - echo "It is NOT allowed to fail, so the pipeline is marked as failed - - echo "when this job completes." - - exit 1 - when: manual - allow_failure: false # Optional behavior - -deploy-job: - stage: deploy - script: - - echo "This is a normal deploy job" - - echo "If a manual job that isn't allowed to fail ran in an earlier stage and failed, - - echo "this job does not run". diff --git a/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml deleted file mode 100644 index aced628aacb..00000000000 --- a/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml +++ /dev/null @@ -1,33 +0,0 @@ -# -# A pipeline is composed of independent jobs that run scripts, grouped into stages. -# Stages run in sequential order, but jobs within stages run in parallel. -# For more information, see: https://docs.gitlab.com/ee/ci/yaml/README.html#stages -# - -stages: - - build - - test - - deploy - -build-job: - stage: build - script: - - echo "This job runs in the build stage, which runs first." - -test-job1: - stage: test - script: - - echo "This job runs in the test stage." - - echo "It only starts when the job in the build stage completes successfully." - -test-job2: - stage: test - script: - - echo "This job also runs in the test stage." - - echo "This job can run at the same time as test-job2." - -deploy-job: - stage: deploy - script: - - echo "This job runs in the deploy stage." - - echo "It only runs when both jobs in the test stage complete successfully" diff --git a/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml deleted file mode 100644 index 2b8cf7bab44..00000000000 --- a/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml +++ /dev/null @@ -1,47 +0,0 @@ -# -# Variables can be used to for more dynamic behavior in jobs and scripts. -# For more information, see https://docs.gitlab.com/ee/ci/variables/README.html -# - -stages: - - test - -variables: - VAR1: "Variable 1 defined globally" - -use-a-variable: - stage: test - script: - - echo "You can use variables in jobs." - - echo "The content of 'VAR1' is = $VAR1" - -override-a-variable: - stage: test - variables: - VAR1: "Variable 1 was overriden in in the job." - script: - - echo "You can override global variables in jobs." - - echo "The content of 'VAR1' is = $VAR1" - -define-a-new-variable: - stage: test - variables: - VAR2: "Variable 2 is new and defined in the job only." - script: - - echo "You can mix global variables with variables defined in jobs." - - echo "The content of 'VAR1' is = $VAR1" - - echo "The content of 'VAR2' is = $VAR2" - -incorrect-variable-usage: - stage: test - script: - - echo "You can't use variables only defined in other jobs." - - echo "The content of 'VAR2' is = $VAR2" - -predefined-variables: - stage: test - script: - - echo "Some variables are predefined by GitLab CI/CD, for example:" - - echo "The commit author's username is $GITLAB_USER_LOGIN" - - echo "The commit branch is $CI_COMMIT_BRANCH" - - echo "The project path is $CI_PROJECT_PATH" diff --git a/lib/gitlab/ci/templates/Getting-started.yml b/lib/gitlab/ci/templates/Getting-started.yml new file mode 100644 index 00000000000..4dc88418671 --- /dev/null +++ b/lib/gitlab/ci/templates/Getting-started.yml @@ -0,0 +1,39 @@ +# This is a sample GitLab CI/CD configuration file that should run without any modifications. +# It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts, +# it uses echo commands to simulate the pipeline execution. +# +# A pipeline is composed of independent jobs that run scripts, grouped into stages. +# Stages run in sequential order, but jobs within stages run in parallel. +# +# For more information, see: https://docs.gitlab.com/ee/ci/yaml/README.html#stages + +stages: # List of stages for jobs, and their order of execution + - build + - test + - deploy + +build-job: # This job runs in the build stage, which runs first. + stage: build + script: + - echo "Compiling the code..." + - echo "Compile complete. + +unit-test-job: # This job runs in the test stage. + stage: test # It only starts when the job in the build stage completes successfully. + script: + - echo "Running unit tests... This will take about 60 seconds." + - sleep 60 + - echo "Code coverage is 90%" + +lint-test-job: # This job also runs in the test stage. + stage: test # It can run at the same time as unit-test-job (in parallel). + script: + - echo "Linting code... This will take about 10 seconds." + - sleep 10 + - echo "No lint issues found." + +deploy-job: # This job runs in the deploy stage. + stage: deploy # It only runs when *both* jobs in the test stage complete successfully. + script: + - echo "Deploying application..." + - echo "Application successfully deployed." diff --git a/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci.yml index c7fb1321055..7f33d048c1e 100644 --- a/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml +++ b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci.yml @@ -29,12 +29,8 @@ default: before_script: - cd ${CI_PROJECT_DIR}/my_folder_with_terraform_content -stages: - - init_and_plan - - cloudrail - init_and_plan: - stage: init_and_plan + stage: build image: registry.gitlab.com/gitlab-org/terraform-images/releases/0.13 rules: - if: $SAST_DISABLED @@ -52,7 +48,7 @@ init_and_plan: - ./**/.terraform cloudrail_scan: - stage: cloudrail + stage: test image: indeni/cloudrail-cli:1.2.44 rules: - if: $SAST_DISABLED diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml new file mode 100644 index 00000000000..5216a46745c --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml @@ -0,0 +1,77 @@ +# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html + +browser_performance: + stage: performance + image: docker:19.03.12 + allow_failure: true + variables: + DOCKER_TLS_CERTDIR: "" + SITESPEED_IMAGE: sitespeedio/sitespeed.io + SITESPEED_VERSION: 14.1.0 + SITESPEED_OPTIONS: '' + services: + - docker:19.03.12-dind + script: + - | + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) + - mkdir gitlab-exporter + # Busybox wget does not support proxied HTTPS, get the real thing. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. + - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js + - mkdir sitespeed-results + - | + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - | + if [ -f .gitlab-urls.txt ] + then + sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS + else + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS + fi + - mv sitespeed-results/data/performance.json browser-performance.json + artifacts: + paths: + - sitespeed-results/ + reports: + browser_performance: browser-performance.json + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$PERFORMANCE_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 1c25d9d583b..abcb347b146 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,10 +1,11 @@ build: stage: build - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.4.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.6.0" variables: DOCKER_TLS_CERTDIR: "" services: - - docker:19.03.12-dind + - name: "docker:20.10.6-dind" + command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | if [[ -z "$CI_COMMIT_TAG" ]]; then @@ -16,6 +17,8 @@ build: fi - /build/build.sh rules: + - if: '$BUILD_DISABLED' + when: never - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' @@ -26,4 +29,6 @@ build_artifact: - printf "To build your project, please create a build_artifact job into your .gitlab-ci.yml file.\nMore information at https://docs.gitlab.com/ee/ci/cloud_deployment\n" - exit 1 rules: + - if: '$BUILD_DISABLED' + when: never - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"' diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml index 3faf07546de..45bddb1bc6a 100644 --- a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -46,27 +46,23 @@ review: name: review/$CI_COMMIT_REF_NAME url: http://$CI_PROJECT_NAME-$CI_ENVIRONMENT_SLUG.$OPENSHIFT_DOMAIN on_stop: stop-review - only: - - branches - except: - - master + rules: + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH stop-review: <<: *deploy stage: cleanup script: - oc delete all -l "app=$APP" - when: manual variables: APP: review-$CI_COMMIT_REF_NAME GIT_STRATEGY: none environment: name: review/$CI_COMMIT_REF_NAME action: stop - only: - - branches - except: - - master + rules: + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH + when: manual staging: <<: *deploy @@ -77,8 +73,8 @@ staging: environment: name: staging url: http://$CI_PROJECT_NAME-staging.$OPENSHIFT_DOMAIN - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH production: <<: *deploy @@ -86,9 +82,9 @@ production: variables: APP: production APP_HOST: $CI_PROJECT_NAME.$OPENSHIFT_DOMAIN - when: manual environment: name: production url: http://$CI_PROJECT_NAME.$OPENSHIFT_DOMAIN - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: manual diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index bf42cd52605..90fad1550ff 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -11,6 +11,7 @@ stages: - fuzz variables: + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" FUZZAPI_PROFILE: Quick FUZZAPI_VERSION: "1.6" FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml @@ -24,7 +25,7 @@ variables: # available (non 500 response to HTTP(s)) FUZZAPI_SERVICE_START_TIMEOUT: "300" # - FUZZAPI_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine + FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION} # apifuzzer_fuzz_unlicensed: diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 215029dc952..8fa33026011 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -5,266 +5,30 @@ # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: - FUZZAPI_PROFILE: Quick - FUZZAPI_VERSION: latest - FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml - FUZZAPI_TIMEOUT: 30 - FUZZAPI_REPORT: gl-api-fuzzing-report.json - FUZZAPI_REPORT_ASSET_PATH: assets - # - FUZZAPI_D_NETWORK: testing-net - # - # Wait up to 5 minutes for API Fuzzer and target url to become - # available (non 500 response to HTTP(s)) - FUZZAPI_SERVICE_START_TIMEOUT: "300" - # - FUZZAPI_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine - # - -apifuzzer_fuzz_unlicensed: - stage: fuzz - allow_failure: true - rules: - - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null' - - when: never - script: - - | - echo "Error: Your GitLab project is not licensed for API Fuzzing." - - exit 1 + FUZZAPI_VERSION: "1" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION} apifuzzer_fuzz: stage: fuzz - image: - name: $FUZZAPI_IMAGE - entrypoint: ["/bin/bash", "-l", "-c"] - variables: - FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://localhost:80 - FUZZAPI_NEW_REPORT: 1 - FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log - TZ: America/Los_Angeles - allow_failure: true - rules: - - if: $FUZZAPI_D_TARGET_IMAGE - when: never - - if: $FUZZAPI_D_WORKER_IMAGE - when: never - - if: $API_FUZZING_DISABLED - when: never - - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ - script: - # - # Validate options - - | - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ - echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ - echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ - exit 1; \ - fi - # - # Run user provided pre-script - - sh -c "$FUZZAPI_PRE_SCRIPT" - # - # Make sure asset path exists - - mkdir -p $FUZZAPI_REPORT_ASSET_PATH - # - # Start API Security background process - - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER & - - APISEC_PID=$! - # - # Start scanning - - worker-entry - # - # Run user provided post-script - - sh -c "$FUZZAPI_POST_SCRIPT" - # - # Shutdown API Security - - kill $APISEC_PID - - wait $APISEC_PID - # - artifacts: - when: always - paths: - - $FUZZAPI_REPORT_ASSET_PATH - - $FUZZAPI_REPORT - - $FUZZAPI_LOG_SCANNER - reports: - api_fuzzing: $FUZZAPI_REPORT - -apifuzzer_fuzz_dnd: - stage: fuzz - image: docker:19.03.12 - variables: - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://apifuzzer:80 + image: $FUZZAPI_IMAGE allow_failure: true rules: - - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null - when: never - if: $API_FUZZING_DISABLED when: never - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ - services: - - docker:19.03.12-dind + - if: $CI_COMMIT_BRANCH script: - # - # - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - # - - docker network create --driver bridge $FUZZAPI_D_NETWORK - # - # Run user provided pre-script - - sh -c "$FUZZAPI_PRE_SCRIPT" - # - # Make sure asset path exists - - mkdir -p $FUZZAPI_REPORT_ASSET_PATH - # - # Start peach testing engine container - - | - docker run -d \ - --name apifuzzer \ - --network $FUZZAPI_D_NETWORK \ - -e Proxy:Port=8000 \ - -e TZ=America/Los_Angeles \ - -e GITLAB_FEATURES \ - -p 80:80 \ - -p 8000:8000 \ - -p 514:514 \ - --restart=no \ - $FUZZAPI_IMAGE \ - dotnet /peach/Peach.Web.dll - # - # Start target container - - | - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \ - docker run -d \ - --name target \ - --network $FUZZAPI_D_NETWORK \ - $FUZZAPI_D_TARGET_ENV \ - $FUZZAPI_D_TARGET_PORTS \ - $FUZZAPI_D_TARGET_VOLUME \ - --restart=no \ - $FUZZAPI_D_TARGET_IMAGE \ - ; fi - # - # Start worker container if provided - - | - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \ - echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \ - docker run \ - --name worker \ - --network $FUZZAPI_D_NETWORK \ - -e FUZZAPI_API=http://apifuzzer:80 \ - -e FUZZAPI_PROJECT \ - -e FUZZAPI_PROFILE \ - -e FUZZAPI_CONFIG \ - -e FUZZAPI_REPORT \ - -e FUZZAPI_REPORT_ASSET_PATH \ - -e FUZZAPI_NEW_REPORT=1 \ - -e FUZZAPI_HAR \ - -e FUZZAPI_OPENAPI \ - -e FUZZAPI_POSTMAN_COLLECTION \ - -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ - -e FUZZAPI_TARGET_URL \ - -e FUZZAPI_OVERRIDES_FILE \ - -e FUZZAPI_OVERRIDES_ENV \ - -e FUZZAPI_OVERRIDES_CMD \ - -e FUZZAPI_OVERRIDES_INTERVAL \ - -e FUZZAPI_TIMEOUT \ - -e FUZZAPI_VERBOSE \ - -e FUZZAPI_SERVICE_START_TIMEOUT \ - -e FUZZAPI_HTTP_USERNAME \ - -e FUZZAPI_HTTP_PASSWORD \ - -e CI_PROJECT_URL \ - -e CI_JOB_ID \ - -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \ - $FUZZAPI_D_WORKER_ENV \ - $FUZZAPI_D_WORKER_PORTS \ - $FUZZAPI_D_WORKER_VOLUME \ - --restart=no \ - $FUZZAPI_D_WORKER_IMAGE \ - ; fi - # - # Start API Fuzzing provided worker if no other worker present - - | - if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \ - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ - echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ - echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ - exit 1; \ - fi; \ - docker run \ - --name worker \ - --network $FUZZAPI_D_NETWORK \ - -e TZ=America/Los_Angeles \ - -e FUZZAPI_API=http://apifuzzer:80 \ - -e FUZZAPI_PROJECT \ - -e FUZZAPI_PROFILE \ - -e FUZZAPI_CONFIG \ - -e FUZZAPI_REPORT \ - -e FUZZAPI_REPORT_ASSET_PATH \ - -e FUZZAPI_NEW_REPORT=1 \ - -e FUZZAPI_HAR \ - -e FUZZAPI_OPENAPI \ - -e FUZZAPI_POSTMAN_COLLECTION \ - -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ - -e FUZZAPI_TARGET_URL \ - -e FUZZAPI_OVERRIDES_FILE \ - -e FUZZAPI_OVERRIDES_ENV \ - -e FUZZAPI_OVERRIDES_CMD \ - -e FUZZAPI_OVERRIDES_INTERVAL \ - -e FUZZAPI_TIMEOUT \ - -e FUZZAPI_VERBOSE \ - -e FUZZAPI_SERVICE_START_TIMEOUT \ - -e FUZZAPI_HTTP_USERNAME \ - -e FUZZAPI_HTTP_PASSWORD \ - -e CI_PROJECT_URL \ - -e CI_JOB_ID \ - -v $CI_PROJECT_DIR:/app \ - -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \ - -p 81:80 \ - -p 8001:8000 \ - -p 515:514 \ - --restart=no \ - $FUZZAPI_IMAGE \ - worker-entry \ - ; fi - # - # Propagate exit code from api fuzzing scanner (if any) - - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi - # - # Run user provided post-script - - sh -c "$FUZZAPI_POST_SCRIPT" - # - after_script: - # - # Shutdown all containers - - echo "Stopping all containers" - - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi - - docker stop worker - - docker stop apifuzzer - # - # Save docker logs - - docker logs apifuzzer &> gl-api_fuzzing-logs.log - - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi - - docker logs worker &> gl-api_fuzzing-worker-logs.log - # + - /peach/analyzer-fuzz-api artifacts: when: always paths: - - ./gl-api_fuzzing*.log - - ./gl-api_fuzzing*.zip - - $FUZZAPI_REPORT_ASSET_PATH - - $FUZZAPI_REPORT + - gl-assets + - gl-api-fuzzing-report.json + - gl-*.log reports: - api_fuzzing: $FUZZAPI_REPORT + api_fuzzing: gl-api-fuzzing-report.json # end diff --git a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml new file mode 100644 index 00000000000..b40c4e982f7 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml @@ -0,0 +1,48 @@ +# To use this template, add the following to your .gitlab-ci.yml file: +# +# include: +# template: DAST-API.gitlab-ci.yml +# +# You also need to add a `dast` stage to your `stages:` configuration. A sample configuration for DAST API: +# +# stages: +# - build +# - test +# - deploy +# - dast + +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html + +# Configure the scanning tool with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html) +# List of variables available to configure the DAST API scanning tool: +# https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables + +variables: + # Setting this variable affects all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + # + DAST_API_VERSION: "1" + DAST_API_IMAGE: $SECURE_ANALYZERS_PREFIX/api-fuzzing:$DAST_API_VERSION + +dast_api: + stage: dast + image: $DAST_API_IMAGE + allow_failure: true + rules: + - if: $DAST_API_DISABLED + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH + script: + - /peach/analyzer-dast-api + artifacts: + when: always + paths: + - gl-assets + - gl-dast-api-report.json + - gl-*.log + reports: + dast: gl-dast-api-report.json diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index 533f8bb25f8..b6282da18a4 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -22,19 +22,6 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - # - DAST_API_PROFILE: Full - DAST_API_VERSION: latest - DAST_API_CONFIG: .gitlab-dast-api.yml - DAST_API_TIMEOUT: 30 - DAST_API_REPORT: gl-dast-api-report.json - DAST_API_REPORT_ASSET_PATH: assets - # - # Wait up to 5 minutes for API Security and target url to become - # available (non 500 response to HTTP(s)) - DAST_API_SERVICE_START_TIMEOUT: "300" - # - DAST_API_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${DAST_API_VERSION}-engine dast: stage: dast @@ -51,11 +38,6 @@ dast: reports: dast: gl-dast-report.json rules: - - if: $DAST_API_BETA && ( $DAST_API_SPECIFICATION || - $DAST_API_OPENAPI || - $DAST_API_POSTMAN_COLLECTION || - $DAST_API_HAR ) - when: never - if: $DAST_DISABLED when: never - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && @@ -71,72 +53,4 @@ dast: - if: $CI_COMMIT_BRANCH && $DAST_WEBSITE - if: $CI_COMMIT_BRANCH && - $DAST_API_BETA == null && $DAST_API_SPECIFICATION - -dast_api: - stage: dast - image: - name: $DAST_API_IMAGE - entrypoint: ["/bin/bash", "-l", "-c"] - variables: - API_SECURITY_MODE: DAST - DAST_API_NEW_REPORT: 1 - DAST_API_PROJECT: $CI_PROJECT_PATH - DAST_API_API: http://127.0.0.1:5000 - DAST_API_LOG_SCANNER: gl-dast-api-scanner.log - TZ: America/Los_Angeles - allow_failure: true - rules: - - if: $DAST_API_BETA == null - when: never - - if: $DAST_DISABLED - when: never - - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && - $REVIEW_DISABLED && - $DAST_API_SPECIFICATION == null && - $DAST_API_OPENAPI == null && - $DAST_API_POSTMAN_COLLECTION == null && - $DAST_API_HAR == null - when: never - - if: $DAST_API_SPECIFICATION == null && - $DAST_API_OPENAPI == null && - $DAST_API_POSTMAN_COLLECTION == null && - $DAST_API_HAR == null - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdast\b/ - script: - # - # Run user provided pre-script - - sh -c "$DAST_API_PRE_SCRIPT" - # - # Make sure asset path exists - - mkdir -p $DAST_API_REPORT_ASSET_PATH - # - # Start API Security background process - - dotnet /peach/Peach.Web.dll &> $DAST_API_LOG_SCANNER & - - APISEC_PID=$! - # - # Start scanning - - worker-entry - # - # Run user provided post-script - - sh -c "$DAST_API_POST_SCRIPT" - # - # Shutdown API Security - - kill $APISEC_PID - - wait $APISEC_PID - # - artifacts: - when: always - paths: - - $DAST_API_REPORT_ASSET_PATH - - $DAST_API_REPORT - - $DAST_API_LOG_SCANNER - - gl-*.log - reports: - dast: $DAST_API_REPORT diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 3039d64514b..53d68c24d26 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -8,8 +8,8 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python" + DS_EXCLUDED_ANALYZERS: "" DS_EXCLUDED_PATHS: "spec, test, tests, tmp" DS_MAJOR_VERSION: 2 @@ -45,6 +45,8 @@ gemnasium-dependency_scanning: rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/ + when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $DS_DEFAULT_ANALYZERS =~ /gemnasium([^-]|$)/ @@ -71,6 +73,8 @@ gemnasium-maven-dependency_scanning: rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-maven/ + when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $DS_DEFAULT_ANALYZERS =~ /gemnasium-maven/ @@ -92,6 +96,8 @@ gemnasium-python-dependency_scanning: rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/ + when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ @@ -120,6 +126,8 @@ bundler-audit-dependency_scanning: rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /bundler-audit/ + when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $DS_DEFAULT_ANALYZERS =~ /bundler-audit/ @@ -138,6 +146,8 @@ retire-js-dependency_scanning: rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /retire.js/ + when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $DS_DEFAULT_ANALYZERS =~ /retire.js/ diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 3ebccfbba4a..a8d45e80356 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -155,13 +155,8 @@ gosec-sast: exists: - '**/*.go' -mobsf-android-sast: +.mobsf-sast: extends: .sast-analyzer - services: - # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile - # Unfortunately, we need to keep track of mobsf version in 2 different places for now. - - name: opensecurity/mobile-security-framework-mobsf:v3.4.0 - alias: mobsf image: name: "$SAST_ANALYZER_IMAGE" variables: @@ -169,7 +164,9 @@ mobsf-android-sast: # override the analyzer image with a custom value. This may be subject to change or # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" - MOBSF_API_KEY: key + +mobsf-android-sast: + extends: .mobsf-sast rules: - if: $SAST_DISABLED when: never @@ -179,23 +176,11 @@ mobsf-android-sast: $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && $SAST_EXPERIMENTAL_FEATURES == 'true' exists: + - '**/*.apk' - '**/AndroidManifest.xml' mobsf-ios-sast: - extends: .sast-analyzer - services: - # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile - # Unfortunately, we need to keep track of mobsf version in 2 different places for now. - - name: opensecurity/mobile-security-framework-mobsf:v3.4.0 - alias: mobsf - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" - MOBSF_API_KEY: key + extends: .mobsf-sast rules: - if: $SAST_DISABLED when: never @@ -205,6 +190,7 @@ mobsf-ios-sast: $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && $SAST_EXPERIMENTAL_FEATURES == 'true' exists: + - '**/*.ipa' - '**/*.xcodeproj/*' nodejs-scan-sast: @@ -292,15 +278,14 @@ semgrep-sast: # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to # override the analyzer image with a custom value. This may be subject to change or # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:latest" + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ when: never - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /semgrep/ && - $SAST_EXPERIMENTAL_FEATURES == 'true' + $SAST_DEFAULT_ANALYZERS =~ /semgrep/ exists: - '**/*.py' - '**/*.js' diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index 232c320562b..ac975fbbeab 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -13,11 +13,11 @@ variables: SECURE_BINARIES_ANALYZERS: >- - bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, + bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, semgrep, bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python, klar, clair-vulnerabilities-db, license-finder, - dast + dast, api-fuzzing SECURE_BINARIES_DOWNLOAD_IMAGES: "true" SECURE_BINARIES_PUSH_IMAGES: "true" @@ -134,6 +134,13 @@ secrets: variables: SECURE_BINARIES_ANALYZER_VERSION: "3" +semgrep: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bsemgrep\b/ + sobelow: extends: .download_images only: @@ -241,3 +248,12 @@ dast: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && $SECURE_BINARIES_ANALYZERS =~ /\bdast\b/ + +api-fuzzing: + extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "1" + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bapi-fuzzing\b/ diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 7e2828d010f..6b9db1c2e0f 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -56,6 +56,6 @@ apply: - terraform apply -input=false $PLAN dependencies: - plan - when: manual - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: manual diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml new file mode 100644 index 00000000000..f0621165f8a --- /dev/null +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml @@ -0,0 +1,52 @@ +# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html + +stages: + - build + - test + - deploy + - performance + +browser_performance: + stage: performance + image: docker:git + variables: + URL: '' + SITESPEED_IMAGE: sitespeedio/sitespeed.io + SITESPEED_VERSION: 14.1.0 + SITESPEED_OPTIONS: '' + services: + - docker:stable-dind + script: + - mkdir gitlab-exporter + # Busybox wget does not support proxied HTTPS, get the real thing. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. + - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js + - mkdir sitespeed-results + - | + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - | + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS + - mv sitespeed-results/data/performance.json browser-performance.json + artifacts: + paths: + - sitespeed-results/ + reports: + browser_performance: browser-performance.json diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index c25c4339c35..c4757edf74e 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -317,4 +317,4 @@ module Gitlab end end -::Gitlab::Ci::Trace.prepend_if_ee('EE::Gitlab::Ci::Trace') +::Gitlab::Ci::Trace.prepend_mod_with('Gitlab::Ci::Trace') diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 618438c8887..fdc598c025a 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -93,7 +93,7 @@ module Gitlab end nil - rescue + rescue StandardError # if bad regex or something goes wrong we dont want to interrupt transition # so we just silently ignore error for now end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index dc4951f76bb..a8c1002f2b9 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -141,7 +141,7 @@ module Gitlab end def error!(message) - raise ValidationError.new(message) + raise ValidationError, message end end end diff --git a/lib/gitlab/class_attributes.rb b/lib/gitlab/class_attributes.rb index 6560c97b2e6..6eea7590cbd 100644 --- a/lib/gitlab/class_attributes.rb +++ b/lib/gitlab/class_attributes.rb @@ -14,6 +14,18 @@ module Gitlab class_attributes[name] || superclass_attributes(name) end + def set_class_attribute(name, value) + class_attributes[name] = value + + after_hooks.each(&:call) + + value + end + + def after_set_class_attribute(&block) + after_hooks << block + end + private def class_attributes @@ -25,6 +37,10 @@ module Gitlab superclass.get_class_attribute(name) end + + def after_hooks + @after_hooks ||= [] + end end end end diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files.rb b/lib/gitlab/cleanup/orphan_job_artifact_files.rb index 48a1ab23fc2..05dfdcd4486 100644 --- a/lib/gitlab/cleanup/orphan_job_artifact_files.rb +++ b/lib/gitlab/cleanup/orphan_job_artifact_files.rb @@ -139,4 +139,4 @@ module Gitlab end end -Gitlab::Cleanup::OrphanJobArtifactFiles.prepend_if_ee('EE::Gitlab::Cleanup::OrphanJobArtifactFiles') +Gitlab::Cleanup::OrphanJobArtifactFiles.prepend_mod_with('Gitlab::Cleanup::OrphanJobArtifactFiles') diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb b/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb index 4b1d16eb974..e222f2834ee 100644 --- a/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb +++ b/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb @@ -79,4 +79,4 @@ module Gitlab end end -Gitlab::Cleanup::OrphanJobArtifactFilesBatch.prepend_if_ee('EE::Gitlab::Cleanup::OrphanJobArtifactFilesBatch') +Gitlab::Cleanup::OrphanJobArtifactFilesBatch.prepend_mod_with('Gitlab::Cleanup::OrphanJobArtifactFilesBatch') diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb index 77231665e7e..ed4b363416c 100644 --- a/lib/gitlab/cleanup/project_uploads.rb +++ b/lib/gitlab/cleanup/project_uploads.rb @@ -44,7 +44,7 @@ module Gitlab return unless upload && upload.local? && upload.model upload.absolute_path - rescue => e + rescue StandardError => e logger.error e.message # absolute_path depends on a lot of code. If it doesn't work, then it @@ -72,7 +72,7 @@ module Gitlab FileUtils.mv(path, new_path) "Did #{action}" - rescue => e + rescue StandardError => e "Error during #{action}: #{e.inspect}" end end diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index 3c71ca9fcf0..b3dc59466ec 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -154,7 +154,7 @@ module Gitlab hooks.each do |hook| hook.call - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, type: 'LifecycleEvents', hook: hook) warn("ERROR: The hook #{name} failed with exception (#{e.class}) \"#{e.message}\".") diff --git a/lib/gitlab/conan_token.rb b/lib/gitlab/conan_token.rb index c3d90aa78fb..d0560807f45 100644 --- a/lib/gitlab/conan_token.rb +++ b/lib/gitlab/conan_token.rb @@ -8,6 +8,7 @@ module Gitlab class ConanToken HMAC_KEY = 'gitlab-conan-packages' + CONAN_TOKEN_EXPIRE_TIME = 1.day.freeze attr_reader :access_token_id, :user_id @@ -57,7 +58,7 @@ module Gitlab JSONWebToken::HMACToken.new(self.class.secret).tap do |token| token['access_token'] = access_token_id token['user_id'] = user_id - token.expire_time = token.issued_at + 1.hour + token.expire_time = token.issued_at + CONAN_TOKEN_EXPIRE_TIME end end end diff --git a/lib/gitlab/consul/internal.rb b/lib/gitlab/consul/internal.rb index 3afc24ddab9..1994369dee9 100644 --- a/lib/gitlab/consul/internal.rb +++ b/lib/gitlab/consul/internal.rb @@ -57,7 +57,7 @@ module Gitlab def parse_response_body(body) Gitlab::Json.parse(body) - rescue + rescue StandardError raise Consul::Internal::UnexpectedResponseError end @@ -69,7 +69,7 @@ module Gitlab raise Consul::Internal::SSLError rescue Errno::ECONNREFUSED raise Consul::Internal::ECONNREFUSED - rescue + rescue StandardError raise Consul::Internal::UnexpectedResponseError end end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index ff844645b11..6f6147f0f32 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -8,11 +8,33 @@ module Gitlab media_src object_src report_uri script_src style_src worker_src).freeze def self.default_settings_hash - { - 'enabled' => false, + settings_hash = { + 'enabled' => true, 'report_only' => false, - 'directives' => DIRECTIVES.each_with_object({}) { |directive, hash| hash[directive] = nil } + 'directives' => { + 'default_src' => "'self'", + 'base_uri' => "'self'", + 'child_src' => "'none'", + 'connect_src' => "'self'", + 'font_src' => "'self'", + 'form_action' => "'self' https: http:", + 'frame_ancestors' => "'self'", + 'frame_src' => "'self' https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com", + 'img_src' => "'self' data: blob: http: https:", + 'manifest_src' => "'self'", + 'media_src' => "'self'", + 'script_src' => "'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.recaptcha.net https://apis.google.com", + 'style_src' => "'self' 'unsafe-inline'", + 'worker_src' => "'self'", + 'object_src' => "'none'", + 'report_uri' => nil + } } + + allow_webpack_dev_server(settings_hash) if Rails.env.development? + allow_cdn(settings_hash) if ENV['GITLAB_CDN_HOST'].present? + + settings_hash end def initialize(csp_directives) @@ -38,6 +60,26 @@ module Gitlab arguments.strip.split(' ').map(&:strip) end + + def self.allow_webpack_dev_server(settings_hash) + secure = Settings.webpack.dev_server['https'] + host_and_port = "#{Settings.webpack.dev_server['host']}:#{Settings.webpack.dev_server['port']}" + http_url = "#{secure ? 'https' : 'http'}://#{host_and_port}" + ws_url = "#{secure ? 'wss' : 'ws'}://#{host_and_port}" + + append_to_directive(settings_hash, 'connect_src', "#{http_url} #{ws_url}") + end + + def self.allow_cdn(settings_hash) + cdn_host = ENV['GITLAB_CDN_HOST'] + + append_to_directive(settings_hash, 'script_src', cdn_host) + append_to_directive(settings_hash, 'style_src', cdn_host) + end + + def self.append_to_directive(settings_hash, directive, text) + settings_hash['directives'][directive] = "#{settings_hash['directives'][directive]} #{text}".strip + end end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 55f381fcb64..7f55734f796 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -43,7 +43,7 @@ module Gitlab begin ::ApplicationSetting.cached - rescue + rescue StandardError # In case Redis isn't running # or the Redis UNIX socket file is not available # or the DB is not running (we use migrations in the cache key) diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb index a825d48fb77..67ad75652b0 100644 --- a/lib/gitlab/cycle_analytics/summary/base.rb +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -11,11 +11,11 @@ module Gitlab end def title - raise NotImplementedError.new("Expected #{self.name} to implement title") + raise NotImplementedError, "Expected #{self.name} to implement title" end def value - raise NotImplementedError.new("Expected #{self.name} to implement value") + raise NotImplementedError, "Expected #{self.name} to implement value" end end end diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb index c247ef0d2a7..e5bf6ef616f 100644 --- a/lib/gitlab/cycle_analytics/summary/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -16,7 +16,7 @@ module Gitlab def deployments_count DeploymentsFinder - .new(project: @project, finished_after: @from, finished_before: @to, status: :success) + .new(project: @project, finished_after: @from, finished_before: @to, status: :success, order_by: :finished_at) .execute .count end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index 0e4fc8efa95..4c31f986be5 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -30,6 +30,7 @@ module Gitlab build_started_at: build.started_at, build_finished_at: build.finished_at, build_duration: build.duration, + build_queued_duration: build.queued_duration, build_allow_failure: build.allow_failure, build_failure_reason: build.failure_reason, pipeline_id: commit.id, diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb index 87ebe832862..f50ca5119b7 100644 --- a/lib/gitlab/data_builder/deployment.rb +++ b/lib/gitlab/data_builder/deployment.rb @@ -5,7 +5,7 @@ module Gitlab module Deployment extend self - def build(deployment) + def build(deployment, status_changed_at) # Deployments will not have a deployable when created using the API. deployable_url = if deployment.deployable @@ -15,6 +15,7 @@ module Gitlab { object_kind: 'deployment', status: deployment.status, + status_changed_at: status_changed_at, deployable_id: deployment.deployable_id, deployable_url: deployable_url, environment: deployment.environment.name, diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index a56029c0d1d..766eaf54afe 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -31,6 +31,7 @@ module Gitlab created_at: pipeline.created_at, finished_at: pipeline.finished_at, duration: pipeline.duration, + queued_duration: pipeline.queued_duration, variables: pipeline.variables.map(&:hook_attrs) } end @@ -59,6 +60,8 @@ module Gitlab created_at: build.created_at, started_at: build.started_at, finished_at: build.finished_at, + duration: build.duration, + queued_duration: build.queued_duration, when: build.when, manual: build.action?, allow_failure: build.allow_failure, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 3dc8976d8c5..59249c8bc1f 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -2,6 +2,16 @@ module Gitlab module Database + # This constant is used when renaming tables concurrently. + # If you plan to rename a table using the `rename_table_safely` method, add your table here one milestone before the rename. + # Example: + # TABLES_TO_BE_RENAMED = { + # 'old_name' => 'new_name' + # }.freeze + TABLES_TO_BE_RENAMED = { + 'analytics_instance_statistics_measurements' => 'analytics_usage_trends_measurements' + }.freeze + # Minimum PostgreSQL version requirement per documentation: # https://docs.gitlab.com/ee/install/requirements.html#postgresql-requirements MINIMUM_POSTGRES_VERSION = 11 @@ -35,8 +45,27 @@ module Gitlab # It does not include the default public schema EXTRA_SCHEMAS = [DYNAMIC_PARTITIONS_SCHEMA, STATIC_PARTITIONS_SCHEMA].freeze + DEFAULT_POOL_HEADROOM = 10 + + # We configure the database connection pool size automatically based on the + # configured concurrency. We also add some headroom, to make sure we don't run + # out of connections when more threads besides the 'user-facing' ones are + # running. + # + # Read more about this in doc/development/database/client_side_connection_pool.md + def self.default_pool_size + headroom = (ENV["DB_POOL_HEADROOM"].presence || DEFAULT_POOL_HEADROOM).to_i + + Gitlab::Runtime.max_threads + headroom + end + def self.config - ActiveRecord::Base.configurations[Rails.env] + default_config_hash = ActiveRecord::Base.configurations.find_db_config(Rails.env)&.config || {} + + default_config_hash.with_indifferent_access.tap do |hash| + # Match config/initializers/database_config.rb + hash[:pool] ||= default_pool_size + end end def self.username @@ -123,6 +152,16 @@ module Gitlab # ignore - happens when Rake tasks yet have to create a database, e.g. for testing end + def self.nulls_order(field, direction = :asc, nulls_order = :nulls_last) + raise ArgumentError unless [:nulls_last, :nulls_first].include?(nulls_order) + raise ArgumentError unless [:asc, :desc].include?(direction) + + case nulls_order + when :nulls_last then nulls_last_order(field, direction) + when :nulls_first then nulls_first_order(field, direction) + end + end + def self.nulls_last_order(field, direction = 'ASC') Arel.sql("#{field} #{direction} NULLS LAST") end @@ -204,23 +243,13 @@ module Gitlab # pool_size - The size of the DB pool. # host - An optional host name to use instead of the default one. def self.create_connection_pool(pool_size, host = nil, port = nil) - env = Rails.env - original_config = ActiveRecord::Base.configurations.to_h - - env_config = original_config[env].merge('pool' => pool_size) - env_config['host'] = host if host - env_config['port'] = port if port - - config = ActiveRecord::DatabaseConfigurations.new( - original_config.merge(env => env_config) - ) + original_config = Gitlab::Database.config - spec = - ActiveRecord:: - ConnectionAdapters:: - ConnectionSpecification::Resolver.new(config).spec(env.to_sym) + env_config = original_config.merge(pool: pool_size) + env_config[:host] = host if host + env_config[:port] = port if port - ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec) + ActiveRecord::ConnectionAdapters::ConnectionHandler.new.establish_connection(env_config) end def self.connection @@ -246,7 +275,7 @@ module Gitlab connection true - rescue + rescue StandardError false end @@ -347,4 +376,4 @@ module Gitlab end end -Gitlab::Database.prepend_if_ee('EE::Gitlab::Database') +Gitlab::Database.prepend_mod_with('Gitlab::Database') diff --git a/lib/gitlab/database/as_with_materialized.rb b/lib/gitlab/database/as_with_materialized.rb index 7c45f416638..e7e3c1766a9 100644 --- a/lib/gitlab/database/as_with_materialized.rb +++ b/lib/gitlab/database/as_with_materialized.rb @@ -3,19 +3,15 @@ module Gitlab module Database # This class is a special Arel node which allows optionally define the `MATERIALIZED` keyword for CTE and Recursive CTE queries. - class AsWithMaterialized < Arel::Nodes::Binary + class AsWithMaterialized < Arel::Nodes::As extend Gitlab::Utils::StrongMemoize - MATERIALIZED = Arel.sql(' MATERIALIZED') - EMPTY_STRING = Arel.sql('') - attr_reader :expr + MATERIALIZED = 'MATERIALIZED ' def initialize(left, right, materialized: true) - @expr = if materialized && self.class.materialized_supported? - MATERIALIZED - else - EMPTY_STRING - end + if materialized && self.class.materialized_supported? + right.prepend(MATERIALIZED) + end super(left, right) end diff --git a/lib/gitlab/database/background_migration/batch_optimizer.rb b/lib/gitlab/database/background_migration/batch_optimizer.rb new file mode 100644 index 00000000000..0668490dda8 --- /dev/null +++ b/lib/gitlab/database/background_migration/batch_optimizer.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + # This is an optimizer for throughput of batched migration jobs + # + # The underyling mechanic is based on the concept of time efficiency: + # time efficiency = job duration / interval + # Ideally, this is close but lower than 1 - so we're using time efficiently. + # + # We aim to land in the 90%-98% range, which gives the database a little breathing room + # in between. + # + # The optimizer is based on calculating the exponential moving average of time efficiencies + # for the last N jobs. If we're outside the range, we add 10% to or decrease by 20% of the batch size. + class BatchOptimizer + # Target time efficiency for a job + # Time efficiency is defined as: job duration / interval + TARGET_EFFICIENCY = (0.9..0.95).freeze + + # Lower and upper bound for the batch size + ALLOWED_BATCH_SIZE = (1_000..2_000_000).freeze + + # Limit for the multiplier of the batch size + MAX_MULTIPLIER = 1.2 + + # When smoothing time efficiency, use this many jobs + NUMBER_OF_JOBS = 20 + + # Smoothing factor for exponential moving average + EMA_ALPHA = 0.4 + + attr_reader :migration, :number_of_jobs, :ema_alpha + + def initialize(migration, number_of_jobs: NUMBER_OF_JOBS, ema_alpha: EMA_ALPHA) + @migration = migration + @number_of_jobs = number_of_jobs + @ema_alpha = ema_alpha + end + + def optimize! + return unless Feature.enabled?(:optimize_batched_migrations, type: :ops, default_enabled: :yaml) + + if multiplier = batch_size_multiplier + migration.batch_size = (migration.batch_size * multiplier).to_i.clamp(ALLOWED_BATCH_SIZE) + migration.save! + end + end + + private + + def batch_size_multiplier + efficiency = migration.smoothed_time_efficiency(number_of_jobs: number_of_jobs, alpha: ema_alpha) + + return if efficiency.nil? || efficiency == 0 + + # We hit the range - no change + return if TARGET_EFFICIENCY.include?(efficiency) + + # Assumption: time efficiency is linear in the batch size + [TARGET_EFFICIENCY.max / efficiency, MAX_MULTIPLIER].min + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 3b624df2bfd..869b97b8ac0 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -4,10 +4,23 @@ module Gitlab module Database module BackgroundMigration class BatchedJob < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord + include FromUnion + self.table_name = :batched_background_migration_jobs + MAX_ATTEMPTS = 3 + STUCK_JOBS_TIMEOUT = 1.hour.freeze + belongs_to :batched_migration, foreign_key: :batched_background_migration_id + scope :active, -> { where(status: [:pending, :running]) } + scope :stuck, -> { active.where('updated_at <= ?', STUCK_JOBS_TIMEOUT.ago) } + scope :retriable, -> { + failed_jobs = where(status: :failed).where('attempts < ?', MAX_ATTEMPTS) + + from_union([failed_jobs, self.stuck]) + } + enum status: { pending: 0, running: 1, @@ -15,8 +28,22 @@ module Gitlab succeeded: 3 } + scope :successful_in_execution_order, -> { where.not(finished_at: nil).succeeded.order(:finished_at) } + delegate :aborted?, :job_class, :table_name, :column_name, :job_arguments, to: :batched_migration, prefix: :migration + + attribute :pause_ms, :integer, default: 100 + + def time_efficiency + return unless succeeded? + return unless finished_at && started_at + + duration = finished_at - started_at + + # TODO: Switch to individual job interval (prereq: https://gitlab.com/gitlab-org/gitlab/-/issues/328801) + duration.to_f / batched_migration.interval + end end end end diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 4aa33ed7946..e85162f355e 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -20,9 +20,12 @@ module Gitlab paused: 0, active: 1, aborted: 2, - finished: 3 + finished: 3, + failed: 4 } + attribute :pause_ms, :integer, default: 100 + def self.active_migration active.queue_order.first end @@ -35,7 +38,13 @@ module Gitlab end def create_batched_job!(min, max) - batched_jobs.create!(min_value: min, max_value: max, batch_size: batch_size, sub_batch_size: sub_batch_size) + batched_jobs.create!( + min_value: min, + max_value: max, + batch_size: batch_size, + sub_batch_size: sub_batch_size, + pause_ms: pause_ms + ) end def next_min_value @@ -58,12 +67,40 @@ module Gitlab write_attribute(:batch_class_name, class_name.demodulize) end + def migrated_tuple_count + batched_jobs.succeeded.sum(:batch_size) + end + def prometheus_labels @prometheus_labels ||= { migration_id: id, migration_identifier: "%s/%s.%s" % [job_class_name, table_name, column_name] } end + + def smoothed_time_efficiency(number_of_jobs: 10, alpha: 0.2) + jobs = batched_jobs.successful_in_execution_order.reverse_order.limit(number_of_jobs) + + return if jobs.size < number_of_jobs + + efficiencies = jobs.map(&:time_efficiency).reject(&:nil?).each_with_index + + dividend = efficiencies.reduce(0) do |total, (job_eff, i)| + total + job_eff * (1 - alpha)**i + end + + divisor = efficiencies.reduce(0) do |total, (job_eff, i)| + total + (1 - alpha)**i + end + + return if divisor == 0 + + (dividend / divisor).round(2) + end + + def optimize! + BatchOptimizer.new(self).optimize! + end end end end diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index cf8b61f5feb..67fe6c536e6 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -19,8 +19,10 @@ module Gitlab # # Note that this method is primarily intended to called by a scheduled worker. def run_migration_job(active_migration) - if next_batched_job = create_next_batched_job!(active_migration) + if next_batched_job = find_or_create_next_batched_job(active_migration) migration_wrapper.perform(next_batched_job) + + active_migration.optimize! else finish_active_migration(active_migration) end @@ -46,12 +48,12 @@ module Gitlab attr_reader :migration_wrapper - def create_next_batched_job!(active_migration) - next_batch_range = find_next_batch_range(active_migration) - - return if next_batch_range.nil? - - active_migration.create_batched_job!(next_batch_range.min, next_batch_range.max) + def find_or_create_next_batched_job(active_migration) + if next_batch_range = find_next_batch_range(active_migration) + active_migration.create_batched_job!(next_batch_range.min, next_batch_range.max) + else + active_migration.batched_jobs.retriable.first + end end def find_next_batch_range(active_migration) @@ -80,7 +82,13 @@ module Gitlab end def finish_active_migration(active_migration) - active_migration.finished! + return if active_migration.batched_jobs.active.exists? + + if active_migration.batched_jobs.failed.exists? + active_migration.failed! + else + active_migration.finished! + end end end end diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb index c276f8ce75b..e37df102872 100644 --- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb +++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb @@ -19,10 +19,10 @@ module Gitlab execute_batch(batch_tracking_record) batch_tracking_record.status = :succeeded - rescue => e + rescue Exception # rubocop:disable Lint/RescueException batch_tracking_record.status = :failed - raise e + raise ensure finish_tracking_execution(batch_tracking_record) track_prometheus_metrics(batch_tracking_record) @@ -31,7 +31,7 @@ module Gitlab private def start_tracking_execution(tracking_record) - tracking_record.update!(attempts: tracking_record.attempts + 1, status: :running, started_at: Time.current) + tracking_record.update!(attempts: tracking_record.attempts + 1, status: :running, started_at: Time.current, finished_at: nil, metrics: {}) end def execute_batch(tracking_record) @@ -43,6 +43,7 @@ module Gitlab tracking_record.migration_table_name, tracking_record.migration_column_name, tracking_record.sub_batch_size, + tracking_record.pause_ms, *tracking_record.migration_job_arguments) if job_instance.respond_to?(:batch_metrics) @@ -61,11 +62,12 @@ module Gitlab metric_for(:gauge_batch_size).set(base_labels, tracking_record.batch_size) metric_for(:gauge_sub_batch_size).set(base_labels, tracking_record.sub_batch_size) + metric_for(:gauge_interval).set(base_labels, tracking_record.batched_migration.interval) + metric_for(:gauge_job_duration).set(base_labels, (tracking_record.finished_at - tracking_record.started_at).to_i) metric_for(:counter_updated_tuples).increment(base_labels, tracking_record.batch_size) - - # Time efficiency: Ratio of duration to interval (ideal: less than, but close to 1) - efficiency = (tracking_record.finished_at - tracking_record.started_at).to_i / migration.interval.to_f - metric_for(:histogram_time_efficiency).observe(base_labels, efficiency) + metric_for(:gauge_migrated_tuples).set(base_labels, tracking_record.batched_migration.migrated_tuple_count) + metric_for(:gauge_total_tuple_count).set(base_labels, tracking_record.batched_migration.total_tuple_count) + metric_for(:gauge_last_update_time).set(base_labels, Time.current.to_i) if metrics = tracking_record.metrics metrics['timings']&.each do |key, timings| @@ -94,21 +96,35 @@ module Gitlab :batched_migration_job_sub_batch_size, 'Sub-batch size for a batched migration job' ), + gauge_interval: Gitlab::Metrics.gauge( + :batched_migration_job_interval_seconds, + 'Interval for a batched migration job' + ), + gauge_job_duration: Gitlab::Metrics.gauge( + :batched_migration_job_duration_seconds, + 'Duration for a batched migration job' + ), counter_updated_tuples: Gitlab::Metrics.counter( :batched_migration_job_updated_tuples_total, 'Number of tuples updated by batched migration job' ), + gauge_migrated_tuples: Gitlab::Metrics.gauge( + :batched_migration_migrated_tuples_total, + 'Total number of tuples migrated by a batched migration' + ), histogram_timings: Gitlab::Metrics.histogram( - :batched_migration_job_duration_seconds, - 'Timings for a batched migration job', + :batched_migration_job_query_duration_seconds, + 'Query timings for a batched migration job', {}, [0.1, 0.25, 0.5, 1, 5].freeze ), - histogram_time_efficiency: Gitlab::Metrics.histogram( - :batched_migration_job_time_efficiency, - 'Ratio of job duration to interval', - {}, - [0.5, 0.9, 1, 1.5, 2].freeze + gauge_total_tuple_count: Gitlab::Metrics.gauge( + :batched_migration_total_tuple_count, + 'Total tuple count the migration needs to touch' + ), + gauge_last_update_time: Gitlab::Metrics.gauge( + :batched_migration_last_update_time_seconds, + 'Unix epoch time in seconds' ) } end diff --git a/lib/gitlab/database/background_migration_job.rb b/lib/gitlab/database/background_migration_job.rb index 1b9d7cbc9a1..1121793917b 100644 --- a/lib/gitlab/database/background_migration_job.rb +++ b/lib/gitlab/database/background_migration_job.rb @@ -9,7 +9,7 @@ module Gitlab scope :for_migration_class, -> (class_name) { where(class_name: normalize_class_name(class_name)) } scope :for_migration_execution, -> (class_name, arguments) do - for_migration_class(class_name).where('arguments = ?', arguments.to_json) + for_migration_class(class_name).where('arguments = ?', arguments.to_json) # rubocop:disable Rails/WhereEquals end scope :for_partitioning_migration, -> (class_name, table_name) do diff --git a/lib/gitlab/database/consistency.rb b/lib/gitlab/database/consistency.rb index b7d06a26ddb..e99ea7a3232 100644 --- a/lib/gitlab/database/consistency.rb +++ b/lib/gitlab/database/consistency.rb @@ -28,4 +28,4 @@ module Gitlab end end -::Gitlab::Database::Consistency.singleton_class.prepend_if_ee('EE::Gitlab::Database::Consistency') +::Gitlab::Database::Consistency.singleton_class.prepend_mod_with('Gitlab::Database::Consistency') diff --git a/lib/gitlab/database/loose_index_scan_distinct_count.rb b/lib/gitlab/database/loose_index_scan_distinct_count.rb index 884f4d47ff8..26be07f91c4 100644 --- a/lib/gitlab/database/loose_index_scan_distinct_count.rb +++ b/lib/gitlab/database/loose_index_scan_distinct_count.rb @@ -11,7 +11,7 @@ module Gitlab # This query will read each element in the index matching the project_id filter. # If for a project_id has 100_000 issues, all 100_000 elements will be read. # - # A loose index scan will read only one entry from the index for each project_id to reduce the number of disk reads. + # A loose index scan will only read one entry from the index for each project_id to reduce the number of disk reads. # # Usage: # @@ -94,7 +94,7 @@ module Gitlab elsif column.is_a?(Arel::Attributes::Attribute) column else - raise ColumnConfigurationError.new("Cannot transform the column: #{column.inspect}, please provide the column name as string") + raise ColumnConfigurationError, "Cannot transform the column: #{column.inspect}, please provide the column name as string" end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index d06a73da8ac..3a94e109d2a 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -5,6 +5,7 @@ module Gitlab module MigrationHelpers include Migrations::BackgroundMigrationHelpers include DynamicModelHelpers + include Migrations::RenameTableHelpers # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS MAX_IDENTIFIER_NAME_LENGTH = 63 @@ -51,7 +52,7 @@ module Gitlab allow_null: options[:null] ) else - add_column(table_name, column_name, :datetime_with_timezone, options) + add_column(table_name, column_name, :datetime_with_timezone, **options) end end end @@ -143,13 +144,13 @@ module Gitlab options = options.merge({ algorithm: :concurrently }) - if index_exists?(table_name, column_name, options) + if index_exists?(table_name, column_name, **options) Gitlab::AppLogger.warn "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" return end disable_statement_timeout do - add_index(table_name, column_name, options) + add_index(table_name, column_name, **options) end end @@ -169,13 +170,13 @@ module Gitlab options = options.merge({ algorithm: :concurrently }) - unless index_exists?(table_name, column_name, options) + unless index_exists?(table_name, column_name, **options) Gitlab::AppLogger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" return end disable_statement_timeout do - remove_index(table_name, options.merge({ column: column_name })) + remove_index(table_name, **options.merge({ column: column_name })) end end @@ -205,7 +206,7 @@ module Gitlab end disable_statement_timeout do - remove_index(table_name, options.merge({ name: index_name })) + remove_index(table_name, **options.merge({ name: index_name })) end end @@ -565,7 +566,7 @@ module Gitlab check_trigger_permissions!(table) - remove_rename_triggers_for_postgresql(table, trigger_name) + remove_rename_triggers(table, trigger_name) remove_column(table, new) end @@ -576,8 +577,19 @@ module Gitlab # table - The name of the table to install the trigger in. # old_column - The name of the old column. # new_column - The name of the new column. - def install_rename_triggers(table, old_column, new_column) - install_rename_triggers_for_postgresql(table, old_column, new_column) + # trigger_name - The name of the trigger to use (optional). + def install_rename_triggers(table, old, new, trigger_name: nil) + Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).create(old, new, trigger_name: trigger_name) + end + + # Removes the triggers used for renaming a column concurrently. + def remove_rename_triggers(table, trigger) + Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).drop(trigger) + end + + # Returns the (base) name to use for triggers when renaming columns. + def rename_trigger_name(table, old, new) + Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).name(old, new) end # Changes the type of a column concurrently. @@ -663,7 +675,7 @@ module Gitlab install_rename_triggers(table, column, temp_column) end - rescue + rescue StandardError # create_column_from can not run inside a transaction, which means # that there is a risk that if any of the operations that follow it # fail, we'll be left with an inconsistent schema @@ -690,7 +702,7 @@ module Gitlab check_trigger_permissions!(table) - remove_rename_triggers_for_postgresql(table, trigger_name) + remove_rename_triggers(table, trigger_name) remove_column(table, old) end @@ -905,7 +917,11 @@ module Gitlab end end - # Initializes the conversion of an integer column to bigint + def convert_to_bigint_column(column) + "#{column}_convert_to_bigint" + end + + # Initializes the conversion of a set of integer columns to bigint # # It can be used for converting both a Primary Key and any Foreign Keys # that may reference it or any other integer column that we may want to @@ -923,14 +939,14 @@ module Gitlab # Note: this helper is intended to be used in a regular (pre-deployment) migration. # # This helper is part 1 of a multi-step migration process: - # 1. initialize_conversion_of_integer_to_bigint to create the new column and database triggers + # 1. initialize_conversion_of_integer_to_bigint to create the new columns and database trigger # 2. backfill_conversion_of_integer_to_bigint to copy historic data using background migrations # 3. remaining steps TBD, see #288005 # # table - The name of the database table containing the column - # column - The name of the column that we want to convert to bigint. + # columns - The name, or array of names, of the column(s) that we want to convert to bigint. # primary_key - The name of the primary key column (most often :id) - def initialize_conversion_of_integer_to_bigint(table, column, primary_key: :id) + def initialize_conversion_of_integer_to_bigint(table, columns, primary_key: :id) unless table_exists?(table) raise "Table #{table} does not exist" end @@ -939,34 +955,54 @@ module Gitlab raise "Column #{primary_key} does not exist on #{table}" end - unless column_exists?(table, column) - raise "Column #{column} does not exist on #{table}" + columns = Array.wrap(columns) + columns.each do |column| + next if column_exists?(table, column) + + raise ArgumentError, "Column #{column} does not exist on #{table}" end check_trigger_permissions!(table) - old_column = column_for(table, column) - tmp_column = "#{column}_convert_to_bigint" + conversions = columns.to_h { |column| [column, convert_to_bigint_column(column)] } with_lock_retries do - if (column.to_s == primary_key.to_s) || !old_column.null - # If the column to be converted is either a PK or is defined as NOT NULL, - # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow - # That way, we skip the expensive validation step required to add - # a NOT NULL constraint at the end of the process - add_column(table, tmp_column, :bigint, default: old_column.default || 0, null: false) - else - add_column(table, tmp_column, :bigint, default: old_column.default) + conversions.each do |(source_column, temporary_name)| + column = column_for(table, source_column) + + if (column.name.to_s == primary_key.to_s) || !column.null + # If the column to be converted is either a PK or is defined as NOT NULL, + # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow + # That way, we skip the expensive validation step required to add + # a NOT NULL constraint at the end of the process + add_column(table, temporary_name, :bigint, default: column.default || 0, null: false) + else + add_column(table, temporary_name, :bigint, default: column.default) + end end - install_rename_triggers(table, column, tmp_column) + install_rename_triggers(table, conversions.keys, conversions.values) end end - # Backfills the new column used in the conversion of an integer column to bigint using background migrations. + # Reverts `initialize_conversion_of_integer_to_bigint` + # + # table - The name of the database table containing the columns + # columns - The name, or array of names, of the column(s) that we're converting to bigint. + def revert_initialize_conversion_of_integer_to_bigint(table, columns) + columns = Array.wrap(columns) + temporary_columns = columns.map { |column| convert_to_bigint_column(column) } + + trigger_name = rename_trigger_name(table, columns, temporary_columns) + remove_rename_triggers(table, trigger_name) + + temporary_columns.each { |column| remove_column(table, column) } + end + + # Backfills the new columns used in an integer-to-bigint conversion using background migrations. # # - This helper should be called from a post-deployment migration. - # - In order for this helper to work properly, the new column must be first initialized with + # - In order for this helper to work properly, the new columns must be first initialized with # the `initialize_conversion_of_integer_to_bigint` helper. # - It tracks the scheduled background jobs through Gitlab::Database::BackgroundMigration::BatchedMigration, # which allows a more thorough check that all jobs succeeded in the @@ -976,12 +1012,12 @@ module Gitlab # deployed (including background job changes) before we begin processing the background migration. # # This helper is part 2 of a multi-step migration process: - # 1. initialize_conversion_of_integer_to_bigint to create the new column and database triggers + # 1. initialize_conversion_of_integer_to_bigint to create the new columns and database trigger # 2. backfill_conversion_of_integer_to_bigint to copy historic data using background migrations # 3. remaining steps TBD, see #288005 # # table - The name of the database table containing the column - # column - The name of the column that we want to convert to bigint. + # columns - The name, or an array of names, of the column(s) we want to convert to bigint. # primary_key - The name of the primary key column (most often :id) # batch_size - The number of rows to schedule in a single background migration # sub_batch_size - The smaller batches that will be used by each scheduled job @@ -1001,7 +1037,7 @@ module Gitlab # between the scheduled jobs def backfill_conversion_of_integer_to_bigint( table, - column, + columns, primary_key: :id, batch_size: 20_000, sub_batch_size: 1000, @@ -1016,46 +1052,43 @@ module Gitlab raise "Column #{primary_key} does not exist on #{table}" end - unless column_exists?(table, column) - raise "Column #{column} does not exist on #{table}" - end + conversions = Array.wrap(columns).to_h do |column| + raise ArgumentError, "Column #{column} does not exist on #{table}" unless column_exists?(table, column) - tmp_column = "#{column}_convert_to_bigint" + temporary_name = convert_to_bigint_column(column) + raise ArgumentError, "Column #{temporary_name} does not exist on #{table}" unless column_exists?(table, temporary_name) - unless column_exists?(table, tmp_column) - raise 'The temporary column does not exist, initialize it with `initialize_conversion_of_integer_to_bigint`' + [column, temporary_name] end - batched_migration = queue_batched_background_migration( + queue_batched_background_migration( 'CopyColumnUsingBackgroundMigrationJob', table, primary_key, - column, - tmp_column, + conversions.keys, + conversions.values, job_interval: interval, batch_size: batch_size, sub_batch_size: sub_batch_size) - - if perform_background_migration_inline? - # To ensure the schema is up to date immediately we perform the - # migration inline in dev / test environments. - Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new.run_entire_migration(batched_migration) - end end - # Performs a concurrent column rename when using PostgreSQL. - def install_rename_triggers_for_postgresql(table, old, new, trigger_name: nil) - Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).create(old, new, trigger_name: trigger_name) - end + # Reverts `backfill_conversion_of_integer_to_bigint` + # + # table - The name of the database table containing the column + # columns - The name, or an array of names, of the column(s) we want to convert to bigint. + # primary_key - The name of the primary key column (most often :id) + def revert_backfill_conversion_of_integer_to_bigint(table, columns, primary_key: :id) + columns = Array.wrap(columns) - # Removes the triggers used for renaming a PostgreSQL column concurrently. - def remove_rename_triggers_for_postgresql(table, trigger) - Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).drop(trigger) - end + conditions = ActiveRecord::Base.sanitize_sql([ + 'job_class_name = :job_class_name AND table_name = :table_name AND column_name = :column_name AND job_arguments = :job_arguments', + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: table, + column_name: primary_key, + job_arguments: [columns, columns.map { |column| convert_to_bigint_column(column) }].to_json + ]) - # Returns the (base) name to use for triggers when renaming columns. - def rename_trigger_name(table, old, new) - Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).name(old, new) + execute("DELETE FROM batched_background_migrations WHERE #{conditions}") end # Returns an Array containing the indexes for the given column @@ -1162,8 +1195,8 @@ module Gitlab end end - def remove_foreign_key_without_error(*args) - remove_foreign_key(*args) + def remove_foreign_key_without_error(*args, **kwargs) + remove_foreign_key(*args, **kwargs) rescue ArgumentError end diff --git a/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb b/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb new file mode 100644 index 00000000000..eecf96acb30 --- /dev/null +++ b/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module CascadingNamespaceSettings + include Gitlab::Database::MigrationHelpers + + # Creates the four required columns that constitutes a single cascading + # namespace settings attribute. This helper is only appropriate if the + # setting is not already present as a non-cascading attribute. + # + # Creates the `setting_name` column along with the `lock_setting_name` + # column in both `namespace_settings` and `application_settings`. + # + # This helper is not reversible and must be defined in conjunction with + # `remove_cascading_namespace_setting` in separate up and down directions. + # + # setting_name - The name of the cascading attribute - same as defined + # in `NamespaceSetting` with the `cascading_attr` method. + # type - The column type for the setting itself (:boolean, :integer, etc.) + # options - Standard Rails column options hash. Accepts keys such as + # `null` and `default`. + # + # `null` and `default` options will only be applied to the `application_settings` + # column. In most cases, a non-null default value should be specified. + def add_cascading_namespace_setting(setting_name, type, **options) + lock_column_name = "lock_#{setting_name}".to_sym + + check_cascading_namespace_setting_consistency(setting_name, lock_column_name) + + namespace_options = options.merge(null: true, default: nil) + + with_lock_retries do + add_column(:namespace_settings, setting_name, type, namespace_options) + add_column(:namespace_settings, lock_column_name, :boolean, default: false, null: false) + end + + add_column(:application_settings, setting_name, type, options) + add_column(:application_settings, lock_column_name, :boolean, default: false, null: false) + end + + def remove_cascading_namespace_setting(setting_name) + lock_column_name = "lock_#{setting_name}".to_sym + + with_lock_retries do + remove_column(:namespace_settings, setting_name) if column_exists?(:namespace_settings, setting_name) + remove_column(:namespace_settings, lock_column_name) if column_exists?(:namespace_settings, lock_column_name) + end + + remove_column(:application_settings, setting_name) if column_exists?(:application_settings, setting_name) + remove_column(:application_settings, lock_column_name) if column_exists?(:application_settings, lock_column_name) + end + + private + + def check_cascading_namespace_setting_consistency(setting_name, lock_name) + existing_columns = [] + + %w(namespace_settings application_settings).each do |table| + existing_columns << "#{table}.#{setting_name}" if column_exists?(table.to_sym, setting_name) + existing_columns << "#{table}.#{lock_name}" if column_exists?(table.to_sym, lock_name) + end + + return if existing_columns.empty? + + raise <<~ERROR + One or more cascading namespace columns already exist. `add_cascading_namespace_setting` helper + can only be used for new settings, when none of the required columns already exist. + Existing columns: #{existing_columns.join(', ')} + ERROR + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb index 959028ce00b..e9ef80d5198 100644 --- a/lib/gitlab/database/migrations/instrumentation.rb +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -4,6 +4,9 @@ module Gitlab module Database module Migrations class Instrumentation + RESULT_DIR = Rails.root.join('tmp', 'migration-testing').freeze + STATS_FILENAME = 'migration-stats.json' + attr_reader :observations def initialize(observers = ::Gitlab::Database::Migrations::Observers.all_observers) @@ -21,7 +24,7 @@ module Gitlab observation.walltime = Benchmark.realtime do yield - rescue => e + rescue StandardError => e exception = e observation.success = false end @@ -47,7 +50,7 @@ module Gitlab def on_each_observer(&block) observers.each do |observer| yield observer - rescue => e + rescue StandardError => e Gitlab::AppLogger.error("Migration observer #{observer.class} failed with: #{e}") end end diff --git a/lib/gitlab/database/migrations/observers.rb b/lib/gitlab/database/migrations/observers.rb index 592993aeac5..b65a303ef30 100644 --- a/lib/gitlab/database/migrations/observers.rb +++ b/lib/gitlab/database/migrations/observers.rb @@ -7,7 +7,8 @@ module Gitlab def self.all_observers [ TotalDatabaseSizeChange.new, - QueryStatistics.new + QueryStatistics.new, + QueryLog.new ] end end diff --git a/lib/gitlab/database/migrations/observers/query_log.rb b/lib/gitlab/database/migrations/observers/query_log.rb new file mode 100644 index 00000000000..45df07fe391 --- /dev/null +++ b/lib/gitlab/database/migrations/observers/query_log.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module Observers + class QueryLog < MigrationObserver + def before + @logger_was = ActiveRecord::Base.logger + @log_file_path = File.join(Instrumentation::RESULT_DIR, 'current.log') + @logger = Logger.new(@log_file_path) + ActiveRecord::Base.logger = @logger + end + + def after + ActiveRecord::Base.logger = @logger_was + @logger.close + end + + def record(observation) + File.rename(@log_file_path, File.join(Instrumentation::RESULT_DIR, "#{observation.migration}.log")) + end + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/partition_creator.rb b/lib/gitlab/database/partitioning/partition_creator.rb index 547e0b9b957..d4b2b8d50e2 100644 --- a/lib/gitlab/database/partitioning/partition_creator.rb +++ b/lib/gitlab/database/partitioning/partition_creator.rb @@ -38,7 +38,7 @@ module Gitlab create(model, partitions_to_create) end - rescue => e + rescue StandardError => e Gitlab::AppLogger.error("Failed to create partition(s) for #{model.table_name}: #{e.class}: #{e.message}") end end diff --git a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb index 0bc1343acca..c0cc97de276 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb @@ -40,7 +40,7 @@ module Gitlab end with_lock_retries do - add_index(table_name, column_names, options) + add_index(table_name, column_names, **options) end end diff --git a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb index e8b49c7f62c..aa46b98be5d 100644 --- a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb +++ b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb @@ -69,10 +69,8 @@ module Gitlab hll_buckets = Buckets.new while batch_start <= finish - begin - hll_buckets.merge_hash!(hll_buckets_for_batch(batch_start, batch_start + batch_size)) - batch_start += batch_size - end + hll_buckets.merge_hash!(hll_buckets_for_batch(batch_start, batch_start + batch_size)) + batch_start += batch_size sleep(SLEEP_TIME_IN_SECONDS) end diff --git a/lib/gitlab/database/reindexing/concurrent_reindex.rb b/lib/gitlab/database/reindexing/concurrent_reindex.rb index a6fe7d61a4f..7e2dd55d21b 100644 --- a/lib/gitlab/database/reindexing/concurrent_reindex.rb +++ b/lib/gitlab/database/reindexing/concurrent_reindex.rb @@ -11,7 +11,14 @@ module Gitlab PG_IDENTIFIER_LENGTH = 63 TEMPORARY_INDEX_PREFIX = 'tmp_reindex_' REPLACED_INDEX_PREFIX = 'old_reindex_' - STATEMENT_TIMEOUT = 6.hours + STATEMENT_TIMEOUT = 9.hours + + # When dropping an index, we acquire a SHARE UPDATE EXCLUSIVE lock, + # which only conflicts with DDL and vacuum. We therefore execute this with a rather + # high lock timeout and a long pause in between retries. This is an alternative to + # setting a high statement timeout, which would lead to a long running query with effects + # on e.g. vacuum. + REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30 attr_reader :index, :logger @@ -70,7 +77,7 @@ module Gitlab ensure begin remove_index(index.schema, replacement_index_name) - rescue => e + rescue StandardError => e logger.error(e) end end @@ -95,7 +102,13 @@ module Gitlab def remove_index(schema, name) logger.info("Removing index #{schema}.#{name}") - set_statement_timeout do + retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( + timing_configuration: REMOVE_INDEX_RETRY_CONFIG, + klass: self.class, + logger: logger + ) + + retries.run(raise_on_exhaustion: false) do connection.execute(<<~SQL) DROP INDEX CONCURRENTLY IF EXISTS #{quote_table_name(schema)}.#{quote_table_name(name)} @@ -121,7 +134,6 @@ module Gitlab def with_lock_retries(&block) arguments = { klass: self.class, logger: logger } - Gitlab::Database::WithLockRetries.new(**arguments).run(raise_on_exhaustion: true, &block) end diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb index 7a7d17ca196..d68f47b5b6c 100644 --- a/lib/gitlab/database/reindexing/coordinator.rb +++ b/lib/gitlab/database/reindexing/coordinator.rb @@ -42,7 +42,7 @@ module Gitlab def perform_for(index, action) ConcurrentReindex.new(index).perform - rescue + rescue StandardError action.state = :failed raise diff --git a/lib/gitlab/database/reindexing/grafana_notifier.rb b/lib/gitlab/database/reindexing/grafana_notifier.rb index b1e5ecb9ade..f4ea59deb50 100644 --- a/lib/gitlab/database/reindexing/grafana_notifier.rb +++ b/lib/gitlab/database/reindexing/grafana_notifier.rb @@ -53,7 +53,7 @@ module Gitlab log_error("Response code #{response.code}") unless success success - rescue => err + rescue StandardError => err log_error(err) false diff --git a/lib/gitlab/database/rename_table_helpers.rb b/lib/gitlab/database/rename_table_helpers.rb new file mode 100644 index 00000000000..7f5af038c6d --- /dev/null +++ b/lib/gitlab/database/rename_table_helpers.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module RenameTableHelpers + def rename_table_safely(old_table_name, new_table_name) + with_lock_retries do + rename_table(old_table_name, new_table_name) + execute("CREATE VIEW #{old_table_name} AS SELECT * FROM #{new_table_name}") + end + end + + def undo_rename_table_safely(old_table_name, new_table_name) + with_lock_retries do + execute("DROP VIEW IF EXISTS #{old_table_name}") + rename_table(new_table_name, old_table_name) + end + end + + def finalize_table_rename(old_table_name, new_table_name) + with_lock_retries do + execute("DROP VIEW IF EXISTS #{old_table_name}") + end + end + + def undo_finalize_table_rename(old_table_name, new_table_name) + with_lock_retries do + execute("CREATE VIEW #{old_table_name} AS SELECT * FROM #{new_table_name}") + end + end + end + end +end diff --git a/lib/gitlab/database/schema_cache_with_renamed_table.rb b/lib/gitlab/database/schema_cache_with_renamed_table.rb new file mode 100644 index 00000000000..28123edd708 --- /dev/null +++ b/lib/gitlab/database/schema_cache_with_renamed_table.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaCacheWithRenamedTable + # Override methods in ActiveRecord::ConnectionAdapters::SchemaCache + + def clear! + super + + clear_renamed_tables_cache! + end + + def clear_data_source_cache!(name) + super(name) + + clear_renamed_tables_cache! + end + + def primary_keys(table_name) + super(underlying_table(table_name)) + end + + def columns(table_name) + super(underlying_table(table_name)) + end + + def columns_hash(table_name) + super(underlying_table(table_name)) + end + + def indexes(table_name) + super(underlying_table(table_name)) + end + + private + + def underlying_table(table_name) + renamed_tables_cache.fetch(table_name, table_name) + end + + def renamed_tables_cache + @renamed_tables ||= begin + Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, new_name| + ActiveRecord::Base.connection.view_exists?(old_name) + end + end + end + + def clear_renamed_tables_cache! + @renamed_tables = nil # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + end + end +end diff --git a/lib/gitlab/database/with_lock_retries.rb b/lib/gitlab/database/with_lock_retries.rb index 3fb52d786ad..bbf8f133f0f 100644 --- a/lib/gitlab/database/with_lock_retries.rb +++ b/lib/gitlab/database/with_lock_retries.rb @@ -92,7 +92,7 @@ module Gitlab end begin - run_block_with_transaction + run_block_with_lock_timeout rescue ActiveRecord::LockWaitTimeout if retry_with_lock_timeout? disable_idle_in_transaction_timeout if ActiveRecord::Base.connection.transaction_open? @@ -121,7 +121,7 @@ module Gitlab block.call end - def run_block_with_transaction + def run_block_with_lock_timeout ActiveRecord::Base.transaction(requires_new: true) do execute("SET LOCAL lock_timeout TO '#{current_lock_timeout_in_ms}ms'") diff --git a/lib/gitlab/database/with_lock_retries_outside_transaction.rb b/lib/gitlab/database/with_lock_retries_outside_transaction.rb new file mode 100644 index 00000000000..175cc493e36 --- /dev/null +++ b/lib/gitlab/database/with_lock_retries_outside_transaction.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # This retry method behaves similar to WithLockRetries + # except it does not wrap itself into a transaction scope. + # + # In our context, this is only useful if directly connected to + # PostgreSQL. When going through pgbouncer, this method **won't work** + # as it relies on using `SET` outside transactions (and hence can be + # multiplexed across different connections). + class WithLockRetriesOutsideTransaction < WithLockRetries + private + + def run_block_with_lock_timeout + execute("SET lock_timeout TO '#{current_lock_timeout_in_ms}ms'") + + log(message: 'Lock timeout is set', current_iteration: current_iteration, lock_timeout_in_ms: current_lock_timeout_in_ms) + + run_block + + log(message: 'Migration finished', current_iteration: current_iteration, lock_timeout_in_ms: current_lock_timeout_in_ms) + end + + def run_block_without_lock_timeout + log(message: "Couldn't acquire lock to perform the migration", current_iteration: current_iteration) + log(message: "Executing without lock timeout", current_iteration: current_iteration) + + disable_lock_timeout + + run_block + + log(message: 'Migration finished', current_iteration: current_iteration) + end + + def disable_lock_timeout + execute("SET lock_timeout TO '0'") + end + end + end +end diff --git a/lib/gitlab/default_branch.rb b/lib/gitlab/default_branch.rb new file mode 100644 index 00000000000..6bd9a5675c4 --- /dev/null +++ b/lib/gitlab/default_branch.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +# Class is used while we're migrating from master to main +module Gitlab + module DefaultBranch + def self.value(object: nil) + Feature.enabled?(:main_branch_over_master, object, default_enabled: :yaml) ? 'main' : 'master' + end + end +end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 627abfbfe7e..9ed03c05f0b 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -117,8 +117,6 @@ module Gitlab end def sort_diffs(diffs) - return diffs unless Feature.enabled?(:sort_diffs, project, default_enabled: :yaml) - Gitlab::Diff::FileCollectionSorter.new(diffs).sort end end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 8385bbbb3de..6a41ed0f29e 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -3,6 +3,8 @@ module Gitlab module Diff class Highlight + PREFIX_REGEXP = /\A(.)/.freeze + attr_reader :diff_file, :diff_lines, :repository, :project delegate :old_path, :new_path, :old_sha, :new_sha, to: :diff_file, prefix: :diff @@ -85,6 +87,7 @@ module Gitlab def highlight_line(diff_line) return unless diff_file && diff_file.diff_refs + return diff_line_highlighting(diff_line, plain: true) if blobs_too_large? if Feature.enabled?(:diff_line_syntax_highlighting, project, default_enabled: :yaml) diff_line_highlighting(diff_line) @@ -93,16 +96,17 @@ module Gitlab end end - def diff_line_highlighting(diff_line) + def diff_line_highlighting(diff_line, plain: false) rich_line = syntax_highlighter(diff_line).highlight( diff_line.text(prefix: false), + plain: plain, context: { line_number: diff_line.line } - )&.html_safe + ) # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. if rich_line - line_prefix = diff_line.text =~ /\A(.)/ ? Regexp.last_match(1) : ' ' + line_prefix = diff_line.text =~ PREFIX_REGEXP ? Regexp.last_match(1) : ' ' rich_line.prepend(line_prefix).concat("\n") end end @@ -131,7 +135,7 @@ module Gitlab # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. if rich_line - line_prefix = diff_line.text =~ /\A(.)/ ? Regexp.last_match(1) : ' ' + line_prefix = diff_line.text =~ PREFIX_REGEXP ? Regexp.last_match(1) : ' ' "#{line_prefix}#{rich_line}".html_safe end end @@ -156,6 +160,13 @@ module Gitlab blob.load_all_data! blob.present.highlight.lines end + + def blobs_too_large? + return false unless Feature.enabled?(:limited_diff_highlighting, project, default_enabled: :yaml) + return true if Gitlab::Highlight.too_large?(diff_file.old_blob&.size) + + Gitlab::Highlight.too_large?(diff_file.new_blob&.size) + end end end end diff --git a/lib/gitlab/doctor/secrets.rb b/lib/gitlab/doctor/secrets.rb index 31c5dded3ff..1a1e9fafb1e 100644 --- a/lib/gitlab/doctor/secrets.rb +++ b/lib/gitlab/doctor/secrets.rb @@ -77,7 +77,7 @@ module Gitlab true rescue OpenSSL::Cipher::CipherError, TypeError false - rescue => e + rescue StandardError => e logger.debug "> Something went wrong for #{data.class.name}[#{data.id}].#{attr}: #{e}".color(:red) false diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 22fc8addcd9..e927a5641e5 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -56,10 +56,12 @@ module Gitlab def create_issue Issues::CreateService.new( - project, - author, - title: mail.subject, - description: message_including_reply + project: project, + current_user: author, + params: { + title: mail.subject, + description: message_including_reply + } ).execute end diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index e8071bcafd0..df12aea1988 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -61,7 +61,7 @@ module Gitlab private def build_merge_request - MergeRequests::BuildService.new(project, author, merge_request_params).execute + MergeRequests::BuildService.new(project: project, current_user: author, params: merge_request_params).execute end def create_merge_request @@ -78,7 +78,7 @@ module Gitlab if merge_request.errors.any? merge_request else - MergeRequests::CreateService.new(project, author).create(merge_request) + MergeRequests::CreateService.new(project: project, current_user: author).create(merge_request) end end diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb index 9e476dd4e2b..63334169c8e 100644 --- a/lib/gitlab/email/handler/reply_processing.rb +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -100,4 +100,4 @@ module Gitlab end end -Gitlab::Email::Handler::ReplyProcessing.prepend_if_ee('::EE::Gitlab::Email::Handler::ReplyProcessing') +Gitlab::Email::Handler::ReplyProcessing.prepend_mod_with('Gitlab::Email::Handler::ReplyProcessing') diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 80e8b726099..cab3538a447 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -38,7 +38,7 @@ module Gitlab if from_address add_email_participant - send_thank_you_email! + send_thank_you_email end end @@ -77,12 +77,14 @@ module Gitlab def create_issue! @issue = Issues::CreateService.new( - project, - User.support_bot, - title: mail.subject, - description: message_including_template, - confidential: true, - external_author: from_address + project: project, + current_user: User.support_bot, + params: { + title: mail.subject, + description: message_including_template, + confidential: true, + external_author: from_address + } ).execute raise InvalidIssueError unless @issue.persisted? @@ -92,8 +94,8 @@ module Gitlab end end - def send_thank_you_email! - Notify.service_desk_thank_you_email(@issue.id).deliver_later! + def send_thank_you_email + Notify.service_desk_thank_you_email(@issue.id).deliver_later end def message_including_template diff --git a/lib/gitlab/email/message/in_product_marketing.rb b/lib/gitlab/email/message/in_product_marketing.rb new file mode 100644 index 00000000000..d538238f26f --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + UnknownTrackError = Class.new(StandardError) + + TRACKS = [:create, :verify, :team, :trial].freeze + + def self.for(track) + raise UnknownTrackError unless TRACKS.include?(track) + + "Gitlab::Email::Message::InProductMarketing::#{track.to_s.classify}".constantize + end + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/base.rb b/lib/gitlab/email/message/in_product_marketing/base.rb new file mode 100644 index 00000000000..6341a7c7596 --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing/base.rb @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + class Base + include Gitlab::Email::Message::InProductMarketing::Helper + include Gitlab::Routing + + attr_accessor :format + + def initialize(group:, series:, format: :html) + raise ArgumentError, "Only #{total_series} series available for this track." unless series.between?(0, total_series - 1) + + @group = group + @series = series + @format = format + end + + def subject_line + raise NotImplementedError + end + + def tagline + raise NotImplementedError + end + + def title + raise NotImplementedError + end + + def subtitle + raise NotImplementedError + end + + def body_line1 + raise NotImplementedError + end + + def body_line2 + raise NotImplementedError + end + + def cta_text + raise NotImplementedError + end + + def cta_link + case format + when :html + link_to cta_text, group_email_campaigns_url(group, track: track, series: series), target: '_blank', rel: 'noopener noreferrer' + else + [cta_text, group_email_campaigns_url(group, track: track, series: series)].join(' >> ') + end + end + + def unsubscribe + parts = Gitlab.com? ? unsubscribe_com : unsubscribe_self_managed(track, series) + + case format + when :html + parts.join(' ') + else + parts.join("\n" + ' ' * 16) + end + end + + def progress + if Gitlab.com? + s_('InProductMarketing|This is email %{current_series} of %{total_series} in the %{track} series.') % { current_series: series + 1, total_series: total_series, track: track.to_s.humanize } + else + s_('InProductMarketing|This is email %{current_series} of %{total_series} in the %{track} series. To disable notification emails sent by your local GitLab instance, either contact your administrator or %{unsubscribe_link}.') % { current_series: series + 1, total_series: total_series, track: track.to_s.humanize, unsubscribe_link: unsubscribe_link } + end + end + + def address + s_('InProductMarketing|%{strong_start}GitLab Inc.%{strong_end} 268 Bush Street, #350, San Francisco, CA 94104, USA').html_safe % strong_options + end + + def footer_links + links = [ + [s_('InProductMarketing|Blog'), 'https://about.gitlab.com/blog'], + [s_('InProductMarketing|Twitter'), 'https://twitter.com/gitlab'], + [s_('InProductMarketing|Facebook'), 'https://www.facebook.com/gitlab'], + [s_('InProductMarketing|YouTube'), 'https://www.youtube.com/channel/UCnMGQ8QHMAnVIsI3xJrihhg'] + ] + case format + when :html + links.map do |text, link| + link_to(text, link) + end + else + '| ' + links.map do |text, link| + [text, link].join(' ') + end.join("\n| ") + end + end + + def logo_path + ["mailers/in_product_marketing", "#{track}-#{series}.png"].join('/') + end + + protected + + attr_reader :group, :series + + def total_series + 3 + end + + private + + def track + self.class.name.demodulize.downcase.to_sym + end + + def unsubscribe_com + [ + s_('InProductMarketing|If you no longer wish to receive marketing emails from us,'), + s_('InProductMarketing|you may %{unsubscribe_link} at any time.') % { unsubscribe_link: unsubscribe_link } + ] + end + + def unsubscribe_self_managed(track, series) + [ + s_('InProductMarketing|To opt out of these onboarding emails, %{unsubscribe_link}.') % { unsubscribe_link: unsubscribe_link }, + s_("InProductMarketing|If you don't want to receive marketing emails directly from GitLab, %{marketing_preference_link}.") % { marketing_preference_link: marketing_preference_link(track, series) } + ] + end + + def unsubscribe_link + unsubscribe_url = Gitlab.com? ? '%tag_unsubscribe_url%' : profile_notifications_url + + link(s_('InProductMarketing|unsubscribe'), unsubscribe_url) + end + + def marketing_preference_link(track, series) + params = { + utm_source: 'SM', + utm_medium: 'email', + utm_campaign: 'onboarding', + utm_term: "#{track}_#{series}" + } + + preference_link = "https://about.gitlab.com/company/preference-center/?#{params.to_query}" + + link(s_('InProductMarketing|update your preferences'), preference_link) + end + end + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/create.rb b/lib/gitlab/email/message/in_product_marketing/create.rb new file mode 100644 index 00000000000..5d3cac0a121 --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing/create.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + class Create < Base + def subject_line + [ + s_('InProductMarketing|Create a project in GitLab in 5 minutes'), + s_('InProductMarketing|Import your project and code from GitHub, Bitbucket and others'), + s_('InProductMarketing|Understand repository mirroring') + ][series] + end + + def tagline + [ + s_('InProductMarketing|Get started today'), + s_('InProductMarketing|Get our import guides'), + s_('InProductMarketing|Need an alternative to importing?') + ][series] + end + + def title + [ + s_('InProductMarketing|Take your first steps with GitLab'), + s_('InProductMarketing|Start by importing your projects'), + s_('InProductMarketing|How (and why) mirroring makes sense') + ][series] + end + + def subtitle + [ + s_('InProductMarketing|Dig in and create a project and a repo'), + s_("InProductMarketing|Here's what you need to know"), + s_('InProductMarketing|Try it out') + ][series] + end + + def body_line1 + [ + s_("InProductMarketing|To understand and get the most out of GitLab, start at the beginning and %{project_link}. In GitLab, repositories are part of a project, so after you've created your project you can go ahead and %{repo_link}.") % { project_link: project_link, repo_link: repo_link }, + s_("InProductMarketing|Making the switch? It's easier than you think to import your projects into GitLab. Move %{github_link}, or import something %{bitbucket_link}.") % { github_link: github_link, bitbucket_link: bitbucket_link }, + s_("InProductMarketing|Sometimes you're not ready to make a full transition to a new tool. If you're not ready to fully commit, %{mirroring_link} gives you a safe way to try out GitLab in parallel with your current tool.") % { mirroring_link: mirroring_link } + ][series] + end + + def body_line2 + [ + s_("InProductMarketing|That's all it takes to get going with GitLab, but if you're new to working with Git, check out our %{basics_link} for helpful tips and tricks for getting started.") % { basics_link: basics_link }, + s_("InProductMarketing|Have a different instance you'd like to import? Here's our %{import_link}.") % { import_link: import_link }, + s_("InProductMarketing|It's also possible to simply %{external_repo_link} in order to take advantage of GitLab's CI/CD.") % { external_repo_link: external_repo_link } + ][series] + end + + def cta_text + [ + s_('InProductMarketing|Create your first project!'), + s_('InProductMarketing|Master the art of importing!'), + s_('InProductMarketing|Understand your project options') + ][series] + end + + private + + def project_link + link(s_('InProductMarketing|create a project'), help_page_url('gitlab-basics/create-project')) + end + + def repo_link + link(s_('InProductMarketing|set up a repo'), help_page_url('user/project/repository/index', anchor: 'create-a-repository')) + end + + def github_link + link(s_('InProductMarketing|GitHub Enterprise projects to GitLab'), help_page_url('integration/github')) + end + + def bitbucket_link + link(s_('InProductMarketing|from Bitbucket'), help_page_url('user/project/import/bitbucket_server')) + end + + def mirroring_link + link(s_('InProductMarketing|repository mirroring'), help_page_url('user/project/repository/repository_mirroring')) + end + + def basics_link + link(s_('InProductMarketing|Git basics'), help_page_url('gitlab-basics/README')) + end + + def import_link + link(s_('InProductMarketing|comprehensive guide'), help_page_url('user/project/import/index')) + end + + def external_repo_link + link(s_('InProductMarketing|connect an external repository'), new_project_url(anchor: 'cicd_for_external_repo')) + end + end + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/helper.rb b/lib/gitlab/email/message/in_product_marketing/helper.rb new file mode 100644 index 00000000000..4780e08322a --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing/helper.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + module Helper + include ActionView::Context + include ActionView::Helpers::TagHelper + include ActionView::Helpers::UrlHelper + + private + + def list(array) + case format + when :html + tag.ul { array.map { |item| tag.li item} } + else + '- ' + array.join("\n- ") + end + end + + def strong_options + case format + when :html + { strong_start: '<b>'.html_safe, strong_end: '</b>'.html_safe } + else + { strong_start: '', strong_end: '' } + end + end + + def link(text, link) + case format + when :html + link_to text, link + else + "#{text} (#{link})" + end + end + end + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/team.rb b/lib/gitlab/email/message/in_product_marketing/team.rb new file mode 100644 index 00000000000..46c2797e534 --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing/team.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + class Team < Base + def subject_line + [ + s_('InProductMarketing|Working in GitLab = more efficient'), + s_("InProductMarketing|Multiple owners, confusing workstreams? We've got you covered"), + s_('InProductMarketing|Your teams can be more efficient') + ][series] + end + + def tagline + [ + s_('InProductMarketing|Invite your colleagues to join in less than one minute'), + s_('InProductMarketing|Get your team set up on GitLab'), + nil + ][series] + end + + def title + [ + s_('InProductMarketing|Team work makes the dream work'), + s_('InProductMarketing|*GitLab*, noun: a synonym for efficient teams'), + s_('InProductMarketing|Find out how your teams are really doing') + ][series] + end + + def subtitle + [ + s_('InProductMarketing|Actually, GitLab makes the team work (better)'), + s_('InProductMarketing|Our tool brings all the things together'), + s_("InProductMarketing|It's all in the stats") + ][series] + end + + def body_line1 + [ + [ + s_('InProductMarketing|Did you know teams that use GitLab are far more efficient?'), + list([ + s_('InProductMarketing|Goldman Sachs went from 1 build every two weeks to thousands of builds a day'), + s_('InProductMarketing|Ticketmaster decreased their CI build time by 15X') + ]) + ].join("\n"), + s_("InProductMarketing|We know a thing or two about efficiency and we don't want to keep that to ourselves. Sign up for a free trial of GitLab Ultimate and your teams will be on it from day one."), + [ + s_('InProductMarketing|Stop wondering and use GitLab to answer questions like:'), + list([ + s_('InProductMarketing|How long does it take us to close issues/MRs by types like feature requests, bugs, tech debt, security?'), + s_('InProductMarketing|How many days does it take our team to complete various tasks?'), + s_('InProductMarketing|What does our value stream timeline look like from product to development to review and production?') + ]) + ].join("\n") + ][series] + end + + def body_line2 + [ + s_('InProductMarketing|Invite your colleagues and start shipping code faster.'), + s_("InProductMarketing|Streamline code review, know at a glance who's unavailable, communicate in comments or in email and integrate with Slack so everyone's on the same page."), + s_('InProductMarketing|When your team is on GitLab these answers are a click away.') + ][series] + end + + def cta_text + [ + s_('InProductMarketing|Invite your colleagues today'), + s_('InProductMarketing|Invite your team in less than 60 seconds'), + s_('InProductMarketing|Invite your team now') + ][series] + end + end + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/trial.rb b/lib/gitlab/email/message/in_product_marketing/trial.rb new file mode 100644 index 00000000000..d87dc5c1b81 --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing/trial.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + class Trial < Base + def subject_line + [ + s_('InProductMarketing|Go farther with GitLab'), + s_('InProductMarketing|Automated security scans directly within GitLab'), + s_('InProductMarketing|Take your source code management to the next level') + ][series] + end + + def tagline + [ + s_('InProductMarketing|Start a free trial of GitLab Ultimate – no CC required'), + s_('InProductMarketing|Improve app security with a 30-day trial'), + s_('InProductMarketing|Start with a GitLab Ultimate free trial') + ][series] + end + + def title + [ + s_('InProductMarketing|Give us one minute...'), + s_("InProductMarketing|Security that's integrated into your development lifecycle"), + s_('InProductMarketing|Improve code quality and streamline reviews') + ][series] + end + + def subtitle + [ + s_('InProductMarketing|...and you can get a free trial of GitLab Ultimate'), + s_('InProductMarketing|Try GitLab Ultimate for free'), + s_('InProductMarketing|Better code in less time') + ][series] + end + + def body_line1 + [ + [ + s_("InProductMarketing|GitLab's premium tiers are designed to make you, your team and your application more efficient and more secure with features including but not limited to:"), + list([ + s_('InProductMarketing|%{strong_start}Company wide portfolio management%{strong_end} — including multi-level epics, scoped labels').html_safe % strong_options, + s_('InProductMarketing|%{strong_start}Multiple approval roles%{strong_end} — including code owners and required merge approvals').html_safe % strong_options, + s_('InProductMarketing|%{strong_start}Advanced application security%{strong_end} — including SAST, DAST scanning, FUZZ testing, dependency scanning, license compliance, secrete detection').html_safe % strong_options, + s_('InProductMarketing|%{strong_start}Executive level insights%{strong_end} — including reporting on productivity, tasks by type, days to completion, value stream').html_safe % strong_options + ]) + ].join("\n"), + s_('InProductMarketing|GitLab provides static application security testing (SAST), dynamic application security testing (DAST), container scanning, and dependency scanning to help you deliver secure applications along with license compliance.'), + s_('InProductMarketing|By enabling code owners and required merge approvals the right person will review the right MR. This is a win-win: cleaner code and a more efficient review process.') + ][series] + end + + def body_line2 + [ + s_('InProductMarketing|Start a GitLab Ultimate trial today in less than one minute, no credit card required.'), + s_('InProductMarketing|Get started today with a 30-day GitLab Ultimate trial, no credit card required.'), + s_('InProductMarketing|Code owners and required merge approvals are part of the paid tiers of GitLab. You can start a free 30-day trial of GitLab Ultimate and enable these features in less than 5 minutes with no credit card required.') + ][series] + end + + def cta_text + [ + s_('InProductMarketing|Start a trial'), + s_('InProductMarketing|Beef up your security'), + s_('InProductMarketing|Start your trial now!') + ][series] + end + end + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/verify.rb b/lib/gitlab/email/message/in_product_marketing/verify.rb new file mode 100644 index 00000000000..d563de6c77e --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing/verify.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + class Verify < Base + def subject_line + [ + s_('InProductMarketing|Feel the need for speed?'), + s_('InProductMarketing|3 ways to dive into GitLab CI/CD'), + s_('InProductMarketing|Explore the power of GitLab CI/CD') + ][series] + end + + def tagline + [ + s_('InProductMarketing|Use GitLab CI/CD'), + s_('InProductMarketing|Test, create, deploy'), + s_('InProductMarketing|Are your runners ready?') + ][series] + end + + def title + [ + s_('InProductMarketing|Rapid development, simplified'), + s_('InProductMarketing|Get started with GitLab CI/CD'), + s_('InProductMarketing|Launch GitLab CI/CD in 20 minutes or less') + ][series] + end + + def subtitle + [ + s_('InProductMarketing|How to build and test faster'), + s_('InProductMarketing|Explore the options'), + s_('InProductMarketing|Follow our steps') + ][series] + end + + def body_line1 + [ + s_("InProductMarketing|Tired of wrestling with disparate tool chains, information silos and inefficient processes? GitLab's CI/CD is built on a DevOps platform with source code management, planning, monitoring and more ready to go. Find out %{ci_link}.") % { ci_link: ci_link }, + s_("InProductMarketing|GitLab's CI/CD makes software development easier. Don't believe us? Here are three ways you can take it for a fast (and satisfying) test drive:"), + s_("InProductMarketing|Get going with CI/CD quickly using our %{quick_start_link}. Start with an available runner and then create a CI .yml file – it's really that easy.") % { quick_start_link: quick_start_link } + ][series] + end + + def body_line2 + [ + nil, + list([ + s_('InProductMarketing|Start by %{performance_link}').html_safe % { performance_link: performance_link }, + s_('InProductMarketing|Move on to easily creating a Pages website %{ci_template_link}').html_safe % { ci_template_link: ci_template_link }, + s_('InProductMarketing|And finally %{deploy_link} a Python application.').html_safe % { deploy_link: deploy_link } + ]), + nil + ][series] + end + + def cta_text + [ + s_('InProductMarketing|Get to know GitLab CI/CD'), + s_('InProductMarketing|Try it yourself'), + s_('InProductMarketing|Explore GitLab CI/CD') + ][series] + end + + private + + def ci_link + link(s_('InProductMarketing|how easy it is to get started'), help_page_url('ci/README')) + end + + def quick_start_link + link(s_('InProductMarketing|quick start guide'), help_page_url('ci/quick_start/README')) + end + + def performance_link + link(s_('InProductMarketing|testing browser performance'), help_page_url('user/project/merge_requests/browser_performance_testing')) + end + + def ci_template_link + link(s_('InProductMarketing|using a CI/CD template'), help_page_url('user/project/pages/getting_started/pages_ci_cd_template')) + end + + def deploy_link + link(s_('InProductMarketing|test and deploy'), help_page_url('ci/examples/test-and-deploy-python-application-to-heroku')) + end + end + end + end + end +end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index f5e47b43a9a..71db8ab6067 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -6,6 +6,8 @@ require_dependency 'gitlab/email/handler' module Gitlab module Email class Receiver + include Gitlab::Utils::StrongMemoize + def initialize(raw) @raw = raw end @@ -13,11 +15,7 @@ module Gitlab def execute raise EmptyEmailError if @raw.blank? - mail = build_mail - - ignore_auto_reply!(mail) - - handler = find_handler(mail) + ignore_auto_reply! raise UnknownIncomingEmail unless handler @@ -26,13 +24,33 @@ module Gitlab end end + def mail_metadata + { + mail_uid: mail.message_id, + from_address: mail.from, + to_address: mail.to, + mail_key: mail_key, + references: Array(mail.references), + delivered_to: delivered_to.map(&:value), + envelope_to: envelope_to.map(&:value), + x_envelope_to: x_envelope_to.map(&:value) + } + end + private - def find_handler(mail) - mail_key = extract_mail_key(mail) + def handler + strong_memoize(:handler) { find_handler } + end + + def find_handler Handler.for(mail, mail_key) end + def mail + strong_memoize(:mail) { build_mail } + end + def build_mail Mail::Message.new(@raw) rescue Encoding::UndefinedConversionError, @@ -40,22 +58,24 @@ module Gitlab raise EmailUnparsableError, e end - def extract_mail_key(mail) - key_from_to_header(mail) || key_from_additional_headers(mail) + def mail_key + strong_memoize(:mail_key) do + key_from_to_header || key_from_additional_headers + end end - def key_from_to_header(mail) + def key_from_to_header mail.to.find do |address| key = Gitlab::IncomingEmail.key_from_address(address) break key if key end end - def key_from_additional_headers(mail) - find_key_from_references(mail) || - find_key_from_delivered_to_header(mail) || - find_key_from_envelope_to_header(mail) || - find_key_from_x_envelope_to_header(mail) + def key_from_additional_headers + find_key_from_references || + find_key_from_delivered_to_header || + find_key_from_envelope_to_header || + find_key_from_x_envelope_to_header end def ensure_references_array(references) @@ -71,41 +91,53 @@ module Gitlab end end - def find_key_from_references(mail) + def find_key_from_references ensure_references_array(mail.references).find do |mail_id| key = Gitlab::IncomingEmail.key_from_fallback_message_id(mail_id) break key if key end end - def find_key_from_delivered_to_header(mail) - Array(mail[:delivered_to]).find do |header| + def delivered_to + Array(mail[:delivered_to]) + end + + def envelope_to + Array(mail[:envelope_to]) + end + + def x_envelope_to + Array(mail[:x_envelope_to]) + end + + def find_key_from_delivered_to_header + delivered_to.find do |header| key = Gitlab::IncomingEmail.key_from_address(header.value) break key if key end end - def find_key_from_envelope_to_header(mail) - Array(mail[:envelope_to]).find do |header| + def find_key_from_envelope_to_header + envelope_to.find do |header| key = Gitlab::IncomingEmail.key_from_address(header.value) break key if key end end - def find_key_from_x_envelope_to_header(mail) - Array(mail[:x_envelope_to]).find do |header| + def find_key_from_x_envelope_to_header + x_envelope_to.find do |header| key = Gitlab::IncomingEmail.key_from_address(header.value) break key if key end end - def ignore_auto_reply!(mail) - if auto_submitted?(mail) || auto_replied?(mail) + def ignore_auto_reply! + if auto_submitted? || auto_replied? raise AutoGeneratedEmailError end end - def auto_submitted?(mail) + def auto_submitted? # Mail::Header#[] is case-insensitive auto_submitted = mail.header['Auto-Submitted']&.value @@ -114,7 +146,7 @@ module Gitlab auto_submitted && auto_submitted != 'no' end - def auto_replied?(mail) + def auto_replied? autoreply = mail.header['X-Autoreply']&.value autoreply && autoreply == 'yes' diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index dc44e9d7481..7579f3d8680 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -68,7 +68,7 @@ module Gitlab else object.body.to_s end - rescue + rescue StandardError nil end end diff --git a/lib/gitlab/email/service_desk_receiver.rb b/lib/gitlab/email/service_desk_receiver.rb index 1ee5c10097b..133c4ee4b45 100644 --- a/lib/gitlab/email/service_desk_receiver.rb +++ b/lib/gitlab/email/service_desk_receiver.rb @@ -5,14 +5,19 @@ module Gitlab class ServiceDeskReceiver < Receiver private - def find_handler(mail) - key = service_desk_key(mail) - return unless key + def find_handler + return unless service_desk_key - Gitlab::Email::Handler::ServiceDeskHandler.new(mail, nil, service_desk_key: key) + Gitlab::Email::Handler::ServiceDeskHandler.new(mail, nil, service_desk_key: service_desk_key) end - def service_desk_key(mail) + def service_desk_key + strong_memoize(:service_desk_key) do + find_service_desk_key + end + end + + def find_service_desk_key mail.to.find do |address| key = ::Gitlab::ServiceDeskEmail.key_from_address(address) break key if key diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 7b79de00c66..8ee53d0de28 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -20,7 +20,7 @@ module Gitlab return message if message.valid_encoding? # return message if message type is binary - detect = CharlockHolmes::EncodingDetector.detect(message) + detect = detect_encoding(message) return message.force_encoding("BINARY") if detect_binary?(message, detect) if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD @@ -37,16 +37,30 @@ module Gitlab "--broken encoding: #{encoding}" end + def detect_encoding(data, limit: CharlockHolmes::EncodingDetector::DEFAULT_BINARY_SCAN_LEN, cache_key: nil) + return if data.nil? + + if Feature.enabled?(:cached_encoding_detection, type: :development, default_enabled: :yaml) + return CharlockHolmes::EncodingDetector.new(limit).detect(data) unless cache_key.present? + + Rails.cache.fetch([:detect_binary, CharlockHolmes::VERSION, cache_key], expires_in: 1.week) do + CharlockHolmes::EncodingDetector.new(limit).detect(data) + end + else + CharlockHolmes::EncodingDetector.new(limit).detect(data) + end + end + def detect_binary?(data, detect = nil) - detect ||= CharlockHolmes::EncodingDetector.detect(data) + detect ||= detect_encoding(data) detect && detect[:type] == :binary && detect[:confidence] == 100 end - def detect_libgit2_binary?(data) - # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks - # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), - # which is what we use below to keep a consistent behavior. - detect = CharlockHolmes::EncodingDetector.new(8000).detect(data) + # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks + # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), + # which is what we use below to keep a consistent behavior. + def detect_libgit2_binary?(data, cache_key: nil) + detect = detect_encoding(data, limit: 8000, cache_key: cache_key) detect && detect[:type] == :binary end @@ -54,7 +68,8 @@ module Gitlab message = force_encode_utf8(message) return message if message.valid_encoding? - detect = CharlockHolmes::EncodingDetector.detect(message) + detect = detect_encoding(message) + if detect && detect[:encoding] begin CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') diff --git a/lib/gitlab/encrypted_configuration.rb b/lib/gitlab/encrypted_configuration.rb index fe49af3ab33..6b64281e631 100644 --- a/lib/gitlab/encrypted_configuration.rb +++ b/lib/gitlab/encrypted_configuration.rb @@ -65,7 +65,7 @@ module Gitlab contents = deserialize(read) - raise InvalidConfigError.new unless contents.is_a?(Hash) + raise InvalidConfigError unless contents.is_a?(Hash) @config = contents.deep_symbolize_keys end @@ -115,7 +115,7 @@ module Gitlab end def handle_missing_key! - raise MissingKeyError.new if @key.nil? + raise MissingKeyError if @key.nil? end end end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 47d361fb95c..38ac5d9af74 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -31,9 +31,6 @@ module Gitlab # Sanitize fields based on those sanitized from Rails. config.sanitize_fields = Rails.application.config.filter_parameters.map(&:to_s) - config.processors << ::Gitlab::ErrorTracking::Processor::SidekiqProcessor - config.processors << ::Gitlab::ErrorTracking::Processor::GrpcErrorProcessor - config.processors << ::Gitlab::ErrorTracking::Processor::ContextPayloadProcessor # Sanitize authentication headers config.sanitize_http_headers = %w[Authorization Private-Token] diff --git a/lib/gitlab/error_tracking/context_payload_generator.rb b/lib/gitlab/error_tracking/context_payload_generator.rb index c99283b3d20..3d0a707608f 100644 --- a/lib/gitlab/error_tracking/context_payload_generator.rb +++ b/lib/gitlab/error_tracking/context_payload_generator.rb @@ -49,7 +49,7 @@ module Gitlab # Static tags that are set on application start def extra_tags_from_env Gitlab::Json.parse(ENV.fetch('GITLAB_SENTRY_EXTRA_TAGS', '{}')).to_hash - rescue => e + rescue StandardError => e Gitlab::AppLogger.debug("GITLAB_SENTRY_EXTRA_TAGS could not be parsed as JSON: #{e.class.name}: #{e.message}") {} diff --git a/lib/gitlab/error_tracking/processor/context_payload_processor.rb b/lib/gitlab/error_tracking/processor/context_payload_processor.rb index 758f6aa11d7..9559d6807da 100644 --- a/lib/gitlab/error_tracking/processor/context_payload_processor.rb +++ b/lib/gitlab/error_tracking/processor/context_payload_processor.rb @@ -3,21 +3,12 @@ module Gitlab module ErrorTracking module Processor - class ContextPayloadProcessor < ::Raven::Processor + module ContextPayloadProcessor # This processor is added to inject application context into Sentry # events generated by Sentry built-in integrations. When the # integrations are re-implemented and use Gitlab::ErrorTracking, this # processor should be removed. - def process(payload) - return payload if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) - - context_payload = Gitlab::ErrorTracking::ContextPayloadGenerator.generate(nil, {}) - payload.deep_merge!(context_payload) - end - def self.call(event) - return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) - Gitlab::ErrorTracking::ContextPayloadGenerator.generate(nil, {}).each do |key, value| event.public_send(key).deep_merge!(value) # rubocop:disable GitlabSecurity/PublicSend end diff --git a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb index 419098dbd09..e2a9192806f 100644 --- a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb +++ b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb @@ -3,22 +3,11 @@ module Gitlab module ErrorTracking module Processor - class GrpcErrorProcessor < ::Raven::Processor + module GrpcErrorProcessor DEBUG_ERROR_STRING_REGEX = RE2('(.*) debug_error_string:(.*)') - def process(payload) - return payload if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) - - self.class.process_first_exception_value(payload) - self.class.process_custom_fingerprint(payload) - - payload - end - class << self def call(event) - return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) - process_first_exception_value(event) process_custom_fingerprint(event) @@ -27,8 +16,9 @@ module Gitlab # Sentry can report multiple exceptions in an event. Sanitize # only the first one since that's what is used for grouping. - def process_first_exception_value(event_or_payload) - exceptions = exceptions(event_or_payload) + def process_first_exception_value(event) + # Better in new version, will be event.exception.values + exceptions = event.instance_variable_get(:@interfaces)[:exception]&.values return unless exceptions.is_a?(Array) @@ -36,18 +26,21 @@ module Gitlab return unless valid_exception?(exception) - exception_type, raw_message = type_and_value(exception) + raw_message = exception.value - return unless exception_type&.start_with?('GRPC::') + return unless exception.type&.start_with?('GRPC::') return unless raw_message.present? message, debug_str = split_debug_error_string(raw_message) - set_new_values!(event_or_payload, exception, message, debug_str) + # Worse in new version, no setter! Have to poke at the + # instance variable + exception.value = message if message + event.extra[:grpc_debug_error_string] = debug_str if debug_str end def process_custom_fingerprint(event) - fingerprint = fingerprint(event) + fingerprint = event.fingerprint return event unless custom_grpc_fingerprint?(fingerprint) @@ -71,61 +64,14 @@ module Gitlab [match[1], match[2]] end - # The below methods can be removed once we remove the - # sentry_processors_before_send feature flag, and we can - # assume we always have an Event object - def exceptions(event_or_payload) - case event_or_payload - when Raven::Event - # Better in new version, will be event_or_payload.exception.values - event_or_payload.instance_variable_get(:@interfaces)[:exception]&.values - when Hash - event_or_payload.dig(:exception, :values) - end - end - def valid_exception?(exception) case exception when Raven::SingleExceptionInterface exception&.value - when Hash - true else false end end - - def type_and_value(exception) - case exception - when Raven::SingleExceptionInterface - [exception.type, exception.value] - when Hash - exception.values_at(:type, :value) - end - end - - def set_new_values!(event_or_payload, exception, message, debug_str) - case event_or_payload - when Raven::Event - # Worse in new version, no setter! Have to poke at the - # instance variable - exception.value = message if message - event_or_payload.extra[:grpc_debug_error_string] = debug_str if debug_str - when Hash - exception[:value] = message if message - extra = event_or_payload[:extra] || {} - extra[:grpc_debug_error_string] = debug_str if debug_str - end - end - - def fingerprint(event_or_payload) - case event_or_payload - when Raven::Event - event_or_payload.fingerprint - when Hash - event_or_payload[:fingerprint] - end - end end end end diff --git a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb index 93310745ece..0d2f673d73c 100644 --- a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb +++ b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb @@ -5,7 +5,7 @@ require 'set' module Gitlab module ErrorTracking module Processor - class SidekiqProcessor < ::Raven::Processor + module SidekiqProcessor FILTERED_STRING = '[FILTERED]' class << self @@ -29,7 +29,7 @@ module Gitlab @permitted_arguments_for_worker[klass] ||= begin klass.constantize&.loggable_arguments&.to_set - rescue + rescue StandardError Set.new end end @@ -42,8 +42,6 @@ module Gitlab end def call(event) - return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) - sidekiq = event&.extra&.dig(:sidekiq) return event unless sidekiq @@ -64,29 +62,6 @@ module Gitlab event end end - - def process(value, key = nil) - return value if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) - - sidekiq = value.dig(:extra, :sidekiq) - - return value unless sidekiq - - sidekiq = sidekiq.deep_dup - sidekiq.delete(:jobstr) - - # 'args' in this hash => from Gitlab::ErrorTracking.track_* - # 'args' in :job => from default error handler - job_holder = sidekiq.key?('args') ? sidekiq : sidekiq[:job] - - if job_holder['args'] - job_holder['args'] = self.class.filter_arguments(job_holder['args'], job_holder['class']).to_a - end - - value[:extra][:sidekiq] = sidekiq - - value - end end end end diff --git a/lib/gitlab/etag_caching/router/graphql.rb b/lib/gitlab/etag_caching/router/graphql.rb index f1737f0ce5a..2b8639b9411 100644 --- a/lib/gitlab/etag_caching/router/graphql.rb +++ b/lib/gitlab/etag_caching/router/graphql.rb @@ -12,6 +12,11 @@ module Gitlab %r(\Apipelines/id/\d+\z), 'pipelines_graph', 'continuous_integration' + ], + [ + %r(\Apipelines/sha/\w{7,40}\z), + 'ci_editor', + 'pipeline_authoring' ] ].map(&method(:build_route)).freeze diff --git a/lib/gitlab/etag_caching/router/restful.rb b/lib/gitlab/etag_caching/router/restful.rb index 08c20e30a48..fba4b9e433a 100644 --- a/lib/gitlab/etag_caching/router/restful.rb +++ b/lib/gitlab/etag_caching/router/restful.rb @@ -109,4 +109,4 @@ module Gitlab end end -Gitlab::EtagCaching::Router::Restful.prepend_if_ee('EE::Gitlab::EtagCaching::Router::Restful') +Gitlab::EtagCaching::Router::Restful.prepend_mod_with('Gitlab::EtagCaching::Router::Restful') diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index ef0236f8275..6749bd6ca60 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -113,4 +113,4 @@ module Gitlab end end -Gitlab::ExclusiveLease.prepend_if_ee('EE::Gitlab::ExclusiveLease') +Gitlab::ExclusiveLease.prepend_mod_with('Gitlab::ExclusiveLease') diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 145bb6d7b8f..e4233b8a935 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -34,10 +34,6 @@ module Gitlab module Experimentation EXPERIMENTS = { - invite_members_version_b: { - tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionB', - use_backwards_compatible_subject_index: true - }, invite_members_empty_group_version_a: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersEmptyGroupVersionA', use_backwards_compatible_subject_index: true @@ -55,10 +51,6 @@ module Gitlab trial_during_signup: { tracking_category: 'Growth::Conversion::Experiment::TrialDuringSignup' }, - ci_syntax_templates_b: { - tracking_category: 'Growth::Activation::Experiment::CiSyntaxTemplates', - rollout_strategy: :user - }, invite_members_new_dropdown: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersNewDropdown' }, @@ -154,7 +146,7 @@ module Gitlab elsif subject.respond_to?(:to_s) subject.to_s else - raise ArgumentError.new('Subject must respond to `to_global_id` or `to_s`') + raise ArgumentError, 'Subject must respond to `to_global_id` or `to_s`' end end end diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index 248abfeada5..e53689eb89b 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -19,13 +19,18 @@ module Gitlab end def set_experimentation_subject_id_cookie - return if cookies[:experimentation_subject_id].present? - - cookies.permanent.signed[:experimentation_subject_id] = { - value: SecureRandom.uuid, - secure: ::Gitlab.config.gitlab.https, - httponly: true - } + if Gitlab.dev_env_or_com? + return if cookies[:experimentation_subject_id].present? + + cookies.permanent.signed[:experimentation_subject_id] = { + value: SecureRandom.uuid, + secure: ::Gitlab.config.gitlab.https, + httponly: true + } + else + # We set the cookie before, although experiments are not conducted on self managed instances. + cookies.delete(:experimentation_subject_id) + end end def push_frontend_experiment(experiment_key, subject: nil) diff --git a/lib/gitlab/external_authorization/client.rb b/lib/gitlab/external_authorization/client.rb index 582051010d3..43f7f042592 100644 --- a/lib/gitlab/external_authorization/client.rb +++ b/lib/gitlab/external_authorization/client.rb @@ -24,7 +24,7 @@ module Gitlab ) ::Gitlab::ExternalAuthorization::Response.new(response) rescue *Gitlab::HTTP::HTTP_ERRORS => e - raise ::Gitlab::ExternalAuthorization::RequestFailed.new(e) + raise ::Gitlab::ExternalAuthorization::RequestFailed, e end private diff --git a/lib/gitlab/fake_application_settings.rb b/lib/gitlab/fake_application_settings.rb index 71d2b2396f8..211c0967f89 100644 --- a/lib/gitlab/fake_application_settings.rb +++ b/lib/gitlab/fake_application_settings.rb @@ -33,4 +33,4 @@ module Gitlab end end -Gitlab::FakeApplicationSettings.prepend_if_ee('EE::Gitlab::FakeApplicationSettings') +Gitlab::FakeApplicationSettings.prepend_mod_with('Gitlab::FakeApplicationSettings') diff --git a/lib/gitlab/faraday/error_callback.rb b/lib/gitlab/faraday/error_callback.rb index f99be5b4d04..9b436c3a08e 100644 --- a/lib/gitlab/faraday/error_callback.rb +++ b/lib/gitlab/faraday/error_callback.rb @@ -28,7 +28,7 @@ module Gitlab def call(env) @app.call(env) - rescue => e + rescue StandardError => e @options.callback&.call(env, e) raise diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb index ce1370bab0f..721518c6fcc 100644 --- a/lib/gitlab/favicon.rb +++ b/lib/gitlab/favicon.rb @@ -61,4 +61,4 @@ module Gitlab end end -Gitlab::Favicon.prepend_if_ee('EE::Gitlab::Favicon') +Gitlab::Favicon.prepend_mod_with('Gitlab::Favicon') diff --git a/lib/gitlab/file_hook.rb b/lib/gitlab/file_hook.rb index 55eba2858fb..e398a3f9585 100644 --- a/lib/gitlab/file_hook.rb +++ b/lib/gitlab/file_hook.rb @@ -28,7 +28,7 @@ module Gitlab exit_status = result.status&.exitstatus [exit_status == 0, result.stderr] - rescue => e + rescue StandardError => e [false, e.message] end end diff --git a/lib/gitlab/fogbugz_import/repository.rb b/lib/gitlab/fogbugz_import/repository.rb index b958dcf6cbf..4a5152021b4 100644 --- a/lib/gitlab/fogbugz_import/repository.rb +++ b/lib/gitlab/fogbugz_import/repository.rb @@ -26,7 +26,7 @@ module Gitlab end def path - safe_name.gsub(/[\s]/, '_') + safe_name.gsub(/\s/, '_') end end end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 5d91eb605e8..1c8e55ecf50 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -110,8 +110,8 @@ module Gitlab end end - def binary?(data) - EncodingHelper.detect_libgit2_binary?(data) + def binary?(data, cache_key: nil) + EncodingHelper.detect_libgit2_binary?(data, cache_key: cache_key) end def size_could_be_lfs?(size) diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 9447cfa0fb6..fbe52db9c0b 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -28,6 +28,10 @@ module Gitlab def state active? ? :active : :stale end + + def cache_key + "branch:" + Digest::SHA1.hexdigest([name, target, dereferenced_target&.sha].join(':')) + end end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 51baed32935..a863b952390 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -263,7 +263,7 @@ module Gitlab def has_zero_stats? stats.total == 0 - rescue + rescue StandardError true end diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index 26e82643a4c..751184b23df 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -20,9 +20,9 @@ module Gitlab gitaly_conflicts_client(@target_repository).list_conflict_files.to_a end rescue GRPC::FailedPrecondition => e - raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing.new(e.message) + raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing, e.message rescue GRPC::BadStatus => e - raise Gitlab::Git::CommandError.new(e) + raise Gitlab::Git::CommandError, e end def resolve_conflicts(source_repository, resolution, source_branch:, target_branch:) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 3361cee733b..102fe60f2cb 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -89,9 +89,9 @@ module Gitlab def root_ref gitaly_ref_client.default_branch_name rescue GRPC::NotFound => e - raise NoRepository.new(e.message) + raise NoRepository, e.message rescue GRPC::Unknown => e - raise Gitlab::Git::CommandError.new(e.message) + raise Gitlab::Git::CommandError, e.message end def exists? @@ -348,7 +348,7 @@ module Gitlab limit = options[:limit] if limit == 0 || !limit.is_a?(Integer) - raise ArgumentError.new("invalid Repository#log limit: #{limit.inspect}") + raise ArgumentError, "invalid Repository#log limit: #{limit.inspect}" end wrapped_gitaly_errors do @@ -414,7 +414,7 @@ module Gitlab end end rescue ArgumentError => e - raise Gitlab::Git::Repository::GitError.new(e) + raise Gitlab::Git::Repository::GitError, e end # Returns the SHA of the most recent common ancestor of +from+ and +to+ @@ -700,11 +700,11 @@ module Gitlab end end - def find_remote_root_ref(remote_name) - return unless remote_name.present? + def find_remote_root_ref(remote_name, remote_url, authorization = nil) + return unless remote_name.present? && remote_url.present? wrapped_gitaly_errors do - gitaly_remote_client.find_remote_root_ref(remote_name) + gitaly_remote_client.find_remote_root_ref(remote_name, remote_url, authorization) end end @@ -836,7 +836,7 @@ module Gitlab def fsck msg, status = gitaly_repository_client.fsck - raise GitError.new("Could not fsck repository: #{msg}") unless status == 0 + raise GitError, "Could not fsck repository: #{msg}" unless status == 0 end def create_from_bundle(bundle_path) diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb index 8679d977773..ea10b4e7cd8 100644 --- a/lib/gitlab/git/rugged_impl/repository.rb +++ b/lib/gitlab/git/rugged_impl/repository.rb @@ -31,7 +31,7 @@ module Gitlab def rugged @rugged ||= ::Rugged::Repository.new(path, alternates: alternate_object_directories) rescue ::Rugged::RepositoryError, ::Rugged::OSError - raise ::Gitlab::Git::Repository::NoRepository.new('no repository for such path') + raise ::Gitlab::Git::Repository::NoRepository, 'no repository for such path' end def cleanup diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 75d6b949874..5616b61de07 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -73,12 +73,6 @@ module Gitlab end end - def delete_page(page_path, commit_details) - wrapped_gitaly_errors do - gitaly_delete_page(page_path, commit_details) - end - end - def update_page(page_path, title, format, content, commit_details) wrapped_gitaly_errors do gitaly_update_page(page_path, title, format, content, commit_details) @@ -102,22 +96,6 @@ module Gitlab end end - # options: - # :page - The Integer page number. - # :per_page - The number of items per page. - # :limit - Total number of items to return. - def page_versions(page_path, options = {}) - versions = wrapped_gitaly_errors do - gitaly_wiki_client.page_versions(page_path, options) - end - - # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20 - # per page, but also fetches 20 if `limit` or `per_page` < 20. - # Slicing returns an array with the expected number of items. - slice_bound = options[:limit] || options[:per_page] || DEFAULT_PAGINATION - versions[0..slice_bound] - end - def count_page_versions(page_path) @repository.count_commits(ref: 'HEAD', path: page_path) end @@ -140,10 +118,6 @@ module Gitlab gitaly_wiki_client.update_page(page_path, title, format, content, commit_details) end - def gitaly_delete_page(page_path, commit_details) - gitaly_wiki_client.delete_page(page_path, commit_details) - end - def gitaly_find_page(title:, version: nil, dir: nil) return unless title.present? diff --git a/lib/gitlab/git/wraps_gitaly_errors.rb b/lib/gitlab/git/wraps_gitaly_errors.rb index 2009683d32c..1d34f3c8eb2 100644 --- a/lib/gitlab/git/wraps_gitaly_errors.rb +++ b/lib/gitlab/git/wraps_gitaly_errors.rb @@ -6,13 +6,13 @@ module Gitlab def wrapped_gitaly_errors(&block) yield block rescue GRPC::NotFound => e - raise Gitlab::Git::Repository::NoRepository.new(e) + raise Gitlab::Git::Repository::NoRepository, e rescue GRPC::InvalidArgument => e - raise ArgumentError.new(e) + raise ArgumentError, e rescue GRPC::DeadlineExceeded => e - raise Gitlab::Git::CommandTimedOut.new(e) + raise Gitlab::Git::CommandTimedOut, e rescue GRPC::BadStatus => e - raise Gitlab::Git::CommandError.new(e) + raise Gitlab::Git::CommandError, e end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 31e4755192e..b5e7220889e 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -22,7 +22,7 @@ module Gitlab auth_download: 'You are not allowed to download code.', deploy_key_upload: 'This deploy key does not have write access to this project.', no_repo: 'A repository for this project does not exist yet.', - project_not_found: 'The project you were looking for could not be found.', + project_not_found: "The project you were looking for could not be found or you don't have permission to view it.", command_not_allowed: "The command you're trying to execute is not allowed.", upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', @@ -538,4 +538,4 @@ module Gitlab end end -Gitlab::GitAccess.prepend_if_ee('EE::Gitlab::GitAccess') +Gitlab::GitAccess.prepend_mod_with('Gitlab::GitAccess') diff --git a/lib/gitlab/git_access_design.rb b/lib/gitlab/git_access_design.rb index 6bea9fe53b3..bf89c01305a 100644 --- a/lib/gitlab/git_access_design.rb +++ b/lib/gitlab/git_access_design.rb @@ -32,4 +32,4 @@ module Gitlab end end -Gitlab::GitAccessDesign.prepend_if_ee('EE::Gitlab::GitAccessDesign') +Gitlab::GitAccessDesign.prepend_mod_with('Gitlab::GitAccessDesign') diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index 88a75f72840..9a431dc7088 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -141,4 +141,4 @@ module Gitlab end end -Gitlab::GitAccessSnippet.prepend_if_ee('EE::Gitlab::GitAccessSnippet') +Gitlab::GitAccessSnippet.prepend_mod_with('Gitlab::GitAccessSnippet') diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 3011b794b8f..0963eb6b72a 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -48,4 +48,4 @@ module Gitlab end end -Gitlab::GitAccessWiki.prepend_if_ee('EE::Gitlab::GitAccessWiki') +Gitlab::GitAccessWiki.prepend_mod_with('Gitlab::GitAccessWiki') diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 19a473e4785..affd3986381 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -115,7 +115,7 @@ module Gitlab # necessary graph walk to detect only new LFS pointers and instead scan # through all quarantined objects. git_env = ::Gitlab::Git::HookEnv.all(@gitaly_repo.gl_repository) - if Feature.enabled?(:lfs_integrity_inspect_quarantined_objects, @project, default_enabled: :yaml) && git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].present? + if git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].present? repository = @gitaly_repo.dup repository.git_alternate_object_directories = Google::Protobuf::RepeatedField.new(:string) diff --git a/lib/gitlab/gitaly_client/blobs_stitcher.rb b/lib/gitlab/gitaly_client/blobs_stitcher.rb index f860d8ce517..2f6d146b5c4 100644 --- a/lib/gitlab/gitaly_client/blobs_stitcher.rb +++ b/lib/gitlab/gitaly_client/blobs_stitcher.rb @@ -19,9 +19,9 @@ module Gitlab yield new_blob(current_blob_data) if current_blob_data current_blob_data = msg.to_h.slice(:oid, :path, :size, :revision, :mode) - current_blob_data[:data] = msg.data.dup + current_blob_data[:data_parts] = [msg.data] else - current_blob_data[:data] << msg.data + current_blob_data[:data_parts] << msg.data end end @@ -31,6 +31,8 @@ module Gitlab private def new_blob(blob_data) + data = blob_data[:data_parts].join + Gitlab::Git::Blob.new( id: blob_data[:oid], mode: blob_data[:mode].to_s(8), @@ -38,8 +40,8 @@ module Gitlab path: blob_data[:path], size: blob_data[:size], commit_id: blob_data[:revision], - data: blob_data[:data], - binary: Gitlab::Git::Blob.binary?(blob_data[:data]) + data: data, + binary: Gitlab::Git::Blob.binary?(data, cache_key: blob_data[:oid]) ) end end diff --git a/lib/gitlab/gitaly_client/call.rb b/lib/gitlab/gitaly_client/call.rb index 4bb184bee2f..3fe3702cfe1 100644 --- a/lib/gitlab/gitaly_client/call.rb +++ b/lib/gitlab/gitaly_client/call.rb @@ -30,7 +30,7 @@ module Gitlab store_timings response end - rescue => err + rescue StandardError => err store_timings raise err end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 5ce1b1f0c87..fd794acb4dd 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -59,7 +59,7 @@ module Gitlab :user_create_branch, request, timeout: GitalyClient.long_timeout) if response.pre_receive_error.present? - raise Gitlab::Git::PreReceiveError.new(response.pre_receive_error) + raise Gitlab::Git::PreReceiveError, response.pre_receive_error end branch = response.branch @@ -159,7 +159,7 @@ module Gitlab branch_update = second_response.branch_update return if branch_update.nil? - raise Gitlab::Git::CommitError.new('failed to apply merge to branch') unless branch_update.commit_id.present? + raise Gitlab::Git::CommitError, 'failed to apply merge to branch' unless branch_update.commit_id.present? Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) ensure diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 97b6813c080..ac2db99ee01 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -292,7 +292,7 @@ module Gitlab end def invalid_ref!(message) - raise Gitlab::Git::Repository::InvalidRef.new(message) + raise Gitlab::Git::Repository::InvalidRef, message end end end diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index 06aaf460751..04dd394a2bd 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -43,11 +43,20 @@ module Gitlab GitalyClient.call(@storage, :remote_service, :remove_remote, request, timeout: GitalyClient.long_timeout).result end - def find_remote_root_ref(remote_name) - request = Gitaly::FindRemoteRootRefRequest.new( - repository: @gitaly_repo, - remote: remote_name - ) + # The remote_name parameter is deprecated and will be removed soon. + def find_remote_root_ref(remote_name, remote_url, authorization) + request = if Feature.enabled?(:find_remote_root_refs_inmemory, default_enabled: :yaml) + Gitaly::FindRemoteRootRefRequest.new( + repository: @gitaly_repo, + remote_url: remote_url, + http_authorization_header: authorization + ) + else + Gitaly::FindRemoteRootRefRequest.new( + repository: @gitaly_repo, + remote: remote_name + ) + end response = GitalyClient.call(@storage, :remote_service, :find_remote_root_ref, request, timeout: GitalyClient.medium_timeout) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index a93f4071efc..d2dbd456180 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -319,7 +319,7 @@ module Gitlab response = GitalyClient.call(@storage, :repository_service, :calculate_checksum, request, timeout: GitalyClient.fast_timeout) response.checksum.presence rescue GRPC::DataLoss => e - raise Gitlab::Git::Repository::InvalidRepository.new(e) + raise Gitlab::Git::Repository::InvalidRepository, e end def raw_changes_between(from, to) diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index dd9e3d5d28b..f66dc3010ea 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -34,7 +34,7 @@ module Gitlab return false if rugged_enabled? !temporarily_allowed?(ALLOW_KEY) - rescue + rescue StandardError false # Err on the side of caution, don't break gitlab for people end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index fecc2b7023d..3613cd01122 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -64,16 +64,6 @@ module Gitlab GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum, timeout: GitalyClient.medium_timeout) end - def delete_page(page_path, commit_details) - request = Gitaly::WikiDeletePageRequest.new( - repository: @gitaly_repo, - page_path: encode_binary(page_path), - commit_details: gitaly_commit_details(commit_details) - ) - - GitalyClient.call(@repository.storage, :wiki_service, :wiki_delete_page, request, timeout: GitalyClient.medium_timeout) - end - def find_page(title:, version: nil, dir: nil) request = Gitaly::WikiFindPageRequest.new( repository: @gitaly_repo, @@ -129,30 +119,6 @@ module Gitlab pages end - # options: - # :page - The Integer page number. - # :per_page - The number of items per page. - # :limit - Total number of items to return. - def page_versions(page_path, options) - request = Gitaly::WikiGetPageVersionsRequest.new( - repository: @gitaly_repo, - page_path: encode_binary(page_path), - page: options[:page] || 1, - per_page: options[:per_page] || Gitlab::Git::Wiki::DEFAULT_PAGINATION - ) - - stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_page_versions, request, timeout: GitalyClient.medium_timeout) - - versions = [] - stream.each do |message| - message.versions.each do |version| - versions << new_wiki_page_version(version) - end - end - - versions - end - private # If a block is given and the yielded value is truthy, iteration will be diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 328f1f742c5..138716b1b53 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -70,7 +70,7 @@ module Gitlab end def pull_request_reviews(repo_name, iid) - with_rate_limit { octokit.pull_request_reviews(repo_name, iid) } + each_object(:pull_request_reviews, repo_name, iid) end # Returns the details of a GitHub repository. diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index 53b17f77ccd..d2f5af63621 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -21,8 +21,7 @@ module Gitlab author_id, author_found = user_finder.author_id_for(note) - note_body = - MarkdownText.format(note.note, note.author, author_found) + note_body = MarkdownText.format(note.note, note.author, author_found) attributes = { noteable_type: 'MergeRequest', diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 41f179d275b..ae9996d81ef 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -21,8 +21,7 @@ module Gitlab author_id, author_found = user_finder.author_id_for(note) - note_body = - MarkdownText.format(note.note, note.author, author_found) + note_body = MarkdownText.format(note.note, note.author, author_found) attributes = { noteable_type: note.noteable_type, diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index f09e0bd9806..3c17ea1195e 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -44,8 +44,7 @@ module Gitlab def create_merge_request author_id, author_found = user_finder.author_id_for(pull_request) - description = MarkdownText - .format(pull_request.description, pull_request.author, author_found) + description = MarkdownText.format(pull_request.description, pull_request.author, author_found) attributes = { iid: pull_request.iid, diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb index 9f495913897..f476ee13392 100644 --- a/lib/gitlab/github_import/importer/pull_request_review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_review_importer.rb @@ -36,12 +36,12 @@ module Gitlab def add_complementary_review_note!(author_id) return if review.note.empty? && !review.approval? - note = "*Created by %{login}*\n\n%{note}" % { - note: review_note_content, - login: review.author.login - } + note_body = MarkdownText.format( + review_note_content, + review.author + ) - add_note!(author_id, note) + add_note!(author_id, note_body) end def review_note_content diff --git a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb index 466288fde4c..94472cd341e 100644 --- a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb @@ -22,14 +22,18 @@ module Gitlab :pull_requests_merged_by end - def id_for_already_imported_cache(pr) - pr.number + def id_for_already_imported_cache(merge_request) + merge_request.id end def each_object_to_import project.merge_requests.with_state(:merged).find_each do |merge_request| + next if already_imported?(merge_request) + pull_request = client.pull_request(project.import_source, merge_request.iid) yield(pull_request) + + mark_as_imported(merge_request) end end end diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb index 6d1b588f0e0..827027203ff 100644 --- a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb @@ -22,17 +22,22 @@ module Gitlab :pull_request_reviews end - def id_for_already_imported_cache(review) - review.github_id + def id_for_already_imported_cache(merge_request) + merge_request.id end def each_object_to_import project.merge_requests.find_each do |merge_request| - reviews = client.pull_request_reviews(project.import_source, merge_request.iid) - reviews.each do |review| - review.merge_request_id = merge_request.id - yield(review) - end + next if already_imported?(merge_request) + + client + .pull_request_reviews(project.import_source, merge_request.iid) + .each do |review| + review.merge_request_id = merge_request.id + yield(review) + end + + mark_as_imported(merge_request) end end end diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb index b25c4f7becf..e5f4dabe42d 100644 --- a/lib/gitlab/github_import/markdown_text.rb +++ b/lib/gitlab/github_import/markdown_text.rb @@ -3,7 +3,7 @@ module Gitlab module GithubImport class MarkdownText - attr_reader :text, :author, :exists + include Gitlab::EncodingHelper def self.format(*args) new(*args).to_s @@ -19,10 +19,19 @@ module Gitlab end def to_s - if exists - text - else + # Gitlab::EncodingHelper#clean remove `null` chars from the string + clean(format) + end + + private + + attr_reader :text, :author, :exists + + def format + if author&.login.present? && !exists "*Created by: #{author.login}*\n\n#{text}" + else + text end end end diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb index 1b4750da868..2429fa4de1d 100644 --- a/lib/gitlab/github_import/parallel_importer.rb +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -40,4 +40,4 @@ module Gitlab end end -Gitlab::GithubImport::ParallelImporter.prepend_if_ee('::EE::Gitlab::GithubImport::ParallelImporter') +Gitlab::GithubImport::ParallelImporter.prepend_mod_with('Gitlab::GithubImport::ParallelImporter') diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 51859010ec3..92f9e8a646d 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -48,7 +48,7 @@ module Gitlab info(project.id, message: "importer finished") retval - rescue => e + rescue StandardError => e error(project.id, e) raise e diff --git a/lib/gitlab/github_import/representation/issue.rb b/lib/gitlab/github_import/representation/issue.rb index f3071b3e2b3..0e04b5ad57f 100644 --- a/lib/gitlab/github_import/representation/issue.rb +++ b/lib/gitlab/github_import/representation/issue.rb @@ -25,6 +25,7 @@ module Gitlab hash = { iid: issue.number, + github_id: issue.number, title: issue.title, description: issue.body, milestone_number: issue.milestone&.number, diff --git a/lib/gitlab/github_import/representation/lfs_object.rb b/lib/gitlab/github_import/representation/lfs_object.rb index a4606173f49..41723759645 100644 --- a/lib/gitlab/github_import/representation/lfs_object.rb +++ b/lib/gitlab/github_import/representation/lfs_object.rb @@ -13,7 +13,12 @@ module Gitlab # Builds a lfs_object def self.from_api_response(lfs_object) - new({ oid: lfs_object.oid, link: lfs_object.link, size: lfs_object.size }) + new( + oid: lfs_object.oid, + link: lfs_object.link, + size: lfs_object.size, + github_id: lfs_object.oid + ) end # Builds a new lfs_object using a Hash that was built from a JSON payload. diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb index be192762e05..e4f54fcc833 100644 --- a/lib/gitlab/github_import/representation/pull_request.rb +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -25,6 +25,7 @@ module Gitlab hash = { iid: pr.number, + github_id: pr.number, title: pr.title, description: pr.body, source_branch: pr.head.ref, diff --git a/lib/gitlab/github_import/representation/user.rb b/lib/gitlab/github_import/representation/user.rb index e00dcfca33d..d97b90b6291 100644 --- a/lib/gitlab/github_import/representation/user.rb +++ b/lib/gitlab/github_import/representation/user.rb @@ -15,7 +15,11 @@ module Gitlab # # user - An instance of `Sawyer::Resource` containing the user details. def self.from_api_response(user) - new(id: user.id, login: user.login) + new( + id: user.id, + github_id: user.id, + login: user.login + ) end # Builds a user using a Hash that was built from a JSON payload. diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 34d1231b9a5..8d584415202 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -63,7 +63,7 @@ module Gitlab # # user - An instance of `Gitlab::GithubImport::Representation::User`. def user_id_for(user) - find(user.id, user.login) + find(user.id, user.login) if user.present? end # Returns the GitLab ID for the given GitHub ID or username. diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb index 4b1f4fcc2a2..05278b2dd35 100644 --- a/lib/gitlab/gl_repository/repo_type.rb +++ b/lib/gitlab/gl_repository/repo_type.rb @@ -81,4 +81,4 @@ module Gitlab end end -Gitlab::GlRepository::RepoType.prepend_if_ee('EE::Gitlab::GlRepository::RepoType') +Gitlab::GlRepository::RepoType.prepend_mod_with('Gitlab::GlRepository::RepoType') diff --git a/lib/gitlab/golang.rb b/lib/gitlab/golang.rb index 31b7a198b92..1b625a3a514 100644 --- a/lib/gitlab/golang.rb +++ b/lib/gitlab/golang.rb @@ -69,13 +69,13 @@ module Gitlab # Error messages are based on the responses of proxy.golang.org # Verify that the SHA fragment references a commit - raise ArgumentError.new 'invalid pseudo-version: unknown commit' unless commit + raise ArgumentError, 'invalid pseudo-version: unknown commit' unless commit # Require the SHA fragment to be 12 characters long - raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless version.commit_id.length == 12 + raise ArgumentError, 'invalid pseudo-version: revision is shorter than canonical' unless version.commit_id.length == 12 # Require the timestamp to match that of the commit - raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == version.timestamp + raise ArgumentError, 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == version.timestamp commit end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 08c17058fcb..1fd210c521e 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -80,4 +80,4 @@ module Gitlab end end -Gitlab::GonHelper.prepend_if_ee('EE::Gitlab::GonHelper') +Gitlab::GonHelper.prepend_mod_with('Gitlab::GonHelper') diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index b1494cf8cf2..3d9b06855ff 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -133,7 +133,7 @@ module Gitlab Retriable.retriable(max_elapsed_time: cleanup_time, base_interval: 0.1, tries: 15) do FileUtils.remove_entry(tmp_dir) if File.exist?(tmp_dir) end - rescue => e + rescue StandardError => e raise CleanupError, e end diff --git a/lib/gitlab/grape_logging/loggers/route_logger.rb b/lib/gitlab/grape_logging/loggers/route_logger.rb index f3146b4dfd9..7cbd2340e85 100644 --- a/lib/gitlab/grape_logging/loggers/route_logger.rb +++ b/lib/gitlab/grape_logging/loggers/route_logger.rb @@ -13,7 +13,7 @@ module Gitlab return {} unless route { route: route } - rescue + rescue StandardError # endpoint.route calls env[Grape::Env::GRAPE_ROUTING_ARGS][:route_info] # but env[Grape::Env::GRAPE_ROUTING_ARGS] is nil in the case of a 405 response # so we're rescuing exceptions and bailing out diff --git a/lib/gitlab/graphql/deprecation.rb b/lib/gitlab/graphql/deprecation.rb index e0176e2d6e0..8b73eeb4e52 100644 --- a/lib/gitlab/graphql/deprecation.rb +++ b/lib/gitlab/graphql/deprecation.rb @@ -41,7 +41,7 @@ module Gitlab parts = [ "#{deprecated_in(format: :markdown)}.", reason_text, - replacement.then { |r| "Use: `#{r}`." if r } + replacement.then { |r| "Use: [`#{r}`](##{r.downcase.tr('.', '')})." if r } ].compact case context diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb index f4173e26224..b598b605141 100644 --- a/lib/gitlab/graphql/docs/helper.rb +++ b/lib/gitlab/graphql/docs/helper.rb @@ -5,11 +5,52 @@ return if Rails.env.production? module Gitlab module Graphql module Docs + # We assume a few things about the schema. We use the graphql-ruby gem, which enforces: + # - All mutations have a single input field named 'input' + # - All mutations have a payload type, named after themselves + # - All mutations have an input type, named after themselves + # If these things change, then some of this code will break. Such places + # are guarded with an assertion that our assumptions are not violated. + ViolatedAssumption = Class.new(StandardError) + + SUGGESTED_ACTION = <<~MSG + We expect it to be impossible to violate our assumptions about + how mutation arguments work. + + If that is not the case, then something has probably changed in the + way we generate our schema, perhaps in the library we use: graphql-ruby + + Please ask for help in the #f_graphql or #backend channels. + MSG + + CONNECTION_ARGS = %w[after before first last].to_set + + FIELD_HEADER = <<~MD + #### Fields + + | Name | Type | Description | + | ---- | ---- | ----------- | + MD + + ARG_HEADER = <<~MD + # Arguments + + | Name | Type | Description | + | ---- | ---- | ----------- | + MD + + CONNECTION_NOTE = <<~MD + This field returns a [connection](#connections). It accepts the + four standard [pagination arguments](#connection-pagination-arguments): + `before: String`, `after: String`, `first: Int`, `last: Int`. + MD + # Helper with functions to be used by HAML templates # This includes graphql-docs gem helpers class. # You can check the included module on: https://github.com/gjtorikian/graphql-docs/blob/v1.6.0/lib/graphql-docs/helpers.rb module Helper include GraphQLDocs::Helpers + include Gitlab::Utils::StrongMemoize def auto_generated_comment <<-MD.strip_heredoc @@ -30,44 +71,52 @@ module Gitlab # Template methods: # Methods that return chunks of Markdown for insertion into the document - def render_name_and_description(object, owner: nil, level: 3) - content = [] + def render_full_field(field, heading_level: 3, owner: nil) + conn = connection?(field) + args = field[:arguments].reject { |arg| conn && CONNECTION_ARGS.include?(arg[:name]) } + arg_owner = [owner, field[:name]] + + chunks = [ + render_name_and_description(field, level: heading_level, owner: owner), + render_return_type(field), + render_input_type(field), + render_connection_note(field), + render_argument_table(heading_level, args, arg_owner), + render_return_fields(field, owner: owner) + ] + + join(:block, chunks) + end - content << "#{'#' * level} `#{object[:name]}`" + def render_argument_table(level, args, owner) + arg_header = ('#' * level) + ARG_HEADER + render_field_table(arg_header, args, owner) + end - if object[:description].present? - desc = object[:description].strip - desc += '.' unless desc.ends_with?('.') - end + def render_name_and_description(object, owner: nil, level: 3) + content = [] - if object[:is_deprecated] - owner = Array.wrap(owner) - deprecation = schema_deprecation(owner, object[:name]) - content << (deprecation&.original_description || desc) - content << render_deprecation(object, owner, :block) - else - content << desc - end + heading = '#' * level + name = [owner, object[:name]].compact.join('.') - content.compact.join("\n\n") - end + content << "#{heading} `#{name}`" + content << render_description(object, owner, :block) - def render_return_type(query) - "Returns #{render_field_type(query[:type])}.\n" + join(:block, content) end - def sorted_by_name(objects) - return [] unless objects.present? + def render_object_fields(fields, owner:, level_bump: 0) + return if fields.blank? - objects.sort_by { |o| o[:name] } - end + (with_args, no_args) = fields.partition { |f| args?(f) } + type_name = owner[:name] if owner + header_prefix = '#' * level_bump + sections = [ + render_simple_fields(no_args, type_name, header_prefix), + render_fields_with_arguments(with_args, type_name, header_prefix) + ] - def render_field(field, owner) - render_row( - render_name(field, owner), - render_field_type(field[:type]), - render_description(field, owner, :inline) - ) + join(:block, sections) end def render_enum_value(enum, value) @@ -82,104 +131,302 @@ module Gitlab # Methods that return parts of the schema, or related information: - # We are ignoring connections and built in types for now, - # they should be added when queries are generated. - def objects - object_types = graphql_object_types.select do |object_type| - !object_type[:name]["__"] - end + def connection_object_types + objects.select { |t| t[:is_edge] || t[:is_connection] } + end + + def object_types + objects.reject { |t| t[:is_edge] || t[:is_connection] || t[:is_payload] } + end + + def interfaces + graphql_interface_types.map { |t| t.merge(fields: t[:fields] + t[:connections]) } + end - object_types.each do |type| - type[:fields] += type[:connections] + def fields_of(type_name) + graphql_operation_types + .find { |type| type[:name] == type_name } + .values_at(:fields, :connections) + .flatten + .then { |fields| sorted_by_name(fields) } + end + + # Place the arguments of the input types on the mutation itself. + # see: `#input_types` - this method must not call `#input_types` to avoid mutual recursion + def mutations + @mutations ||= sorted_by_name(graphql_mutation_types).map do |t| + inputs = t[:input_fields] + input = inputs.first + name = t[:name] + + assert!(inputs.one?, "Expected exactly 1 input field named #{name}. Found #{inputs.count} instead.") + assert!(input[:name] == 'input', "Expected the input of #{name} to be named 'input'") + + input_type_name = input[:type][:name] + input_type = graphql_input_object_types.find { |t| t[:name] == input_type_name } + assert!(input_type.present?, "Cannot find #{input_type_name} for #{name}.input") + + arguments = input_type[:input_fields] + seen_type!(input_type_name) + t.merge(arguments: arguments) end end - def queries - graphql_operation_types.find { |type| type[:name] == 'Query' }.to_h.values_at(:fields, :connections).flatten + # We assume that the mutations have been processed first, marking their + # inputs as `seen_type?` + def input_types + mutations # ensure that mutations have seen their inputs first + graphql_input_object_types.reject { |t| seen_type?(t[:name]) } end - # We ignore the built-in enum types. + # We ignore the built-in enum types, and sort values by name def enums - graphql_enum_types.select do |enum_type| - !enum_type[:name].in?(%w[__DirectiveLocation __TypeKind]) - end + graphql_enum_types + .reject { |type| type[:values].empty? } + .reject { |enum_type| enum_type[:name].start_with?('__') } + .map { |type| type.merge(values: sorted_by_name(type[:values])) } end private # DO NOT CALL THESE METHODS IN TEMPLATES # Template methods + def render_return_type(query) + return unless query[:type] # for example, mutations + + "Returns #{render_field_type(query[:type])}." + end + + def render_simple_fields(fields, type_name, header_prefix) + render_field_table(header_prefix + FIELD_HEADER, fields, type_name) + end + + def render_fields_with_arguments(fields, type_name, header_prefix) + return if fields.empty? + + level = 5 + header_prefix.length + sections = sorted_by_name(fields).map do |f| + render_full_field(f, heading_level: level, owner: type_name) + end + + <<~MD.chomp + #{header_prefix}#### Fields with arguments + + #{join(:block, sections)} + MD + end + + def render_field_table(header, fields, owner) + return if fields.empty? + + fields = sorted_by_name(fields) + header + join(:table, fields.map { |f| render_field(f, owner) }) + end + + def render_field(field, owner) + render_row( + render_name(field, owner), + render_field_type(field[:type]), + render_description(field, owner, :inline) + ) + end + + def render_return_fields(mutation, owner:) + fields = mutation[:return_fields] + return if fields.blank? + + name = owner.to_s + mutation[:name] + render_object_fields(fields, owner: { name: name }) + end + + def render_connection_note(field) + return unless connection?(field) + + CONNECTION_NOTE.chomp + end + def render_row(*values) "| #{values.map { |val| val.to_s.squish }.join(' | ')} |" end def render_name(object, owner = nil) rendered_name = "`#{object[:name]}`" - rendered_name += ' **{warning-solid}**' if object[:is_deprecated] - rendered_name + rendered_name += ' **{warning-solid}**' if deprecated?(object, owner) + + return rendered_name unless owner + + owner = Array.wrap(owner).join('') + id = (owner + object[:name]).downcase + + %(<a id="#{id}"></a>) + rendered_name end # Returns the object description. If the object has been deprecated, # the deprecation reason will be returned in place of the description. def render_description(object, owner = nil, context = :block) - owner = Array.wrap(owner) - return render_deprecation(object, owner, context) if object[:is_deprecated] - return if object[:description].blank? + if deprecated?(object, owner) + render_deprecation(object, owner, context) + else + render_description_of(object, owner, context) + end + end + + def deprecated?(object, owner) + return true if object[:is_deprecated] # only populated for fields, not arguments! + + key = [*Array.wrap(owner), object[:name]].join('.') + deprecations.key?(key) + end + + def render_description_of(object, owner, context = nil) + desc = if object[:is_edge] + base = object[:name].chomp('Edge') + "The edge type for [`#{base}`](##{base.downcase})." + elsif object[:is_connection] + base = object[:name].chomp('Connection') + "The connection type for [`#{base}`](##{base.downcase})." + else + object[:description]&.strip + end + + return if desc.blank? - desc = object[:description].strip desc += '.' unless desc.ends_with?('.') + see = doc_reference(object, owner) + desc += " #{see}" if see + desc += " (see [Connections](#connections))" if connection?(object) && context != :block desc end + def doc_reference(object, owner) + field = schema_field(owner, object[:name]) if owner + return unless field + + ref = field.try(:doc_reference) + return if ref.blank? + + parts = ref.to_a.map do |(title, url)| + "[#{title.strip}](#{url.strip})" + end + + "See #{parts.join(', ')}." + end + def render_deprecation(object, owner, context) + buff = [] deprecation = schema_deprecation(owner, object[:name]) - return deprecation.markdown(context: context) if deprecation - reason = object[:deprecation_reason] || 'Use of this is deprecated.' - "**Deprecated:** #{reason}" + buff << (deprecation&.original_description || render_description_of(object, owner)) if context == :block + buff << if deprecation + deprecation.markdown(context: context) + else + "**Deprecated:** #{object[:deprecation_reason]}" + end + + join(context, buff) end def render_field_type(type) "[`#{type[:info]}`](##{type[:name].downcase})" end + def join(context, chunks) + chunks.compact! + return if chunks.blank? + + case context + when :block + chunks.join("\n\n") + when :inline + chunks.join(" ").squish.presence + when :table + chunks.join("\n") + end + end + # Queries + def sorted_by_name(objects) + return [] unless objects.present? + + objects.sort_by { |o| o[:name] } + end + + def connection?(field) + type_name = field.dig(:type, :name) + type_name.present? && type_name.ends_with?('Connection') + end + + # We are ignoring connections and built in types for now, + # they should be added when queries are generated. + def objects + strong_memoize(:objects) do + mutations = schema.mutation&.fields&.keys&.to_set || [] + + graphql_object_types + .reject { |object_type| object_type[:name]["__"] || object_type[:name] == 'Subscription' } # We ignore introspection and subscription types. + .map do |type| + name = type[:name] + type.merge( + is_edge: name.ends_with?('Edge'), + is_connection: name.ends_with?('Connection'), + is_payload: name.ends_with?('Payload') && mutations.include?(name.chomp('Payload').camelcase(:lower)), + fields: type[:fields] + type[:connections] + ) + end + end + end + + def args?(field) + args = field[:arguments] + return false if args.blank? + return true unless connection?(field) + + args.any? { |arg| CONNECTION_ARGS.exclude?(arg[:name]) } + end + # returns the deprecation information for a field or argument # See: Gitlab::Graphql::Deprecation def schema_deprecation(type_name, field_name) - schema_member(type_name, field_name)&.deprecation - end - - # Return a part of the schema. - # - # This queries the Schema by owner and name to find: - # - # - fields (e.g. `schema_member('Query', 'currentUser')`) - # - arguments (e.g. `schema_member(['Query', 'project], 'fullPath')`) - def schema_member(type_name, field_name) - type_name = Array.wrap(type_name) - if type_name.size == 2 - arg_name = field_name - type_name, field_name = type_name - else - type_name = type_name.first - arg_name = nil - end + key = [*Array.wrap(type_name), field_name].join('.') + deprecations[key] + end - return if type_name.nil? || field_name.nil? + def render_input_type(query) + input_field = query[:input_fields]&.first + return unless input_field + "Input type: `#{input_field[:type][:name]}`" + end + + def schema_field(type_name, field_name) type = schema.types[type_name] return unless type && type.kind.fields? - field = type.fields[field_name] - return field if arg_name.nil? + type.fields[field_name] + end + + def deprecations + strong_memoize(:deprecations) do + mapping = {} + + schema.types.each do |type_name, type| + next unless type.kind.fields? - args = field.arguments - is_mutation = field.mutation && field.mutation <= ::Mutations::BaseMutation - args = args['input'].type.unwrap.arguments if is_mutation + type.fields.each do |field_name, field| + mapping["#{type_name}.#{field_name}"] = field.try(:deprecation) + field.arguments.each do |arg_name, arg| + mapping["#{type_name}.#{field_name}.#{arg_name}"] = arg.try(:deprecation) + end + end + end + + mapping.compact + end + end - args[arg_name] + def assert!(claim, message) + raise ViolatedAssumption, "#{message}\n#{SUGGESTED_ACTION}" unless claim end end end diff --git a/lib/gitlab/graphql/docs/renderer.rb b/lib/gitlab/graphql/docs/renderer.rb index 497567f9389..ae0898e6198 100644 --- a/lib/gitlab/graphql/docs/renderer.rb +++ b/lib/gitlab/graphql/docs/renderer.rb @@ -24,6 +24,7 @@ module Gitlab @layout = Haml::Engine.new(File.read(template)) @parsed_schema = GraphQLDocs::Parser.new(schema.graphql_definition, {}).parse @schema = schema + @seen = Set.new end def contents @@ -37,6 +38,16 @@ module Gitlab FileUtils.mkdir_p(@output_dir) File.write(filename, contents) end + + private + + def seen_type?(name) + @seen.include?(name) + end + + def seen_type!(name) + @seen << name + end end end end diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml index fe73297d0d9..7d42fb3a9f8 100644 --- a/lib/gitlab/graphql/docs/templates/default.md.haml +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -17,7 +17,9 @@ Items (fields, enums, etc) that have been removed according to our [deprecation process](../index.md#deprecation-and-removal-process) can be found in [Removed Items](../removed_items.md). - <!-- vale gitlab.Spelling = NO --> + <!-- vale off --> + <!-- Docs linting disabled after this line. --> + <!-- See https://docs.gitlab.com/ee/development/documentation/testing.html#disable-vale-tests --> \ :plain @@ -26,17 +28,81 @@ The `Query` type contains the API's top-level entry points for all executable queries. \ -- sorted_by_name(queries).each do |query| - = render_name_and_description(query, owner: 'Query') +- fields_of('Query').each do |field| + = render_full_field(field, heading_level: 3, owner: 'Query') + \ + +:plain + ## `Mutation` type + + The `Mutation` type contains all the mutations you can execute. + + All mutations receive their arguments in a single input object named `input`, and all mutations + support at least a return field `errors` containing a list of error messages. + + All input objects may have a `clientMutationId: String` field, identifying the mutation. + + For example: + + ```graphql + mutation($id: NoteableID!, $body: String!) { + createNote(input: { noteableId: $id, body: $body }) { + errors + } + } + ``` +\ + +- mutations.each do |field| + = render_full_field(field, heading_level: 3, owner: 'Mutation') + \ + +:plain + ## Connections + + Some types in our schema are `Connection` types - they represent a paginated + collection of edges between two nodes in the graph. These follow the + [Relay cursor connections specification](https://relay.dev/graphql/connections.htm). + + ### Pagination arguments {#connection-pagination-arguments} + + All connection fields support the following pagination arguments: + + | Name | Type | Description | + |------|------|-------------| + | `after` | [`String`](#string) | Returns the elements in the list that come after the specified cursor. | + | `before` | [`String`](#string) | Returns the elements in the list that come before the specified cursor. | + | `first` | [`Int`](#int) | Returns the first _n_ elements from the list. | + | `last` | [`Int`](#int) | Returns the last _n_ elements from the list. | + + Since these arguments are common to all connection fields, they are not repeated for each connection. + + ### Connection fields + + All connections have at least the following fields: + + | Name | Type | Description | + |------|------|-------------| + | `pageInfo` | [`PageInfo!`](#pageinfo) | Pagination information. | + | `edges` | `[edge!]` | The edges. | + | `nodes` | `[item!]` | The items in the current page. | + + The precise type of `Edge` and `Item` depends on the kind of connection. A + [`ProjectConnection`](#projectconnection) will have nodes that have the type + [`[Project!]`](#project), and edges that have the type [`ProjectEdge`](#projectedge). + + ### Connection types + + Some of the types in the schema exist solely to model connections. Each connection + has a distinct, named type, with a distinct named edge type. These are listed separately + below. +\ + +- connection_object_types.each do |type| + = render_name_and_description(type, level: 4) + \ + = render_object_fields(type[:fields], owner: type, level_bump: 1) \ - = render_return_type(query) - - unless query[:arguments].empty? - ~ "#### Arguments\n" - ~ "| Name | Type | Description |" - ~ "| ---- | ---- | ----------- |" - - sorted_by_name(query[:arguments]).each do |argument| - = render_field(argument, query[:type][:name]) - \ :plain ## Object types @@ -44,22 +110,20 @@ Object types represent the resources that the GitLab GraphQL API can return. They contain _fields_. Each field has its own type, which will either be one of the basic GraphQL [scalar types](https://graphql.org/learn/schema/#scalar-types) - (e.g.: `String` or `Boolean`) or other object types. + (e.g.: `String` or `Boolean`) or other object types. Fields may have arguments. + Fields with arguments are exactly like top-level queries, and are listed beneath + the table of fields for each object type. For more information, see [Object Types and Fields](https://graphql.org/learn/schema/#object-types-and-fields) on `graphql.org`. \ -- objects.each do |type| - - unless type[:fields].empty? - = render_name_and_description(type) - \ - ~ "| Field | Type | Description |" - ~ "| ----- | ---- | ----------- |" - - sorted_by_name(type[:fields]).each do |field| - = render_field(field, type[:name]) - \ +- object_types.each do |type| + = render_name_and_description(type) + \ + = render_object_fields(type[:fields], owner: type) + \ :plain ## Enumeration types @@ -73,14 +137,13 @@ \ - enums.each do |enum| - - unless enum[:values].empty? - = render_name_and_description(enum) - \ - ~ "| Value | Description |" - ~ "| ----- | ----------- |" - - sorted_by_name(enum[:values]).each do |value| - = render_enum_value(enum, value) - \ + = render_name_and_description(enum) + \ + ~ "| Value | Description |" + ~ "| ----- | ----------- |" + - enum[:values].each do |value| + = render_enum_value(enum, value) + \ :plain ## Scalar types @@ -133,7 +196,7 @@ ### Interfaces \ -- graphql_interface_types.each do |type| +- interfaces.each do |type| = render_name_and_description(type, level: 4) \ Implementations: @@ -141,8 +204,21 @@ - type[:implemented_by].each do |type_name| ~ "- [`#{type_name}`](##{type_name.downcase})" \ - ~ "| Field | Type | Description |" - ~ "| ----- | ---- | ----------- |" - - sorted_by_name(type[:fields] + type[:connections]).each do |field| - = render_field(field, type[:name]) + = render_object_fields(type[:fields], owner: type, level_bump: 1) + \ + +:plain + ## Input types + + Types that may be used as arguments (all scalar types may also + be used as arguments). + + Only general use input types are listed here. For mutation input types, + see the associated mutation type above. +\ + +- input_types.each do |type| + = render_name_and_description(type) + \ + = render_argument_table(3, type[:input_fields], type[:name]) \ diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index e525996ec10..61903c566f0 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -114,7 +114,7 @@ module Gitlab def limited_nodes strong_memoize(:limited_nodes) do if first && last - raise Gitlab::Graphql::Errors::ArgumentError.new("Can only provide either `first` or `last`, not both") + raise Gitlab::Graphql::Errors::ArgumentError, "Can only provide either `first` or `last`, not both" end if last @@ -158,7 +158,7 @@ module Gitlab def ordered_items strong_memoize(:ordered_items) do unless items.primary_key.present? - raise ArgumentError.new('Relation must have a primary key') + raise ArgumentError, 'Relation must have a primary key' end list = OrderInfo.build_order_list(items) diff --git a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb index 318c6e1734f..f1b74999897 100644 --- a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb +++ b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb @@ -10,6 +10,8 @@ module Gitlab extend ActiveSupport::Concern def ordered_items + raise ArgumentError, 'Relation must have a primary key' unless items.primary_key.present? + return super unless Gitlab::Pagination::Keyset::Order.keyset_aware?(items) items @@ -40,6 +42,17 @@ module Gitlab sliced = slice_nodes(sliced, after, :after) if after.present? sliced end + + def items + original_items = super + return original_items if Gitlab::Pagination::Keyset::Order.keyset_aware?(original_items) || Feature.disabled?(:new_graphql_keyset_pagination) + + strong_memoize(:generic_keyset_pagination_items) do + rebuilt_items_with_keyset_order, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(original_items) + + success ? rebuilt_items_with_keyset_order : original_items + end + end end end end diff --git a/lib/gitlab/graphql/pagination/keyset/order_info.rb b/lib/gitlab/graphql/pagination/keyset/order_info.rb index 0494329bfd9..57e85ebe7f6 100644 --- a/lib/gitlab/graphql/pagination/keyset/order_info.rb +++ b/lib/gitlab/graphql/pagination/keyset/order_info.rb @@ -36,24 +36,24 @@ module Gitlab def self.validate_ordering(relation, order_list) if order_list.empty? - raise ArgumentError.new('A minimum of 1 ordering field is required') + raise ArgumentError, 'A minimum of 1 ordering field is required' end if order_list.count > 2 # Keep in mind an order clause for primary key is added if one is not present # lib/gitlab/graphql/pagination/keyset/connection.rb:97 - raise ArgumentError.new('A maximum of 2 ordering fields are allowed') + raise ArgumentError, 'A maximum of 2 ordering fields are allowed' end # make sure the last ordering field is non-nullable attribute_name = order_list.last&.attribute_name if relation.columns_hash[attribute_name].null - raise ArgumentError.new("Column `#{attribute_name}` must not allow NULL") + raise ArgumentError, "Column `#{attribute_name}` must not allow NULL" end if order_list.last.attribute_name != relation.primary_key - raise ArgumentError.new("Last ordering field must be the primary key, `#{relation.primary_key}`") + raise ArgumentError, "Last ordering field must be the primary key, `#{relation.primary_key}`" end end @@ -121,4 +121,4 @@ module Gitlab end end -Gitlab::Graphql::Pagination::Keyset::OrderInfo.prepend_if_ee('EE::Gitlab::Graphql::Pagination::Keyset::OrderInfo') +Gitlab::Graphql::Pagination::Keyset::OrderInfo.prepend_mod_with('Gitlab::Graphql::Pagination::Keyset::OrderInfo') diff --git a/lib/gitlab/graphql/pagination/keyset/query_builder.rb b/lib/gitlab/graphql/pagination/keyset/query_builder.rb index ee9c902c735..a2f53ae83dd 100644 --- a/lib/gitlab/graphql/pagination/keyset/query_builder.rb +++ b/lib/gitlab/graphql/pagination/keyset/query_builder.rb @@ -12,7 +12,7 @@ module Gitlab @before_or_after = before_or_after if order_list.empty? - raise ArgumentError.new('No ordering scopes have been supplied') + raise ArgumentError, 'No ordering scopes have been supplied' end end @@ -49,7 +49,7 @@ module Gitlab end if order_list.count == 1 && attr_values.first.nil? - raise Gitlab::Graphql::Errors::ArgumentError.new('Before/after cursor invalid: `nil` was provided as only sortable value') + raise Gitlab::Graphql::Errors::ArgumentError, 'Before/after cursor invalid: `nil` was provided as only sortable value' end if order_list.count == 1 || attr_values.first.present? diff --git a/lib/gitlab/graphql/present.rb b/lib/gitlab/graphql/present.rb index fdaf075eb25..3608cb4c0e8 100644 --- a/lib/gitlab/graphql/present.rb +++ b/lib/gitlab/graphql/present.rb @@ -10,14 +10,14 @@ module Gitlab end def self.presenter_class - @presenter_class + @presenter_class || superclass.try(:presenter_class) end def self.present(object, attrs) - klass = @presenter_class + klass = presenter_class return object if !klass || object.is_a?(klass) - @presenter_class.new(object, **attrs) + klass.new(object, **attrs) end end diff --git a/lib/gitlab/graphql/present/field_extension.rb b/lib/gitlab/graphql/present/field_extension.rb index 2e211b70d35..050a3a276ea 100644 --- a/lib/gitlab/graphql/present/field_extension.rb +++ b/lib/gitlab/graphql/present/field_extension.rb @@ -13,7 +13,8 @@ module Gitlab # inner Schema::Object#object. This depends on whether the field # has a @resolver_proc or not. if object.is_a?(::Types::BaseObject) - object.present(field.owner, attrs) + type = field.owner.kind.abstract? ? object.class : field.owner + object.present(type, attrs) yield(object, arguments) else # This is the legacy code-path, hit if the field has a @resolver_proc diff --git a/lib/gitlab/graphql/queries.rb b/lib/gitlab/graphql/queries.rb index 74f55abccbc..5d3a9245427 100644 --- a/lib/gitlab/graphql/queries.rb +++ b/lib/gitlab/graphql/queries.rb @@ -264,7 +264,7 @@ module Gitlab definitions = [] ::Find.find(root.to_s) do |path| - definitions << Definition.new(path, fragments) if query?(path) + definitions << Definition.new(path, fragments) if query_for_gitlab_schema?(path) end definitions @@ -288,10 +288,11 @@ module Gitlab @known_failures.fetch('filenames', []).any? { |known_failure| path.to_s.ends_with?(known_failure) } end - def self.query?(path) + def self.query_for_gitlab_schema?(path) path.ends_with?('.graphql') && !path.ends_with?('.fragment.graphql') && - !path.ends_with?('typedefs.graphql') + !path.ends_with?('typedefs.graphql') && + !/.*\.customer\.(query|mutation)\.graphql$/.match?(path) end end end diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb index c6f22e0bd4f..b8d2f5b0f29 100644 --- a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb +++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb @@ -16,7 +16,7 @@ module Gitlab query_string: query.query_string, variables: variables }) - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) default_initial_values(query) end @@ -41,7 +41,7 @@ module Gitlab RequestStore.store[:graphql_logs] ||= [] RequestStore.store[:graphql_logs] << memo GraphqlLogger.info(memo.except!(:time_started, :query)) - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end diff --git a/lib/gitlab/graphql/variables.rb b/lib/gitlab/graphql/variables.rb index 1c6fb011012..e17ca56d022 100644 --- a/lib/gitlab/graphql/variables.rb +++ b/lib/gitlab/graphql/variables.rb @@ -32,7 +32,7 @@ module Gitlab raise Invalid, "Unexpected parameter: #{ambiguous_param}" end rescue JSON::ParserError => e - raise Invalid.new(e) + raise Invalid, e end end end diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index dd872caee0e..4eea96f8344 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -39,4 +39,4 @@ module Gitlab end end -Gitlab::GroupSearchResults.prepend_if_ee('EE::Gitlab::GroupSearchResults') +Gitlab::GroupSearchResults.prepend_mod_with('Gitlab::GroupSearchResults') diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb index b57560544c8..912e2ee99e9 100644 --- a/lib/gitlab/hashed_storage/migrator.rb +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -66,7 +66,7 @@ module Gitlab Gitlab::AppLogger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..." project.migrate_to_hashed_storage! - rescue => err + rescue StandardError => err Gitlab::AppLogger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") end @@ -77,7 +77,7 @@ module Gitlab Gitlab::AppLogger.info "Starting storage rollback of #{project.full_path} (ID=#{project.id})..." project.rollback_to_legacy_storage! - rescue => err + rescue StandardError => err Gitlab::AppLogger.error("#{err.message} rolling-back storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") end diff --git a/lib/gitlab/health_checks/probes/collection.rb b/lib/gitlab/health_checks/probes/collection.rb index b34e4273d85..76ad1c84214 100644 --- a/lib/gitlab/health_checks/probes/collection.rb +++ b/lib/gitlab/health_checks/probes/collection.rb @@ -20,7 +20,7 @@ module Gitlab success ? 200 : 503, status(success).merge(payload(readiness)) ) - rescue => e + rescue StandardError => e exception_payload = { message: "#{e.class} : #{e.message}" } Probes::Status.new( diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index ae99768b7b4..432d5d5e5ea 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -16,7 +16,7 @@ module Gitlab else HealthChecks::Result.new(name, false, "unexpected #{human_name} check result: #{check_result}") end - rescue => e + rescue StandardError => e HealthChecks::Result.new(name, false, "unexpected #{human_name} check result: #{e}") end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 765d3dfca56..e4857280969 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -4,13 +4,20 @@ module Gitlab class Highlight TIMEOUT_BACKGROUND = 30.seconds TIMEOUT_FOREGROUND = 1.5.seconds - MAXIMUM_TEXT_HIGHLIGHT_SIZE = 512.kilobytes def self.highlight(blob_name, blob_content, language: nil, plain: false) new(blob_name, blob_content, language: language) .highlight(blob_content, continue: false, plain: plain) end + def self.too_large?(size) + return false unless size.to_i > Gitlab.config.extra['maximum_text_highlight_size_kilobytes'] + + over_highlight_size_limit.increment(source: "text highlighter") if Feature.enabled?(:track_file_size_over_highlight_limit) + + true + end + attr_reader :blob_name def initialize(blob_name, blob_content, language: nil) @@ -23,7 +30,7 @@ module Gitlab def highlight(text, continue: false, plain: false, context: {}) @context = context - plain ||= text.length > MAXIMUM_TEXT_HIGHLIGHT_SIZE + plain ||= self.class.too_large?(text.length) highlighted_text = highlight_text(text, continue: continue, plain: plain) highlighted_text = link_dependencies(text, highlighted_text) if blob_name @@ -65,9 +72,11 @@ module Gitlab tokens = lexer.lex(text, continue: continue) Timeout.timeout(timeout_time) { @formatter.format(tokens, context.merge(tag: tag)).html_safe } rescue Timeout::Error => e + add_highlight_timeout_metric + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) highlight_plain(text) - rescue + rescue StandardError highlight_plain(text) end @@ -78,5 +87,25 @@ module Gitlab def link_dependencies(text, highlighted_text) Gitlab::DependencyLinker.link(blob_name, text, highlighted_text) end + + def add_highlight_timeout_metric + return unless Feature.enabled?(:track_highlight_timeouts) + + highlight_timeout.increment(source: Gitlab::Runtime.sidekiq? ? "background" : "foreground") + end + + def highlight_timeout + @highlight_timeout ||= Gitlab::Metrics.counter( + :highlight_timeout, + 'Counts the times highlights have timed out' + ) + end + + def self.over_highlight_size_limit + @over_highlight_size_limit ||= Gitlab::Metrics.counter( + :over_highlight_size_limit, + 'Count the times files have been over the highlight size limit' + ) + end end end diff --git a/lib/gitlab/hook_data/group_member_builder.rb b/lib/gitlab/hook_data/group_member_builder.rb index 32cfd032ffe..2998550a4b5 100644 --- a/lib/gitlab/hook_data/group_member_builder.rb +++ b/lib/gitlab/hook_data/group_member_builder.rb @@ -62,4 +62,4 @@ module Gitlab end end -Gitlab::HookData::GroupMemberBuilder.prepend_if_ee('EE::Gitlab::HookData::GroupMemberBuilder') +Gitlab::HookData::GroupMemberBuilder.prepend_mod_with('Gitlab::HookData::GroupMemberBuilder') diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index f38012c9804..d5595e80bdf 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -58,4 +58,4 @@ module Gitlab end end -Gitlab::HookData::IssueBuilder.prepend_if_ee('EE::Gitlab::HookData::IssueBuilder') +Gitlab::HookData::IssueBuilder.prepend_mod_with('Gitlab::HookData::IssueBuilder') diff --git a/lib/gitlab/hook_data/key_builder.rb b/lib/gitlab/hook_data/key_builder.rb new file mode 100644 index 00000000000..8eaf4dfd762 --- /dev/null +++ b/lib/gitlab/hook_data/key_builder.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class KeyBuilder < BaseBuilder + alias_method :key, :object + + # Sample data + # { + # event_name: "key_create", + # created_at: "2021-04-19T06:13:24Z", + # updated_at: "2021-04-19T06:13:24Z", + # key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQClDn/5BaESHlSb3NxQtiUc0BXgK6lsqdAUIdS3lwZ2gbACDhtoLYnc+qhZ4b8gWzE+2A8RmkvLe98T7noRoW4DAYs67NSqMs/kXd2ESPNV8qqv0u7tCxPz+c7DaYp2oC/avlxVQ2AeULZLCEwalYZ7irde0EZMeTwNIRu5s88gOw== dummy@gitlab.com", + # id: 1, + # username: "johndoe" + # } + + def build(event) + [ + event_data(event), + timestamps_data, + key_data, + user_data + ].reduce(:merge) + end + + private + + def key_data + { + key: key.key, + id: key.id + } + end + + def user_data + user = key.user + return {} unless user + + { + username: user.username + } + end + end + end +end diff --git a/lib/gitlab/hook_data/project_builder.rb b/lib/gitlab/hook_data/project_builder.rb new file mode 100644 index 00000000000..65c237f743f --- /dev/null +++ b/lib/gitlab/hook_data/project_builder.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class ProjectBuilder < BaseBuilder + alias_method :project, :object + + # Sample data + # { + # event_name: "project_rename", + # created_at: "2021-04-19T07:05:36Z", + # updated_at: "2021-04-19T07:05:36Z", + # name: "my_project", + # path: "my_project", + # path_with_namespace: "namespace2/my_project", + # project_id: 1, + # owner_name: "John", + # owner_email: "user1@example.org", + # project_visibility: "internal", + # old_path_with_namespace: "old-path-with-namespace" + # } + + def build(event) + [ + event_data(event), + timestamps_data, + project_data, + event_specific_project_data(event) + ].reduce(:merge) + end + + private + + def project_data + owner = project.owner + + { + name: project.name, + path: project.path, + path_with_namespace: project.full_path, + project_id: project.id, + owner_name: owner.name, + owner_email: owner.respond_to?(:email) ? owner.email : "", + project_visibility: project.visibility.downcase + } + end + + def event_specific_project_data(event) + return {} unless event == :rename || event == :transfer + + { + old_path_with_namespace: project.old_path_with_namespace + } + end + end + end +end diff --git a/lib/gitlab/hook_data/user_builder.rb b/lib/gitlab/hook_data/user_builder.rb index 537245e948f..54f03b863e5 100644 --- a/lib/gitlab/hook_data/user_builder.rb +++ b/lib/gitlab/hook_data/user_builder.rb @@ -50,4 +50,4 @@ module Gitlab end end -Gitlab::HookData::UserBuilder.prepend_if_ee('EE::Gitlab::HookData::UserBuilder') +Gitlab::HookData::UserBuilder.prepend_mod_with('Gitlab::HookData::UserBuilder') diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 3b19ae3d7ff..023dbd1c601 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -4,20 +4,6 @@ module Gitlab module I18n extend self - # Languages with less then 2% of available translations will not - # be available in the UI. - # https://gitlab.com/gitlab-org/gitlab/-/issues/221012 - NOT_AVAILABLE_IN_UI = %w[ - fil_PH - pl_PL - nl_NL - id_ID - cs_CZ - bg - eo - gl_ES - ].freeze - AVAILABLE_LANGUAGES = { 'bg' => 'Bulgarian - български', 'cs_CZ' => 'Czech - čeština', @@ -42,9 +28,49 @@ module Gitlab 'zh_HK' => 'Chinese, Traditional (Hong Kong) - 繁體中文 (香港)', 'zh_TW' => 'Chinese, Traditional (Taiwan) - 繁體中文 (台灣)' }.freeze + private_constant :AVAILABLE_LANGUAGES + + # Languages with less then MINIMUM_TRANSLATION_LEVEL% of available translations will not + # be available in the UI. + # https://gitlab.com/gitlab-org/gitlab/-/issues/221012 + MINIMUM_TRANSLATION_LEVEL = 2 + + # Currently monthly updated manually by ~group::import PM. + # https://gitlab.com/gitlab-org/gitlab/-/issues/18923 + TRANSLATION_LEVELS = { + 'bg' => 1, + 'cs_CZ' => 1, + 'de' => 19, + 'en' => 100, + 'eo' => 1, + 'es' => 41, + 'fil_PH' => 1, + 'fr' => 14, + 'gl_ES' => 1, + 'id_ID' => 0, + 'it' => 2, + 'ja' => 45, + 'ko' => 14, + 'nl_NL' => 1, + 'pl_PL' => 1, + 'pt_BR' => 22, + 'ru' => 32, + 'tr_TR' => 17, + 'uk' => 43, + 'zh_CN' => 72, + 'zh_HK' => 3, + 'zh_TW' => 4 + }.freeze + private_constant :TRANSLATION_LEVELS def selectable_locales - AVAILABLE_LANGUAGES.reject { |key, _value| NOT_AVAILABLE_IN_UI.include? key } + AVAILABLE_LANGUAGES.reject do |code, _name| + percentage_translated_for(code) < MINIMUM_TRANSLATION_LEVEL + end + end + + def percentage_translated_for(code) + TRANSLATION_LEVELS.fetch(code, 0) end def available_locales diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index c4867746b0f..231f2a977c0 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -99,11 +99,19 @@ module Gitlab def group_config_file Rails.root.join('lib/gitlab/import_export/group/import_export.yml') end + + def group_wiki_repo_bundle_filename(group_id) + "#{group_id}.wiki.bundle" + end + + def group_wiki_repo_bundle_path(shared, filename) + File.join(shared.export_path, 'repositories', filename) + end + + def group_wiki_repo_bundle_full_path(shared, group_id) + group_wiki_repo_bundle_path(shared, group_wiki_repo_bundle_filename(group_id)) + end end end -Gitlab::ImportExport.prepend_if_ee('EE::Gitlab::ImportExport') - -# The methods in `Gitlab::ImportExport::GroupHelper` should be available as both -# instance and class methods. -Gitlab::ImportExport.extend_if_ee('Gitlab::ImportExport::GroupHelper') +Gitlab::ImportExport.prepend_mod diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb index b30258123d4..b43d0a0c3eb 100644 --- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -42,7 +42,7 @@ module Gitlab strategy_execute true - rescue => e + rescue StandardError => e project.import_export_shared.error(e) false ensure diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index e2dba831661..1e8009d29c2 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -28,7 +28,7 @@ module Gitlab def handle_response_error(response) unless response.success? - raise StrategyError.new("Error uploading the project. Code #{response.code}: #{response.message}") + raise StrategyError, "Error uploading the project. Code #{response.code}: #{response.message}" end end diff --git a/lib/gitlab/import_export/after_export_strategy_builder.rb b/lib/gitlab/import_export/after_export_strategy_builder.rb index 37394f46a99..d7b30f46903 100644 --- a/lib/gitlab/import_export/after_export_strategy_builder.rb +++ b/lib/gitlab/import_export/after_export_strategy_builder.rb @@ -12,7 +12,7 @@ module Gitlab klass = strategy_klass.constantize rescue nil unless klass && klass < AfterExportStrategies::BaseAfterExportStrategy - raise StrategyNotFoundError.new("Strategy #{strategy_klass} not found") + raise StrategyNotFoundError, "Strategy #{strategy_klass} not found" end klass.new(**attributes.symbolize_keys) diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 1e98595bb07..4abc3da1190 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -3,7 +3,7 @@ module Gitlab module ImportExport class AttributesFinder - attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads + attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads, :export_reorders def initialize(config:) @tree = config[:tree] || {} @@ -11,6 +11,7 @@ module Gitlab @excluded_attributes = config[:excluded_attributes] || {} @methods = config[:methods] || {} @preloads = config[:preloads] || {} + @export_reorders = config[:export_reorders] || {} end def find_root(model_key) @@ -33,7 +34,8 @@ module Gitlab except: @excluded_attributes[model_key], methods: @methods[model_key], include: resolve_model_tree(model_tree), - preload: resolve_preloads(model_key, model_tree) + preload: resolve_preloads(model_key, model_tree), + export_reorder: @export_reorders[model_key] }.compact end diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb index be1b97bd7a7..01ff99798d5 100644 --- a/lib/gitlab/import_export/avatar_restorer.rb +++ b/lib/gitlab/import_export/avatar_restorer.rb @@ -13,7 +13,7 @@ module Gitlab @project.avatar = File.open(avatar_export_file) @project.save! - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb index 47ca898c690..7534ab5a9ce 100644 --- a/lib/gitlab/import_export/avatar_saver.rb +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -16,7 +16,7 @@ module Gitlab shared: @shared, relative_export_path: 'avatar' ).save - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 05a4a8f4c93..959ece4b903 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -44,8 +44,9 @@ module Gitlab relation_name.to_s.constantize end - def initialize(relation_sym:, relation_hash:, members_mapper:, object_builder:, user:, importable:, excluded_keys: []) + def initialize(relation_sym:, relation_index:, relation_hash:, members_mapper:, object_builder:, user:, importable:, excluded_keys: []) @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym + @relation_index = relation_index @relation_hash = relation_hash.except('noteable_id') @members_mapper = members_mapper @object_builder = object_builder diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 2f8769e261d..ace9d83dc9a 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -14,6 +14,19 @@ module Gitlab untar_with_options(archive: archive, dir: dir, options: 'zxf') end + def gzip(dir:, filename:) + filepath = File.join(dir, filename) + cmd = %W(gzip #{filepath}) + + _, status = Gitlab::Popen.popen(cmd) + + if status == 0 + status + else + raise Gitlab::ImportExport::Error.file_compression_error + end + end + def mkdir_p(path) FileUtils.mkdir_p(path, mode: DEFAULT_DIR_MODE) FileUtils.chmod(DEFAULT_DIR_MODE, path) diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index 37f1bdc3009..2baf2c61f7c 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -61,7 +61,7 @@ module Gitlab Process.kill(-1, pgrp) if pgrp false - rescue => e + rescue StandardError => e log_error(e.message) Process.kill(-1, pgrp) if pgrp diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index f11b7a0a298..4af6b03fe94 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -3,12 +3,20 @@ module Gitlab module ImportExport class Error < StandardError - def self.permission_error(user, importable) + def self.permission_error(user, object) self.new( "User with ID: %s does not have required permissions for %s: %s with ID: %s" % - [user.id, importable.class.name, importable.name, importable.id] + [user.id, object.class.name, object.name, object.id] ) end + + def self.unsupported_object_type_error + self.new('Unknown object type') + end + + def self.file_compression_error + self.new('File compression failed') + end end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 51d58aae54f..4b3258f8caa 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -33,7 +33,7 @@ module Gitlab validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size) decompress_archive end - rescue => e + rescue StandardError => e @shared.error(e) false ensure @@ -57,7 +57,7 @@ module Gitlab def decompress_archive result = untar_zxf(archive: @archive_file, dir: @shared.export_path) - raise ImporterError.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result + raise ImporterError, "Unable to decompress #{@archive_file} into #{@shared.export_path}" unless result result end @@ -67,7 +67,17 @@ module Gitlab @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @importable)) - download_or_copy_upload(@importable.import_export_upload.import_file, @archive_file) + remote_download_or_download_or_copy_upload + end + + def remote_download_or_download_or_copy_upload + import_export_upload = @importable.import_export_upload + + if import_export_upload.remote_import_url.present? + download(import_export_upload.remote_import_url, @archive_file) + else + download_or_copy_upload(import_export_upload.import_file, @archive_file) + end end def remove_symlinks @@ -87,7 +97,7 @@ module Gitlab end def validate_decompressed_archive_size - raise ImporterError.new(_('Decompressed archive size validation failed.')) unless size_validator.valid? + raise ImporterError, _('Decompressed archive size validation failed.') unless size_validator.valid? end def size_validator diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index e30206dc509..aceb4821a06 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -58,6 +58,8 @@ methods: preloads: +export_reorders: + # EE specific relationships and settings to include. All of this will be merged # into the previous structures if EE is used. ee: diff --git a/lib/gitlab/import_export/group/legacy_import_export.yml b/lib/gitlab/import_export/group/legacy_import_export.yml index 5008639077c..19611e1b010 100644 --- a/lib/gitlab/import_export/group/legacy_import_export.yml +++ b/lib/gitlab/import_export/group/legacy_import_export.yml @@ -60,6 +60,8 @@ methods: preloads: +export_reorders: + # EE specific relationships and settings to include. All of this will be merged # into the previous structures if EE is used. ee: diff --git a/lib/gitlab/import_export/group/legacy_tree_restorer.rb b/lib/gitlab/import_export/group/legacy_tree_restorer.rb index 5499b79cee6..2b95c098b59 100644 --- a/lib/gitlab/import_export/group/legacy_tree_restorer.rb +++ b/lib/gitlab/import_export/group/legacy_tree_restorer.rb @@ -45,7 +45,7 @@ module Gitlab return false if @shared.errors.any? true - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/group/legacy_tree_saver.rb b/lib/gitlab/import_export/group/legacy_tree_saver.rb index 7ab81c09885..0f74fabeac3 100644 --- a/lib/gitlab/import_export/group/legacy_tree_saver.rb +++ b/lib/gitlab/import_export/group/legacy_tree_saver.rb @@ -19,7 +19,7 @@ module Gitlab tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename) true - rescue => e + rescue StandardError => e @shared.error(e) false end @@ -35,7 +35,7 @@ module Gitlab end group_tree - rescue => e + rescue StandardError => e @shared.error(e) end diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb index 925ab6680ba..ea7de4cc896 100644 --- a/lib/gitlab/import_export/group/tree_restorer.rb +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -26,7 +26,7 @@ module Gitlab end true - rescue => e + rescue StandardError => e shared.error(e) false end @@ -74,7 +74,7 @@ module Gitlab group = create_group(group_attributes) restore_group(group, group_attributes) - rescue => e + rescue StandardError => e import_failure_service.log_import_failure( source: 'process_child', relation_key: 'group', diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb index d538de33c51..0f588a55f9d 100644 --- a/lib/gitlab/import_export/group/tree_saver.rb +++ b/lib/gitlab/import_export/group/tree_saver.rb @@ -25,7 +25,7 @@ module Gitlab json_writer.write_relation_array('groups', '_all', all_groups) true - rescue => e + rescue StandardError => e @shared.error(e) false ensure diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 390909efe36..c2510bbe938 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -21,15 +21,15 @@ module Gitlab if import_file && check_version! && restorers.all?(&:restore) && overwrite_project project else - raise Projects::ImportService::Error.new(shared.errors.to_sentence) + raise Projects::ImportService::Error, shared.errors.to_sentence end - rescue => e + rescue StandardError => e # If some exception was raised could mean that the SnippetsRepoRestorer # was not called. This would leave us with snippets without a repository. # This is a state we don't want them to be, so we better delete them. remove_non_migrated_snippets - raise Projects::ImportService::Error.new(e.message) + raise Projects::ImportService::Error, e.message ensure remove_base_tmp_dir remove_import_file diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb index 12d6458aedc..f29c0a44188 100644 --- a/lib/gitlab/import_export/json/legacy_reader.rb +++ b/lib/gitlab/import_export/json/legacy_reader.rb @@ -28,9 +28,9 @@ module Gitlab def read_hash ActiveSupport::JSON.decode(IO.read(@path)) - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.log_exception(e) - raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + raise Gitlab::ImportExport::Error, 'Incorrect JSON format' end end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 05b7679e0ff..ec42c5e51c0 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -38,16 +38,6 @@ module Gitlab end end - private - - attr_reader :json_writer, :relations_schema, :exportable - - def serialize_root - attributes = exportable.as_json( - relations_schema.merge(include: nil, preloads: nil)) - json_writer.write_attributes(@exportable_path, attributes) - end - def serialize_relation(definition) raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash) raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one? @@ -64,17 +54,22 @@ module Gitlab end end + private + + attr_reader :json_writer, :relations_schema, :exportable + + def serialize_root + attributes = exportable.as_json( + relations_schema.merge(include: nil, preloads: nil)) + json_writer.write_attributes(@exportable_path, attributes) + end + def serialize_many_relations(key, records, options) enumerator = Enumerator.new do |items| key_preloads = preloads&.dig(key) - records = records.preload(key_preloads) if key_preloads - records.in_batches(of: batch_size) do |batch| # rubocop:disable Cop/InBatches - # order each batch by its primary key to ensure - # consistent and predictable ordering of each exported relation - # as additional `WHERE` clauses can impact the order in which data is being - # returned by database when no `ORDER` is specified - batch = batch.reorder(batch.klass.primary_key) + batch(records, key) do |batch| + batch = batch.preload(key_preloads) if key_preloads batch.each do |record| items << Raw.new(record.to_json(options)) @@ -85,6 +80,29 @@ module Gitlab json_writer.write_relation_array(@exportable_path, key, enumerator) end + def batch(relation, key) + opts = { of: batch_size } + order_by = reorders(relation, key) + + # we need to sort issues by non primary key column(relative_position) + # and `in_batches` does not support that + if order_by + scope = relation.reorder(order_by) + + Gitlab::Pagination::Keyset::Iterator.new(scope: scope, use_union_optimization: true).each_batch(**opts) do |batch| + yield batch + end + else + relation.in_batches(**opts) do |batch| # rubocop:disable Cop/InBatches + # order each batch by its primary key to ensure + # consistent and predictable ordering of each exported relation + # as additional `WHERE` clauses can impact the order in which data is being + # returned by database when no `ORDER` is specified + yield batch.reorder(batch.klass.primary_key) + end + end + end + def serialize_many_each(key, records, options) enumerator = Enumerator.new do |items| records.each do |record| @@ -112,6 +130,42 @@ module Gitlab def batch_size @batch_size ||= self.class.batch_size(@exportable) end + + def reorders(relation, key) + export_reorder = relations_schema[:export_reorder]&.dig(key) + return unless export_reorder + + custom_reorder(relation.klass, export_reorder) + end + + def custom_reorder(klass, order_by) + arel_table = klass.arel_table + column = order_by[:column] || klass.primary_key + direction = order_by[:direction] || :asc + nulls_position = order_by[:nulls_position] || :nulls_last + + arel_order_classes = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::AREL_ORDER_CLASSES.invert + reverse_direction = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::REVERSED_ORDER_DIRECTIONS[direction] + reverse_nulls_position = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::REVERSED_NULL_POSITIONS[nulls_position] + order_expression = ::Gitlab::Database.nulls_order(column, direction, nulls_position) + reverse_order_expression = ::Gitlab::Database.nulls_order(column, reverse_direction, reverse_nulls_position) + + ::Gitlab::Pagination::Keyset::Order.build([ + ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: column, + column_expression: arel_table[column], + order_expression: order_expression, + reversed_order_expression: reverse_order_expression, + order_direction: direction, + nullable: nulls_position, + distinct: false + ), + ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: klass.primary_key, + order_expression: arel_order_classes[direction].new(arel_table[klass.primary_key.to_sym]) + ) + ]) + end end end end diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb index ef83cdf24b1..d73ae1410a3 100644 --- a/lib/gitlab/import_export/lfs_restorer.rb +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -20,7 +20,7 @@ module Gitlab end true - rescue => e + rescue StandardError => e shared.error(e) false end @@ -73,8 +73,8 @@ module Gitlab begin json = IO.read(lfs_json_path) ActiveSupport::JSON.decode(json) - rescue - raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + rescue StandardError + raise Gitlab::ImportExport::Error, 'Incorrect JSON format' end end diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 4964b8b16f4..47acd49d529 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -27,7 +27,7 @@ module Gitlab write_lfs_json true - rescue => e + rescue StandardError => e shared.error(e) false diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 6b37683ea68..ff972cf9352 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -52,7 +52,7 @@ module Gitlab @importable.members.destroy_all # rubocop: disable Cop/DestroyAll relation_class.create!(user: @user, access_level: highest_access_level, source_id: @importable.id, importing: true) - rescue => e + rescue StandardError => e raise e, "Error adding importer user to #{@importable.class} members. #{e.message}" end diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index 4643742b607..3910afef108 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -40,7 +40,7 @@ module Gitlab # the commits are missing. def create_source_branch @project.repository.create_branch(@merge_request.source_branch, @diff_head_sha) - rescue => err + rescue StandardError => err Gitlab::Import::Logger.warn( message: 'Import warning: Failed to create source branch', source_branch: @merge_request.source_branch, diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 42d32593cbd..d000c331b6d 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -132,6 +132,7 @@ excluded_attributes: - :avatar - :import_type - :import_source + - :integrations - :mirror - :runners_token - :runners_token_encrypted @@ -152,6 +153,7 @@ excluded_attributes: - :bfg_object_map - :detected_repository_languages - :tag_list + - :topic_list - :mirror_user_id - :mirror_trigger_builds - :only_mirror_protected_branches @@ -261,6 +263,8 @@ excluded_attributes: - :resource_group_id - :waiting_for_resource_at - :processed + - :id_convert_to_bigint + - :stage_id_convert_to_bigint sentry_issue: - :issue_id push_event_payload: @@ -393,6 +397,8 @@ methods: - :state preloads: + issues: + project: :route statuses: # TODO: We cannot preload tags, as they are not part of `GenericCommitStatus` # tags: # needed by tag_list @@ -402,6 +408,29 @@ preloads: target_project: # needed by target_branch_sha assignees: # needed by assigne_id that is implemented by DeprecatedAssignee +# Specify a custom export reordering for a given relationship +# For example for issues we use a custom export reordering by relative_position, so that on import, we can reset the +# relative position value, but still keep the issues order to the order in which issues were in the exported project. +# By default the ordering of relations is done by PK. +# column - specify the column by which to reorder, by default it is relation's PK +# direction - specify the ordering direction :asc or :desc, default :asc +# nulls_position - specify where would null values be positioned. Because custom ordering column can contain nulls we +# need to also specify where would the nulls be placed. It can be :nulls_last or :nulls_first, defaults +# to :nulls_last +# Example: +# export_reorders: +# project: +# issues: +# column: :relative_position +# direction: :asc +# nulls_position: :nulls_last +export_reorders: + project: + issues: + column: :relative_position + direction: :asc + nulls_position: :nulls_last + # EE specific relationships and settings to include. All of this will be merged # into the previous structures if EE is used. ee: diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index ae92228276e..4678396f97e 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -80,6 +80,7 @@ module Gitlab when :notes then setup_note when :'Ci::Pipeline' then setup_pipeline when *BUILD_MODELS then setup_build + when :issues then setup_issue end update_project_references @@ -135,6 +136,22 @@ module Gitlab end end + def setup_issue + @relation_hash['relative_position'] = compute_relative_position + end + + def compute_relative_position + return unless max_relative_position + + max_relative_position + (@relation_index + 1) * Gitlab::RelativePositioning::IDEAL_DISTANCE + end + + def max_relative_position + Rails.cache.fetch("import:#{@importable.model_name.plural}:#{@importable.id}:hierarchy_max_issues_relative_position", expires_in: 24.hours) do + ::RelativePositioning.mover.context(Issue.in_projects(@importable.root_ancestor.all_projects).first)&.max_relative_position || ::Gitlab::RelativePositioning::START_POSITION + end + end + def legacy_trigger? @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil? end @@ -158,4 +175,4 @@ module Gitlab end end -Gitlab::ImportExport::Project::RelationFactory.prepend_if_ee('::EE::Gitlab::ImportExport::Project::RelationFactory') +Gitlab::ImportExport::Project::RelationFactory.prepend_mod_with('Gitlab::ImportExport::Project::RelationFactory') diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index fb9e5be1877..113502b4e3c 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -39,7 +39,7 @@ module Gitlab else false end - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 80dacf2eb20..16012f3c0c0 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -22,7 +22,7 @@ module Gitlab ).execute true - rescue => e + rescue StandardError => e @shared.error(e) false ensure diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 8d36d05ca6f..b9a1aee3b8e 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -35,7 +35,7 @@ module Gitlab def tree_by_key(key) attributes_finder.find_root(key) - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index 8bc87ecb071..46b82240ef7 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -48,7 +48,7 @@ module Gitlab @importable.reload # rubocop:disable Cop/ActiveRecordAssociationReload true - rescue => e + rescue StandardError => e @shared.error(e) false end @@ -81,7 +81,7 @@ module Gitlab relation_object.save! log_relation_creation(@importable, relation_key, relation_object) end - rescue => e + rescue StandardError => e import_failure_service.log_import_failure( source: 'process_relation_item!', relation_key: relation_key, @@ -155,7 +155,7 @@ module Gitlab transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition, relation_index) end - relation = @relation_factory.create(**relation_factory_params(relation_key, data_hash)) + relation = @relation_factory.create(**relation_factory_params(relation_key, relation_index, data_hash)) if relation && !relation.valid? @shared.logger.warn( @@ -221,8 +221,9 @@ module Gitlab importable_class.to_s.downcase.to_sym end - def relation_factory_params(relation_key, data_hash) + def relation_factory_params(relation_key, relation_index, data_hash) { + relation_index: relation_index, relation_sym: relation_key.to_sym, relation_hash: data_hash, importable: @importable, diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 998da3e4afb..1c6629cf942 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -22,7 +22,7 @@ module Gitlab update_importable_repository_info true - rescue => e + rescue StandardError => e shared.error(e) false end @@ -52,4 +52,4 @@ module Gitlab end end -Gitlab::ImportExport::RepoRestorer.prepend_if_ee('EE::Gitlab::ImportExport::RepoRestorer') +Gitlab::ImportExport::RepoRestorer.prepend_mod_with('Gitlab::ImportExport::RepoRestorer') diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index 0fdd0722b65..fae07039139 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -40,7 +40,7 @@ module Gitlab mkdir_p(File.dirname(bundle_full_path)) repository.bundle_to_disk(bundle_full_path) - rescue => e + rescue StandardError => e shared.error(e) false end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index bb2bbda4bd6..bec709f4a36 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -27,7 +27,7 @@ module Gitlab @shared.error(Gitlab::ImportExport::Error.new(error_message)) false end - rescue => e + rescue StandardError => e @shared.error(e) false ensure diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 09ed4eb568d..f295ab38de0 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -90,7 +90,7 @@ module Gitlab when 'Group' @exportable.full_path else - raise Gitlab::ImportExport::Error.new("Unsupported Exportable Type #{@exportable&.class}") + raise Gitlab::ImportExport::Error, "Unsupported Exportable Type #{@exportable&.class}" end end diff --git a/lib/gitlab/import_export/snippet_repo_restorer.rb b/lib/gitlab/import_export/snippet_repo_restorer.rb index 2d0aa05fc3c..cb13972f8f2 100644 --- a/lib/gitlab/import_export/snippet_repo_restorer.rb +++ b/lib/gitlab/import_export/snippet_repo_restorer.rb @@ -23,7 +23,7 @@ module Gitlab end true - rescue => e + rescue StandardError => e shared.error(e) false end diff --git a/lib/gitlab/import_export/statistics_restorer.rb b/lib/gitlab/import_export/statistics_restorer.rb index 3fafb01c37c..a3ad5edc2cc 100644 --- a/lib/gitlab/import_export/statistics_restorer.rb +++ b/lib/gitlab/import_export/statistics_restorer.rb @@ -10,7 +10,7 @@ module Gitlab def restore @project.statistics.refresh! - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index 2f15cdd7506..ad19508fb99 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -17,7 +17,7 @@ module Gitlab copy_project_uploads true - rescue => e + rescue StandardError => e @shared.error(e) false end @@ -30,7 +30,7 @@ module Gitlab end true - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb index 5f422dcbefa..741c6555aad 100644 --- a/lib/gitlab/import_export/uploads_restorer.rb +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -8,7 +8,7 @@ module Gitlab project: @project, shared: @shared ).restore - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index be1066c30b2..9f58609fa17 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -13,7 +13,7 @@ module Gitlab project: @project, shared: @shared ).save - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index 48f5b558e52..5ec9db00d0a 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -14,7 +14,7 @@ module Gitlab def check! version = File.open(version_file, &:readline) verify_version!(version) - rescue => e + rescue StandardError => e @shared.error(e) false end @@ -27,7 +27,7 @@ module Gitlab def verify_version!(version) if different_version?(version) - raise Gitlab::ImportExport::Error.new("Import version mismatch: Required #{Gitlab::ImportExport.version} but was #{version}") + raise Gitlab::ImportExport::Error, "Import version mismatch: Required #{Gitlab::ImportExport.version} but was #{version}" else true end @@ -35,13 +35,13 @@ module Gitlab def different_version?(version) Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) - rescue => e + rescue StandardError => e Gitlab::Import::Logger.error( message: 'Import error', error: e.message ) - raise Gitlab::ImportExport::Error.new('Incorrect VERSION format') + raise Gitlab::ImportExport::Error, 'Incorrect VERSION format' end end end diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index dab8bbf539d..e8f68f93af0 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -15,7 +15,7 @@ module Gitlab File.write(version_file, Gitlab::ImportExport.version, mode: 'w') File.write(gitlab_version_file, Gitlab::VERSION, mode: 'w') File.write(gitlab_revision_file, Gitlab.revision, mode: 'w') - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 4b1cf4915e4..162b474bfeb 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -20,4 +20,4 @@ module Gitlab end end -Gitlab::ImportExport::WikiRepoSaver.prepend_if_ee('EE::Gitlab::ImportExport::WikiRepoSaver') +Gitlab::ImportExport::WikiRepoSaver.prepend_mod_with('Gitlab::ImportExport::WikiRepoSaver') diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 95c002edf0a..c9f5005cede 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -25,7 +25,7 @@ module Gitlab ].freeze class << self - prepend_if_ee('EE::Gitlab::ImportSources') # rubocop: disable Cop/InjectEnterpriseEditionModule + prepend_mod_with('Gitlab::ImportSources') # rubocop: disable Cop/InjectEnterpriseEditionModule def options import_table.to_h { |importer| [importer.title, importer.name] } diff --git a/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb b/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb index 768c8bb4cbb..6aeeb1d31aa 100644 --- a/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb +++ b/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb @@ -33,7 +33,7 @@ module Gitlab def incident_created_at Time.zone.parse(incident_payload['created_at']) - rescue + rescue StandardError Time.current.utc # PagerDuty provides time in UTC end diff --git a/lib/gitlab/instrumentation/redis_cluster_validator.rb b/lib/gitlab/instrumentation/redis_cluster_validator.rb index 644a5fc4fff..005751fb0db 100644 --- a/lib/gitlab/instrumentation/redis_cluster_validator.rb +++ b/lib/gitlab/instrumentation/redis_cluster_validator.rb @@ -62,7 +62,7 @@ module Gitlab end if key_slots.uniq.many? # rubocop: disable CodeReuse/ActiveRecord - raise CrossSlotError.new("Redis command #{command_name} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands") + raise CrossSlotError, "Redis command #{command_name} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands" end end diff --git a/lib/gitlab/integrations/sti_type.rb b/lib/gitlab/integrations/sti_type.rb new file mode 100644 index 00000000000..e6ea98e6d66 --- /dev/null +++ b/lib/gitlab/integrations/sti_type.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Integrations + class StiType < ActiveRecord::Type::String + NAMESPACED_INTEGRATIONS = Set.new(%w( + Asana Assembla Bamboo Campfire Confluence Datadog EmailsOnPush + )).freeze + + def cast(value) + new_cast(value) || super + end + + def serialize(value) + new_serialize(value) || super + end + + def deserialize(value) + value + end + + def changed?(original_value, value, _new_value_before_type_cast) + original_value != serialize(value) + end + + def changed_in_place?(original_value_for_database, value) + original_value_for_database != serialize(value) + end + + private + + def new_cast(value) + value = prepare_value(value) + return unless value + + stripped_name = value.delete_suffix('Service') + return unless NAMESPACED_INTEGRATIONS.include?(stripped_name) + + "Integrations::#{stripped_name}" + end + + def new_serialize(value) + value = prepare_value(value) + return unless value&.starts_with?('Integrations::') + + "#{value.delete_prefix('Integrations::')}Service" + end + + # Returns value cast to a `String`, or `nil` if value is `nil`. + def prepare_value(value) + return value if value.nil? || value.is_a?(String) + + value.to_s + end + end + end +end diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb index f96c937aec3..d0702fb5c7d 100644 --- a/lib/gitlab/issuable_metadata.rb +++ b/lib/gitlab/issuable_metadata.rb @@ -98,4 +98,4 @@ module Gitlab end end -Gitlab::IssuableMetadata.prepend_if_ee('EE::Gitlab::IssuableMetadata') +Gitlab::IssuableMetadata.prepend_mod_with('Gitlab::IssuableMetadata') diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb index f0b08bb6b6a..3e7659db240 100644 --- a/lib/gitlab/jira/http_client.rb +++ b/lib/gitlab/jira/http_client.rb @@ -12,7 +12,7 @@ module Gitlab def request(*args) result = make_request(*args) - raise JIRA::HTTPError.new(result.response) unless result.response.is_a?(Net::HTTPSuccess) + raise JIRA::HTTPError, result.response unless result.response.is_a?(Net::HTTPSuccess) result end diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb index 26fa01755d1..8a03162f111 100644 --- a/lib/gitlab/jira_import/issues_importer.rb +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -70,7 +70,7 @@ module Gitlab # These ids are cleaned-up when import finishes. # see Gitlab::JiraImport::Stage::FinishImportWorker mark_as_imported(jira_issue.id) - rescue => ex + rescue StandardError => ex # handle exceptionn here and skip the failed to import issue, instead of # failing to import the entire batch of issues diff --git a/lib/gitlab/jira_import/labels_importer.rb b/lib/gitlab/jira_import/labels_importer.rb index 6e6842e06bf..046dc3fd04f 100644 --- a/lib/gitlab/jira_import/labels_importer.rb +++ b/lib/gitlab/jira_import/labels_importer.rb @@ -47,7 +47,7 @@ module Gitlab Gitlab::JiraImport::HandleLabelsService.new(project, response['values']).execute response['isLast'] - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, project_id: project.id, request: request) end end diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index b51c0a33457..561cd4509b1 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -84,7 +84,7 @@ module Gitlab Oj.load(string, opts) rescue Oj::ParseError, Encoding::UndefinedConversionError => ex - raise parser_error.new(ex) + raise parser_error, ex end # Take a Ruby object and convert it to a string. This method varies @@ -169,7 +169,7 @@ module Gitlab # @return [Boolean] def feature_table_exists? Feature::FlipperFeature.table_exists? - rescue + rescue StandardError false end end diff --git a/lib/gitlab/jwt_token.rb b/lib/gitlab/jwt_token.rb new file mode 100644 index 00000000000..11bc5479b6e --- /dev/null +++ b/lib/gitlab/jwt_token.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + class JWTToken < JSONWebToken::HMACToken + HMAC_ALGORITHM = 'SHA256' + HMAC_KEY = 'gitlab-jwt' + HMAC_EXPIRES_IN = 5.minutes.freeze + + class << self + def decode(jwt) + payload = super(jwt, secret).first + + new.tap do |jwt_token| + jwt_token.id = payload.delete('jti') + jwt_token.issued_at = payload.delete('iat') + jwt_token.not_before = payload.delete('nbf') + jwt_token.expire_time = payload.delete('exp') + + payload.each do |key, value| + jwt_token[key] = value + end + end + rescue JWT::DecodeError, JWT::ExpiredSignature, JWT::ImmatureSignature => ex + # we want to log and return on expired and errored tokens + Gitlab::ErrorTracking.track_exception(ex) + nil + end + + def secret + OpenSSL::HMAC.hexdigest( + HMAC_ALGORITHM, + ::Settings.attr_encrypted_db_key_base, + HMAC_KEY + ) + end + end + + def initialize + super(self.class.secret) + self.expire_time = self.issued_at + HMAC_EXPIRES_IN.to_i + end + + def ==(other) + self.id == other.id && + self.payload == other.payload + end + + def issued_at=(value) + super(convert_time(value)) + end + + def not_before=(value) + super(convert_time(value)) + end + + def expire_time=(value) + super(convert_time(value)) + end + + private + + def convert_time(value) + # JSONWebToken::Token truncates subsecond precision causing comparisons to + # fail unless we truncate it here first + value = value.to_i if value.is_a?(Float) + value = Time.zone.at(value) if value.is_a?(Integer) + value + end + end +end diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index 7a674cb5c21..7b2c792ebca 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -3,6 +3,7 @@ module Gitlab module Kas INTERNAL_API_REQUEST_HEADER = 'Gitlab-Kas-Api-Request' + VERSION_FILE = 'GITLAB_KAS_VERSION' JWT_ISSUER = 'gitlab-kas' include JwtAuthenticatable @@ -29,6 +30,27 @@ module Gitlab Feature.enabled?(:kubernetes_agent_on_gitlab_com, project, default_enabled: :yaml) end + + # Return GitLab KAS version + # + # @return [String] version + def version + @_version ||= Rails.root.join(VERSION_FILE).read.chomp + end + + # Return GitLab KAS external_url + # + # @return [String] external_url + def external_url + Gitlab.config.gitlab_kas.external_url + end + + # Return whether GitLab KAS is enabled + # + # @return [Boolean] external_url + def enabled? + !!Gitlab.config['gitlab_kas']&.fetch('enabled', false) + end end end end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index a25f005d81e..6caebf445e5 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -116,7 +116,7 @@ module Gitlab { status: :authentication_failure, connection_error: :authentication_error } rescue Kubeclient::HttpError => e { status: kubeclient_error_status(e.message), connection_error: :http_error } - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, cluster_id: cluster_id) { status: :unknown_failure, connection_error: :unknown_error } diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index a17e3b1ad5c..fc5834613fd 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -94,7 +94,7 @@ module Gitlab labels.each do |raw| gh_label = LabelFormatter.new(project, raw) gh_label.create! - rescue => e + rescue StandardError => e errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(gh_label.url), errors: e.message } end end @@ -107,7 +107,7 @@ module Gitlab milestones.each do |raw| gh_milestone = MilestoneFormatter.new(project, raw) gh_milestone.create! - rescue => e + rescue StandardError => e errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(gh_milestone.url), errors: e.message } end end @@ -128,7 +128,7 @@ module Gitlab end apply_labels(issuable, raw) - rescue => e + rescue StandardError => e errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(gh_issue.url), errors: e.message } end end @@ -153,7 +153,7 @@ module Gitlab if project.gitea_import? apply_labels(merge_request, raw) end - rescue => e + rescue StandardError => e errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), errors: e.message } ensure clean_up_restored_branches(gh_pull_request) @@ -236,7 +236,7 @@ module Gitlab next unless issuable issuable.notes.create!(comment.attributes) - rescue => e + rescue StandardError => e errors << { type: :comment, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } end end @@ -280,7 +280,7 @@ module Gitlab releases.each do |raw| gh_release = ReleaseFormatter.new(project, raw) gh_release.create! if gh_release.valid? - rescue => e + rescue StandardError => e errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(gh_release.url), errors: e.message } end end diff --git a/lib/gitlab/legacy_github_import/label_formatter.rb b/lib/gitlab/legacy_github_import/label_formatter.rb index 0b6e4612843..415b1b8878f 100644 --- a/lib/gitlab/legacy_github_import/label_formatter.rb +++ b/lib/gitlab/legacy_github_import/label_formatter.rb @@ -20,7 +20,7 @@ module Gitlab service = ::Labels::FindOrCreateService.new(nil, project, params) label = service.execute(skip_authorization: true) - raise ActiveRecord::RecordInvalid.new(label) unless label.persisted? + raise ActiveRecord::RecordInvalid, label unless label.persisted? label end diff --git a/lib/gitlab/lfs/client.rb b/lib/gitlab/lfs/client.rb index 825d7399190..a05e8107cad 100644 --- a/lib/gitlab/lfs/client.rb +++ b/lib/gitlab/lfs/client.rb @@ -43,7 +43,7 @@ module Gitlab body = Gitlab::Json.parse(rsp.body) transfer = body.fetch('transfer', 'basic') - raise UnsupportedTransferError.new(transfer.inspect) unless transfer == 'basic' + raise UnsupportedTransferError, transfer.inspect unless transfer == 'basic' body end @@ -97,7 +97,10 @@ module Gitlab end def basic_auth - return unless credentials[:auth_method] == "password" + # Some legacy credentials have a nil auth_method, which means password + # https://gitlab.com/gitlab-org/gitlab/-/issues/328674 + return unless credentials.fetch(:auth_method, 'password') == 'password' + return if credentials.empty? { username: credentials[:user], password: credentials[:password] } end diff --git a/lib/gitlab/local_and_remote_storage_migration/artifact_migrater.rb b/lib/gitlab/local_and_remote_storage_migration/artifact_migrater.rb new file mode 100644 index 00000000000..b25305382b2 --- /dev/null +++ b/lib/gitlab/local_and_remote_storage_migration/artifact_migrater.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module LocalAndRemoteStorageMigration + class ArtifactMigrater < Gitlab::LocalAndRemoteStorageMigration::BaseMigrater + private + + def items_with_files_stored_locally + ::Ci::JobArtifact.with_files_stored_locally + end + + def items_with_files_stored_remotely + ::Ci::JobArtifact.with_files_stored_remotely + end + end + end +end diff --git a/lib/gitlab/local_and_remote_storage_migration/base_migrater.rb b/lib/gitlab/local_and_remote_storage_migration/base_migrater.rb new file mode 100644 index 00000000000..f859d293e76 --- /dev/null +++ b/lib/gitlab/local_and_remote_storage_migration/base_migrater.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module LocalAndRemoteStorageMigration + class BaseMigrater + def initialize(logger = nil) + @logger = logger + end + + def migrate_to_remote_storage + logger.info('Starting transfer to remote storage') + + migrate(items_with_files_stored_locally, ObjectStorage::Store::REMOTE) + end + + def migrate_to_local_storage + logger.info('Starting transfer to local storage') + + migrate(items_with_files_stored_remotely, ObjectStorage::Store::LOCAL) + end + + private + + attr_reader :logger + + def batch_size + ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i + end + + def migrate(items, store) + items.find_each(batch_size: batch_size) do |item| # rubocop:disable CodeReuse/ActiveRecord + item.file.migrate!(store) + + log_success(item, store) + rescue StandardError => e + log_error(e, item) + end + end + + def log_success(item, store) + logger.info("Transferred #{item.class.name} ID #{item.id} of type #{item.file_type} with size #{item.size} to #{storage_label(store)} storage") + end + + def log_error(err, item) + logger.warn("Failed to transfer #{item.class.name} of type #{item.file_type} and ID #{item.id} with error: #{err.message}") + end + + def storage_label(store) + if store == ObjectStorage::Store::LOCAL + 'local' + else + 'object' + end + end + end + end +end diff --git a/lib/gitlab/local_and_remote_storage_migration/pages_deployment_migrater.rb b/lib/gitlab/local_and_remote_storage_migration/pages_deployment_migrater.rb new file mode 100644 index 00000000000..70437936332 --- /dev/null +++ b/lib/gitlab/local_and_remote_storage_migration/pages_deployment_migrater.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module LocalAndRemoteStorageMigration + class PagesDeploymentMigrater < Gitlab::LocalAndRemoteStorageMigration::BaseMigrater + private + + def items_with_files_stored_locally + ::PagesDeployment.with_files_stored_locally + end + + def items_with_files_stored_remotely + ::PagesDeployment.with_files_stored_remotely + end + end + end +end diff --git a/lib/gitlab/markdown_cache.rb b/lib/gitlab/markdown_cache.rb index 3ec5f2339b5..d0702190ac0 100644 --- a/lib/gitlab/markdown_cache.rb +++ b/lib/gitlab/markdown_cache.rb @@ -2,8 +2,12 @@ module Gitlab module MarkdownCache - # Increment this number every time the renderer changes its output - CACHE_COMMONMARK_VERSION = 27 + # Increment this number every time the renderer changes its output. + # Changing this value puts strain on the database, as every row with + # cached markdown needs to be updated. As a result, this line should + # not be changed. + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330313 + CACHE_COMMONMARK_VERSION = 28 CACHE_COMMONMARK_VERSION_START = 10 BaseError = Class.new(StandardError) diff --git a/lib/gitlab/memory/instrumentation.rb b/lib/gitlab/memory/instrumentation.rb index 8f9f6d19ce8..e800fe14cf1 100644 --- a/lib/gitlab/memory/instrumentation.rb +++ b/lib/gitlab/memory/instrumentation.rb @@ -45,9 +45,12 @@ module Gitlab end # This method returns a hash with the following keys: - # - mem_objects: a number of allocated heap slots (as reflected by GC) - # - mem_mallocs: a number of malloc calls - # - mem_bytes: a number of bytes allocated with a mallocs tied to heap slots + # - mem_objects: number of allocated heap slots (as reflected by GC) + # - mem_mallocs: number of malloc calls + # - mem_bytes: number of bytes allocated by malloc for objects that did not fit + # into a heap slot + # - mem_total_bytes: number of bytes allocated for both objects consuming an object slot + # and objects that required a malloc (mem_malloc_bytes) def self.measure_thread_memory_allocations(previous) return unless available? return unless previous @@ -56,9 +59,13 @@ module Gitlab return unless current # calculate difference in a memory allocations - previous.to_h do |key, value| + result = previous.to_h do |key, value| [KEY_MAPPING.fetch(key), current[key].to_i - value] end + + result[:mem_total_bytes] = result[:mem_bytes] + result[:mem_objects] * GC::INTERNAL_CONSTANTS[:RVALUE_SIZE] + + result end def self.with_memory_allocations diff --git a/lib/gitlab/metrics/dashboard/errors.rb b/lib/gitlab/metrics/dashboard/errors.rb index 07ddd315bcc..1a951172f74 100644 --- a/lib/gitlab/metrics/dashboard/errors.rb +++ b/lib/gitlab/metrics/dashboard/errors.rb @@ -33,7 +33,7 @@ module Gitlab end def panels_not_found!(opts) - raise PanelNotFoundError.new(_("No panels matching properties %{opts}") % { opts: opts }) + raise PanelNotFoundError, _("No panels matching properties %{opts}") % { opts: opts } end end end diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb index ee2d36621b4..c2a8a88108f 100644 --- a/lib/gitlab/metrics/dashboard/stages/base_stage.rb +++ b/lib/gitlab/metrics/dashboard/stages/base_stage.rb @@ -23,15 +23,15 @@ module Gitlab protected def missing_panel_groups! - raise Errors::LayoutError.new('Top-level key :panel_groups must be an array') + raise Errors::LayoutError, 'Top-level key :panel_groups must be an array' end def missing_panels! - raise Errors::LayoutError.new('Each "panel_group" must define an array :panels') + raise Errors::LayoutError, 'Each "panel_group" must define an array :panels' end def missing_metrics! - raise Errors::LayoutError.new('Each "panel" must define an array :metrics') + raise Errors::LayoutError, 'Each "panel" must define an array :metrics' end def for_metrics diff --git a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb index a12082b704c..2c17982d299 100644 --- a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb @@ -39,7 +39,7 @@ module Gitlab end def error!(message) - raise Errors::DashboardProcessingError.new(message) + raise Errors::DashboardProcessingError, message end def group_url(metric) @@ -67,14 +67,14 @@ module Gitlab def query_for_metric(metric) query = metric[query_type(metric)] - raise Errors::MissingQueryError.new('Each "metric" must define one of :query or :query_range') unless query + raise Errors::MissingQueryError, 'Each "metric" must define one of :query or :query_range' unless query query end def verify_params - raise Errors::DashboardProcessingError.new(_('Cluster is required for Stages::ClusterEndpointInserter')) unless params[:cluster] - raise Errors::DashboardProcessingError.new(_('Cluster type must be specificed for Stages::ClusterEndpointInserter')) unless params[:cluster_type] + raise Errors::DashboardProcessingError, _('Cluster is required for Stages::ClusterEndpointInserter') unless params[:cluster] + raise Errors::DashboardProcessingError, _('Cluster type must be specificed for Stages::ClusterEndpointInserter') unless params[:cluster_type] end end end diff --git a/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb index dd85bd0beb1..d885d978524 100644 --- a/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb @@ -6,7 +6,7 @@ module Gitlab module Stages class MetricEndpointInserter < BaseStage def transform! - raise Errors::DashboardProcessingError.new(_('Environment is required for Stages::MetricEndpointInserter')) unless params[:environment] + raise Errors::DashboardProcessingError, _('Environment is required for Stages::MetricEndpointInserter') unless params[:environment] for_metrics do |metric| metric[:prometheus_endpoint_path] = endpoint_for_metric(metric) @@ -43,7 +43,7 @@ module Gitlab def query_for_metric(metric) query = metric[query_type(metric)] - raise Errors::MissingQueryError.new('Each "metric" must define one of :query or :query_range') unless query + raise Errors::MissingQueryError, 'Each "metric" must define one of :query or :query_range' unless query # We need to remove any newlines since our UrlBlocker does not allow # multiline URLs. diff --git a/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb index 20e7fe477e5..b3ce0b79675 100644 --- a/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb @@ -8,7 +8,7 @@ module Gitlab VARIABLE_TYPE_METRIC_LABEL_VALUES = 'metric_label_values' def transform! - raise Errors::DashboardProcessingError.new(_('Environment is required for Stages::VariableEndpointInserter')) unless params[:environment] + raise Errors::DashboardProcessingError, _('Environment is required for Stages::VariableEndpointInserter') unless params[:environment] for_variables do |variable_name, variable| if variable.is_a?(Hash) && variable[:type] == VARIABLE_TYPE_METRIC_LABEL_VALUES diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index 23d7eb67312..19a835b9fc4 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -83,7 +83,7 @@ module Gitlab end [status, headers, body] - rescue + rescue StandardError RequestsRackMiddleware.rack_uncaught_errors_count.increment raise ensure diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index 7f9055fed5d..258aa93be38 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -22,7 +22,7 @@ module Gitlab def safe_sample sample - rescue => e + rescue StandardError => e Gitlab::AppLogger.warn("#{self.class}: #{e}, stopping") stop end diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb index c0336a4d0fb..0a0ac6c5386 100644 --- a/lib/gitlab/metrics/samplers/database_sampler.rb +++ b/lib/gitlab/metrics/samplers/database_sampler.rb @@ -55,4 +55,4 @@ module Gitlab end end -Gitlab::Metrics::Samplers::DatabaseSampler.prepend_if_ee('EE::Gitlab::Metrics::Samplers::DatabaseSampler') +Gitlab::Metrics::Samplers::DatabaseSampler.prepend_mod_with('Gitlab::Metrics::Samplers::DatabaseSampler') diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 0d1cd641ffe..3db3317e833 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -87,4 +87,4 @@ module Gitlab end end -Gitlab::Metrics::Subscribers::ActiveRecord.prepend_if_ee('EE::Gitlab::Metrics::Subscribers::ActiveRecord') +Gitlab::Metrics::Subscribers::ActiveRecord.prepend_mod_with('Gitlab::Metrics::Subscribers::ActiveRecord') diff --git a/lib/gitlab/metrics/subscribers/rack_attack.rb b/lib/gitlab/metrics/subscribers/rack_attack.rb index 2791a39fb16..1c7767f5ca9 100644 --- a/lib/gitlab/metrics/subscribers/rack_attack.rb +++ b/lib/gitlab/metrics/subscribers/rack_attack.rb @@ -19,7 +19,8 @@ module Gitlab :throttle_authenticated_api, :throttle_authenticated_web, :throttle_authenticated_protected_paths_api, - :throttle_authenticated_protected_paths_web + :throttle_authenticated_protected_paths_web, + :throttle_authenticated_packages_api ].freeze PAYLOAD_KEYS = [ diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb index 1811389a744..ee9e6f449d3 100644 --- a/lib/gitlab/metrics/web_transaction.rb +++ b/lib/gitlab/metrics/web_transaction.rb @@ -57,7 +57,7 @@ module Gitlab begin route = endpoint.route - rescue + rescue StandardError # endpoint.route is calling env[Grape::Env::GRAPE_ROUTING_ARGS][:route_info] # but env[Grape::Env::GRAPE_ROUTING_ARGS] is nil in the case of a 405 response # so we're rescuing exceptions and bailing out diff --git a/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb b/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb index d16c068c3c0..1686c3324b4 100644 --- a/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb +++ b/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb @@ -14,9 +14,7 @@ module Gitlab end def call(env) - if ENV['GITLAB_TEMPFILE_IMMEDIATE_UNLINK'] == '1' - env[Rack::RACK_MULTIPART_TEMPFILE_FACTORY] = FACTORY - end + env[Rack::RACK_MULTIPART_TEMPFILE_FACTORY] = FACTORY @app.call(env) end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index 226ef2041b2..65c08664a2b 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -153,4 +153,4 @@ module Gitlab end end -Gitlab::Middleware::ReadOnly::Controller.prepend_if_ee('EE::Gitlab::Middleware::ReadOnly::Controller') +Gitlab::Middleware::ReadOnly::Controller.prepend_mod_with('Gitlab::Middleware::ReadOnly::Controller') diff --git a/lib/gitlab/middleware/speedscope.rb b/lib/gitlab/middleware/speedscope.rb new file mode 100644 index 00000000000..74f334d9ab3 --- /dev/null +++ b/lib/gitlab/middleware/speedscope.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module Middleware + class Speedscope + def initialize(app) + @app = app + end + + def call(env) + request = ActionDispatch::Request.new(env) + + return @app.call(env) unless rendering_flamegraph?(request) + + body = nil + + ::Gitlab::SafeRequestStore[:capturing_flamegraph] = true + + require 'stackprof' + + begin + flamegraph = ::StackProf.run( + mode: :wall, + raw: true, + aggregate: false, + interval: ::Gitlab::StackProf::DEFAULT_INTERVAL_US + ) do + _, _, body = @app.call(env) + end + ensure + body.close if body.respond_to?(:close) + end + + render_flamegraph(flamegraph, request) + end + + private + + def rendering_flamegraph?(request) + request.params['performance_bar'] == 'flamegraph' && ::Gitlab::PerformanceBar.allowed_for_user?(request.env['warden']&.user) + end + + def render_flamegraph(graph, request) + headers = { 'Content-Type' => 'text/html' } + path = request.env['PATH_INFO'].sub('//', '/') + + speedscope_path = ::Gitlab::Utils.append_path(::Gitlab.config.gitlab.relative_url_root, '/-/speedscope/index.html') + + html = <<~HTML + <!DOCTYPE html> + <html> + <head> + <style> + body { margin: 0; height: 100vh; } + #speedscope-iframe { width: 100%; height: 100%; border: none; } + </style> + </head> + <body> + <script type="text/javascript" nonce="#{request.content_security_policy_nonce}"> + var graph = #{Gitlab::Json.generate(graph)}; + var json = JSON.stringify(graph); + var blob = new Blob([json], { type: 'text/plain' }); + var objUrl = encodeURIComponent(URL.createObjectURL(blob)); + var iframe = document.createElement('IFRAME'); + iframe.setAttribute('id', 'speedscope-iframe'); + document.body.appendChild(iframe); + var iframeUrl = '#{speedscope_path}#profileURL=' + objUrl + '&title=' + 'Flamegraph for #{CGI.escape(path)}'; + iframe.setAttribute('src', iframeUrl); + </script> + </body> + </html> + HTML + + [200, headers, [html]] + end + end + end +end diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb index 002171854ad..87cc0a0d3d2 100644 --- a/lib/gitlab/multi_collection_paginator.rb +++ b/lib/gitlab/multi_collection_paginator.rb @@ -5,7 +5,7 @@ module Gitlab attr_reader :first_collection, :second_collection, :per_page def initialize(*collections, per_page: nil) - raise ArgumentError.new('Only 2 collections are supported') if collections.size != 2 + raise ArgumentError, 'Only 2 collections are supported' if collections.size != 2 @per_page = (per_page || Kaminari.config.default_per_page).to_i @first_collection, @second_collection = collections diff --git a/lib/gitlab/nav/top_nav_menu_builder.rb b/lib/gitlab/nav/top_nav_menu_builder.rb new file mode 100644 index 00000000000..721ae1889b8 --- /dev/null +++ b/lib/gitlab/nav/top_nav_menu_builder.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Nav + class TopNavMenuBuilder + def initialize + @primary = [] + @secondary = [] + end + + def add_primary_menu_item(**args) + add_menu_item(dest: @primary, **args) + end + + def add_secondary_menu_item(**args) + add_menu_item(dest: @secondary, **args) + end + + def build + { + primary: @primary, + secondary: @secondary + } + end + + private + + def add_menu_item(dest:, **args) + item = ::Gitlab::Nav::TopNavMenuItem.build(**args) + + dest.push(item) + end + end + end +end diff --git a/lib/gitlab/nav/top_nav_menu_item.rb b/lib/gitlab/nav/top_nav_menu_item.rb new file mode 100644 index 00000000000..ee11f1f4560 --- /dev/null +++ b/lib/gitlab/nav/top_nav_menu_item.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Nav + class TopNavMenuItem + # We want to have all keyword arguments for type safety. + # Ordinarily we could introduce a params object, but that's kind of what + # this is already :/. We could also take a hash and manually check every + # entry, but it's much more maintainable to do rely on native Ruby. + # rubocop: disable Metrics/ParameterLists + def self.build(id:, title:, active: false, icon: '', href: '', method: nil, view: '', css_class: '', data: {}) + { + id: id, + title: title, + active: active, + icon: icon, + href: href, + method: method, + view: view.to_s, + css_class: css_class, + data: data + } + end + # rubocop: enable Metrics/ParameterLists + end + end +end diff --git a/lib/gitlab/nav/top_nav_view_model_builder.rb b/lib/gitlab/nav/top_nav_view_model_builder.rb new file mode 100644 index 00000000000..60f5b267071 --- /dev/null +++ b/lib/gitlab/nav/top_nav_view_model_builder.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Nav + class TopNavViewModelBuilder + def initialize + @menu_builder = ::Gitlab::Nav::TopNavMenuBuilder.new + @views = {} + end + + delegate :add_primary_menu_item, :add_secondary_menu_item, to: :@menu_builder + + def add_view(name, props) + @views[name] = props + end + + def build + menu = @menu_builder.build + + menu.merge({ + views: @views, + activeTitle: _('Menu') + }) + end + end + end +end diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index 9a74266693b..e6e7d97d296 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -7,18 +7,19 @@ module Gitlab class ObjectHierarchy DEPTH_COLUMN = :depth - attr_reader :ancestors_base, :descendants_base, :model, :options + attr_reader :ancestors_base, :descendants_base, :model, :options, :unscoped_model # ancestors_base - An instance of ActiveRecord::Relation for which to # get parent objects. # descendants_base - An instance of ActiveRecord::Relation for which to # get child objects. If omitted, ancestors_base is used. def initialize(ancestors_base, descendants_base = ancestors_base, options: {}) - raise ArgumentError.new("Model of ancestors_base does not match model of descendants_base") if ancestors_base.model != descendants_base.model + raise ArgumentError, "Model of ancestors_base does not match model of descendants_base" if ancestors_base.model != descendants_base.model @ancestors_base = ancestors_base @descendants_base = descendants_base @model = ancestors_base.model + @unscoped_model = @model.unscoped @options = options end @@ -70,23 +71,23 @@ module Gitlab # if hierarchy_order is given, the calculated `depth` should be present in SELECT if expose_depth - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all).distinct - read_only(model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)).order(depth: hierarchy_order)) + recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(unscoped_model.all).distinct + read_only(unscoped_model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)).order(depth: hierarchy_order)) else - recursive_query = base_and_ancestors_cte(upto).apply_to(model.all) + recursive_query = base_and_ancestors_cte(upto).apply_to(unscoped_model.all) if skip_ordering? recursive_query = recursive_query.distinct else recursive_query = recursive_query.reselect(*recursive_query.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct - recursive_query = model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)) + recursive_query = unscoped_model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)) recursive_query = remove_depth_and_maintain_order(recursive_query, hierarchy_order: hierarchy_order) end read_only(recursive_query) end else - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all) + recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(unscoped_model.all) recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order read_only(recursive_query) end @@ -103,23 +104,23 @@ module Gitlab if use_distinct? # Always calculate `depth`, remove it later if with_depth is false if with_depth - base_cte = base_and_descendants_cte(with_depth: true).apply_to(model.all).distinct - read_only(model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)).order(depth: :asc)) + base_cte = base_and_descendants_cte(with_depth: true).apply_to(unscoped_model.all).distinct + read_only(unscoped_model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)).order(depth: :asc)) else - base_cte = base_and_descendants_cte.apply_to(model.all) + base_cte = base_and_descendants_cte.apply_to(unscoped_model.all) if skip_ordering? base_cte = base_cte.distinct else base_cte = base_cte.reselect(*base_cte.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct - base_cte = model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)) + base_cte = unscoped_model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)) base_cte = remove_depth_and_maintain_order(base_cte, hierarchy_order: :asc) end read_only(base_cte) end else - read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(model.all)) + read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(unscoped_model.all)) end end # rubocop: enable CodeReuse/ActiveRecord @@ -154,16 +155,15 @@ module Gitlab ancestors_table = ancestors.alias_to(objects_table) descendants_table = descendants.alias_to(objects_table) - ancestors_scope = model.unscoped.from(ancestors_table) - descendants_scope = model.unscoped.from(descendants_table) + ancestors_scope = unscoped_model.from(ancestors_table) + descendants_scope = unscoped_model.from(descendants_table) if use_distinct? ancestors_scope = ancestors_scope.distinct descendants_scope = descendants_scope.distinct end - relation = model - .unscoped + relation = unscoped_model .with .recursive(ancestors.to_arel, descendants.to_arel) .from_union([ @@ -215,7 +215,7 @@ module Gitlab cte << base_query # Recursively get all the ancestors of the base set. - parent_query = model + parent_query = unscoped_model .from(from_tables(cte)) .where(ancestor_conditions(cte)) .except(:order) @@ -248,7 +248,7 @@ module Gitlab cte << base_query # Recursively get all the descendants of the base set. - descendants_query = model + descendants_query = unscoped_model .from(from_tables(cte)) .where(descendant_conditions(cte)) .except(:order) @@ -298,4 +298,4 @@ module Gitlab end end -Gitlab::ObjectHierarchy.prepend_if_ee('EE::Gitlab::ObjectHierarchy') +Gitlab::ObjectHierarchy.prepend_mod_with('Gitlab::ObjectHierarchy') diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index 541f9b06842..3e14e1789bb 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -123,4 +123,4 @@ module Gitlab end end -Gitlab::OmniauthInitializer.prepend_if_ee('::EE::Gitlab::OmniauthInitializer') +Gitlab::OmniauthInitializer.prepend_mod_with('Gitlab::OmniauthInitializer') diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb index 1d3200aa099..b65c8613d00 100644 --- a/lib/gitlab/otp_key_rotator.rb +++ b/lib/gitlab/otp_key_rotator.rb @@ -32,8 +32,8 @@ module Gitlab def rotate!(old_key:, new_key:) old_key ||= Gitlab::Application.secrets.otp_key_base - raise ArgumentError.new("Old key is the same as the new key") if old_key == new_key - raise ArgumentError.new("New key is too short! Must be 256 bits") if new_key.size < 64 + raise ArgumentError, "Old key is the same as the new key" if old_key == new_key + raise ArgumentError, "New key is too short! Must be 256 bits" if new_key.size < 64 write_csv do |csv| ActiveRecord::Base.transaction do diff --git a/lib/gitlab/pages/migration_helper.rb b/lib/gitlab/pages/migration_helper.rb deleted file mode 100644 index 8f8667fafd9..00000000000 --- a/lib/gitlab/pages/migration_helper.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Pages - class MigrationHelper - def initialize(logger = nil) - @logger = logger - end - - def migrate_to_remote_storage - deployments = ::PagesDeployment.with_files_stored_locally - migrate(deployments, ObjectStorage::Store::REMOTE) - end - - def migrate_to_local_storage - deployments = ::PagesDeployment.with_files_stored_remotely - migrate(deployments, ObjectStorage::Store::LOCAL) - end - - private - - def batch_size - ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i - end - - def migrate(deployments, store) - deployments.find_each(batch_size: batch_size) do |deployment| # rubocop:disable CodeReuse/ActiveRecord - deployment.file.migrate!(store) - - log_success(deployment, store) - rescue => e - log_error(e, deployment) - end - end - - def log_success(deployment, store) - logger.info("Transferred deployment ID #{deployment.id} of type #{deployment.file_type} with size #{deployment.size} to #{storage_label(store)} storage") - end - - def log_error(err, deployment) - logger.warn("Failed to transfer deployment of type #{deployment.file_type} and ID #{deployment.id} with error: #{err.message}") - end - - def storage_label(store) - if store == ObjectStorage::Store::LOCAL - 'local' - else - 'object' - end - end - end - end -end diff --git a/lib/gitlab/pages/settings.rb b/lib/gitlab/pages/settings.rb index be71018e851..b35683c9dec 100644 --- a/lib/gitlab/pages/settings.rb +++ b/lib/gitlab/pages/settings.rb @@ -11,10 +11,6 @@ module Gitlab super end - def local_store - @local_store ||= ::Gitlab::Pages::Stores::LocalStore.new(super) - end - private def disk_access_denied? @@ -25,7 +21,7 @@ module Gitlab def report_denied_disk_access raise DiskAccessDenied if disk_access_denied? - rescue => e + rescue StandardError => e ::Gitlab::ErrorTracking.track_exception(e) end end diff --git a/lib/gitlab/pages/stores/local_store.rb b/lib/gitlab/pages/stores/local_store.rb deleted file mode 100644 index 68a7ebaceff..00000000000 --- a/lib/gitlab/pages/stores/local_store.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Pages - module Stores - class LocalStore < ::SimpleDelegator - def enabled - return false unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true) - - super - end - end - end - end -end diff --git a/lib/gitlab/pagination/keyset/iterator.rb b/lib/gitlab/pagination/keyset/iterator.rb new file mode 100644 index 00000000000..3bc8c0bf616 --- /dev/null +++ b/lib/gitlab/pagination/keyset/iterator.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + class Iterator + def initialize(scope:, use_union_optimization: false) + @scope = scope + @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) + @use_union_optimization = use_union_optimization + end + + # rubocop: disable CodeReuse/ActiveRecord + def each_batch(of: 1000) + cursor_attributes = {} + + loop do + current_scope = scope.dup.limit(of) + relation = order + .apply_cursor_conditions(current_scope, cursor_attributes, { use_union_optimization: @use_union_optimization }) + .reorder(order) + .limit(of) + + yield relation + + last_record = relation.last + break unless last_record + + cursor_attributes = order.cursor_attributes_for_node(last_record) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + private + + attr_reader :scope, :order + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb index e596e1bac9d..cef3a7b291a 100644 --- a/lib/gitlab/pagination/keyset/order.rb +++ b/lib/gitlab/pagination/keyset/order.rb @@ -135,7 +135,7 @@ module Gitlab # # (id < 3 AND created_at IS NULL) OR (created_at IS NOT NULL) def build_where_values(values) - return if values.blank? + return [] if values.blank? verify_incoming_values!(values) @@ -156,13 +156,26 @@ module Gitlab end end - build_or_query(where_values) + where_values + end + + def where_values_with_or_query(values) + build_or_query(build_where_values(values.with_indifferent_access)) end # rubocop: disable CodeReuse/ActiveRecord - def apply_cursor_conditions(scope, values = {}) + def apply_cursor_conditions(scope, values = {}, options = { use_union_optimization: false }) + values ||= {} + transformed_values = values.with_indifferent_access scope = apply_custom_projections(scope) - scope.where(build_where_values(values.with_indifferent_access)) + + where_values = build_where_values(transformed_values) + + if options[:use_union_optimization] && where_values.size > 1 + build_union_query(scope, where_values).reorder(self) + else + scope.where(build_or_query(where_values)) # rubocop: disable CodeReuse/ActiveRecord + end end # rubocop: enable CodeReuse/ActiveRecord @@ -170,6 +183,8 @@ module Gitlab self.class.build(column_definitions.map(&:reverse)) end + alias_method :to_sql, :to_s + private # Adds extra columns to the SELECT clause @@ -210,11 +225,19 @@ module Gitlab end def build_or_query(expressions) - or_expression = expressions.reduce { |or_expression, expression| Arel::Nodes::Or.new(or_expression, expression) } + return [] if expressions.blank? + or_expression = expressions.reduce { |or_expression, expression| Arel::Nodes::Or.new(or_expression, expression) } Arel::Nodes::Grouping.new(or_expression) end + def build_union_query(scope, where_values) + scopes = where_values.map do |where_value| + scope.dup.where(where_value).reorder(self) # rubocop: disable CodeReuse/ActiveRecord + end + scope.model.from_union(scopes, remove_duplicates: false, remove_order: false) + end + def to_sql_literal(column_definitions) column_definitions.map do |column_definition| if column_definition.order_expression.respond_to?(:to_sql) diff --git a/lib/gitlab/pagination/keyset/simple_order_builder.rb b/lib/gitlab/pagination/keyset/simple_order_builder.rb new file mode 100644 index 00000000000..5ac5737c3be --- /dev/null +++ b/lib/gitlab/pagination/keyset/simple_order_builder.rb @@ -0,0 +1,137 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + # This class transforms the `order()` values from an Activerecord scope into a + # Gitlab::Pagination::Keyset::Order instance so the query later can be used in + # keyset pagination. + # + # Return values: + # [transformed_scope, true] # true indicates that the new scope was successfully built + # [orginal_scope, false] # false indicates that the order values are not supported in this class + class SimpleOrderBuilder + def self.build(scope) + new(scope: scope).build + end + + def initialize(scope:) + @scope = scope + @order_values = scope.order_values + @model_class = scope.model + @arel_table = @model_class.arel_table + @primary_key = @model_class.primary_key + end + + def build + order = if order_values.empty? + primary_key_descending_order + elsif ordered_by_primary_key? + primary_key_order + elsif ordered_by_other_column? + column_with_tie_breaker_order + elsif ordered_by_other_column_with_tie_breaker? + tie_breaker_attribute = order_values.second + + tie_breaker_column_order = Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: model_class.primary_key, + order_expression: tie_breaker_attribute + ) + + column_with_tie_breaker_order(tie_breaker_column_order) + end + + order ? [scope.reorder!(order), true] : [scope, false] # [scope, success] + end + + private + + attr_reader :scope, :order_values, :model_class, :arel_table, :primary_key + + def primary_key_descending_order + Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: model_class.primary_key, + order_expression: arel_table[primary_key].desc + ) + ]) + end + + def primary_key_order + Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: model_class.primary_key, + order_expression: order_values.first + ) + ]) + end + + def column_with_tie_breaker_order(tie_breaker_column_order = default_tie_breaker_column_order) + order_expression = order_values.first + attribute_name = order_expression.expr.name + + column_nullable = model_class.columns.find { |column| column.name == attribute_name }.null + + nullable = if column_nullable && order_expression.is_a?(Arel::Nodes::Ascending) + :nulls_last + elsif column_nullable && order_expression.is_a?(Arel::Nodes::Descending) + :nulls_first + else + :not_nullable + end + + Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: attribute_name, + order_expression: order_expression, + nullable: nullable, + distinct: false + ), + tie_breaker_column_order + ]) + end + + def ordered_by_primary_key? + return unless order_values.one? + + attribute = order_values.first.try(:expr) + + return unless attribute + + arel_table[primary_key].to_s == attribute.to_s + end + + def ordered_by_other_column? + return unless order_values.one? + + attribute = order_values.first.try(:expr) + + return unless attribute + return unless attribute.try(:name) + + model_class.column_names.include?(attribute.name.to_s) + end + + def ordered_by_other_column_with_tie_breaker? + return unless order_values.size == 2 + + attribute = order_values.first.try(:expr) + tie_breaker_attribute = order_values.second.try(:expr) + + return unless attribute + return unless tie_breaker_attribute + + model_class.column_names.include?(attribute.name.to_s) && + arel_table[primary_key].to_s == tie_breaker_attribute.to_s + end + + def default_tie_breaker_column_order + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: model_class.primary_key, + order_expression: arel_table[primary_key].desc + ) + end + end + end + end +end diff --git a/lib/gitlab/patch/draw_route.rb b/lib/gitlab/patch/draw_route.rb index f5fcd5c6093..61b25065e8f 100644 --- a/lib/gitlab/patch/draw_route.rb +++ b/lib/gitlab/patch/draw_route.rb @@ -10,7 +10,7 @@ module Gitlab def draw(routes_name) drawn_any = draw_ee(routes_name) | draw_ce(routes_name) - drawn_any || raise(RoutesNotFound.new("Cannot find #{routes_name}")) + drawn_any || raise(RoutesNotFound, "Cannot find #{routes_name}") end def draw_ce(routes_name) @@ -37,4 +37,4 @@ module Gitlab end end -Gitlab::Patch::DrawRoute.prepend_if_ee('EE::Gitlab::Patch::DrawRoute') +Gitlab::Patch::DrawRoute.prepend_mod_with('Gitlab::Patch::DrawRoute') diff --git a/lib/gitlab/patch/prependable.rb b/lib/gitlab/patch/prependable.rb index dde78cd9178..1ed341e1c26 100644 --- a/lib/gitlab/patch/prependable.rb +++ b/lib/gitlab/patch/prependable.rb @@ -21,7 +21,12 @@ module Gitlab def prepend_features(base) return false if prepended?(base) - super + # Rails 6.1 allows prepending of the modules, but it doesn't + # work well when both modules extend ActiveSupport::Concern + # https://github.com/rails/rails/pull/42067 + # + # Let's keep our own implementation, until the issue is fixed + Module.instance_method(:prepend_features).bind(self).call(base) if const_defined?(:ClassMethods) klass_methods = const_get(:ClassMethods, false) diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 2ff23980ebd..8618d2da77c 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -288,4 +288,4 @@ module Gitlab end end -Gitlab::PathRegex.prepend_if_ee('EE::Gitlab::PathRegex') +Gitlab::PathRegex.prepend_mod_with('Gitlab::PathRegex') diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index e26309b5dfd..10b13e7f55f 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -7,10 +7,10 @@ module Gitlab EXPIRY_TIME_L2_CACHE = 5.minutes def self.enabled_for_request? - Gitlab::SafeRequestStore[:peek_enabled] + !Gitlab::SafeRequestStore[:capturing_flamegraph] && Gitlab::SafeRequestStore[:peek_enabled] end - def self.enabled_for_user?(user = nil) + def self.allowed_for_user?(user = nil) return true if Rails.env.development? return true if user&.admin? return false unless user && allowed_group_id diff --git a/lib/gitlab/performance_bar/stats.rb b/lib/gitlab/performance_bar/stats.rb index c2a4602fd16..103cd65cb4b 100644 --- a/lib/gitlab/performance_bar/stats.rb +++ b/lib/gitlab/performance_bar/stats.rb @@ -20,7 +20,7 @@ module Gitlab return unless data log_sql_queries(id, data) - rescue => err + rescue StandardError => err logger.error(message: "failed to process request id #{id}: #{err.message}") end diff --git a/lib/gitlab/phabricator_import/conduit/client.rb b/lib/gitlab/phabricator_import/conduit/client.rb index 4469a3f5849..5945cde9618 100644 --- a/lib/gitlab/phabricator_import/conduit/client.rb +++ b/lib/gitlab/phabricator_import/conduit/client.rb @@ -13,7 +13,7 @@ module Gitlab Response.parse!(response) rescue *Gitlab::HTTP::HTTP_ERRORS => e # Wrap all errors from the API into an API-error. - raise ApiError.new(e) + raise ApiError, e end private diff --git a/lib/gitlab/phabricator_import/conduit/response.rb b/lib/gitlab/phabricator_import/conduit/response.rb index 1b03cfa05e6..26037ba183e 100644 --- a/lib/gitlab/phabricator_import/conduit/response.rb +++ b/lib/gitlab/phabricator_import/conduit/response.rb @@ -18,7 +18,7 @@ module Gitlab response rescue JSON::JSONError => e - raise ResponseError.new(e) + raise ResponseError, e end def initialize(json) diff --git a/lib/gitlab/phabricator_import/importer.rb b/lib/gitlab/phabricator_import/importer.rb index ac85b96de08..0666fa0df01 100644 --- a/lib/gitlab/phabricator_import/importer.rb +++ b/lib/gitlab/phabricator_import/importer.rb @@ -22,7 +22,7 @@ module Gitlab schedule_first_tasks_page true - rescue => e + rescue StandardError => e fail_import(e.message) false diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 32d3eeb8cd2..8875e6320c7 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -87,4 +87,4 @@ module Gitlab end end -Gitlab::ProjectTemplate.prepend_if_ee('EE::Gitlab::ProjectTemplate') +Gitlab::ProjectTemplate.prepend_mod_with('Gitlab::ProjectTemplate') diff --git a/lib/gitlab/prometheus/adapter.rb b/lib/gitlab/prometheus/adapter.rb index 76e65d29c7a..45438d9bf7c 100644 --- a/lib/gitlab/prometheus/adapter.rb +++ b/lib/gitlab/prometheus/adapter.rb @@ -19,13 +19,11 @@ module Gitlab end def cluster_prometheus_adapter - if cluster&.integration_prometheus - return cluster.integration_prometheus - end - application = cluster&.application_prometheus + return application if application&.available? - application if application&.available? + integration = cluster&.integration_prometheus + integration if integration&.available? end private diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb index ee3d98f3602..f5eb27b6916 100644 --- a/lib/gitlab/prometheus/additional_metrics_parser.rb +++ b/lib/gitlab/prometheus/additional_metrics_parser.rb @@ -14,7 +14,7 @@ module Gitlab private def validate!(obj) - raise ParsingError.new(obj.errors.full_messages.join('\n')) unless obj.valid? + raise ParsingError, obj.errors.full_messages.join('\n') unless obj.valid? end def group_from_entry(entry) diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb index 4a39260a340..020d4cf74a3 100644 --- a/lib/gitlab/prometheus/metric_group.rb +++ b/lib/gitlab/prometheus/metric_group.rb @@ -31,4 +31,4 @@ module Gitlab end end -Gitlab::Prometheus::MetricGroup.prepend_if_ee('EE::Gitlab::Prometheus::MetricGroup') +Gitlab::Prometheus::MetricGroup.prepend_mod_with('Gitlab::Prometheus::MetricGroup') diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index d24b98e790b..a870bb6bc5f 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -98,4 +98,4 @@ module Gitlab end end -Gitlab::Prometheus::Queries::QueryAdditionalMetrics.prepend_if_ee('EE::Gitlab::Prometheus::Queries::QueryAdditionalMetrics') +Gitlab::Prometheus::Queries::QueryAdditionalMetrics.prepend_mod_with('Gitlab::Prometheus::Queries::QueryAdditionalMetrics') diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 0fcf63d03fc..8182dbad4f8 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -47,7 +47,7 @@ module Gitlab # From Prometheus docs: This endpoint returns 200 when Prometheus is ready to serve traffic (i.e. respond to queries). response.code == 200 - rescue => e + rescue StandardError => e raise PrometheusClient::UnexpectedResponseError, "#{e.message}" end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 012e495502f..ff17ecf8024 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -267,7 +267,7 @@ module Gitlab private def zoom_link_service - Issues::ZoomLinkService.new(quick_action_target, current_user) + Issues::ZoomLinkService.new(project: quick_action_target.project, current_user: current_user, params: { issue: quick_action_target }) end end end diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 6a404c34044..f3c6315cd6a 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -148,7 +148,7 @@ module Gitlab quick_action_target.persisted? && quick_action_target.can_be_approved_by?(current_user) end command :approve do - success = MergeRequests::ApprovalService.new(quick_action_target.project, current_user).execute(quick_action_target) + success = MergeRequests::ApprovalService.new(project: quick_action_target.project, current_user: current_user).execute(quick_action_target) next unless success diff --git a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb index 4a62e83e8e9..03b2a1086bb 100644 --- a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb +++ b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb @@ -19,7 +19,7 @@ module Gitlab def execute return if @spend_arg.blank? - return [get_time, DateTime.now.to_date] unless date_present? + return [get_time, DateTime.current] unless date_present? return unless valid_date? [get_time, get_date] diff --git a/lib/gitlab/quick_actions/substitution_definition.rb b/lib/gitlab/quick_actions/substitution_definition.rb index 24b4e3c62b3..2cc4a6d90ce 100644 --- a/lib/gitlab/quick_actions/substitution_definition.rb +++ b/lib/gitlab/quick_actions/substitution_definition.rb @@ -13,7 +13,7 @@ module Gitlab return unless content all_names.each do |a_name| - content = content.sub(%r{/#{a_name}(?![\S]) ?(.*)$}i, execute_block(action_block, context, '\1')) + content = content.sub(%r{/#{a_name}(?!\S) ?(.*)$}i, execute_block(action_block, context, '\1')) end content diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb index ae3c89c3565..175f32bd4c6 100644 --- a/lib/gitlab/rack_attack.rb +++ b/lib/gitlab/rack_attack.rb @@ -83,16 +83,13 @@ module Gitlab def self.configure_throttles(rack_attack) throttle_or_track(rack_attack, 'throttle_unauthenticated', Gitlab::Throttle.unauthenticated_options) do |req| - if !req.should_be_skipped? && - Gitlab::Throttle.settings.throttle_unauthenticated_enabled && - req.unauthenticated? + if req.throttle_unauthenticated? req.ip end end throttle_or_track(rack_attack, 'throttle_authenticated_api', Gitlab::Throttle.authenticated_api_options) do |req| - if req.api_request? && - Gitlab::Throttle.settings.throttle_authenticated_api_enabled + if req.throttle_authenticated_api? req.throttled_user_id([:api]) end end @@ -107,40 +104,41 @@ module Gitlab end throttle_or_track(rack_attack, 'throttle_authenticated_web', Gitlab::Throttle.authenticated_web_options) do |req| - if req.web_request? && - Gitlab::Throttle.settings.throttle_authenticated_web_enabled + if req.throttle_authenticated_web? req.throttled_user_id([:api, :rss, :ics]) end end throttle_or_track(rack_attack, 'throttle_unauthenticated_protected_paths', Gitlab::Throttle.protected_paths_options) do |req| - if req.post? && - !req.should_be_skipped? && - req.protected_path? && - Gitlab::Throttle.protected_paths_enabled? && - req.unauthenticated? + if req.throttle_unauthenticated_protected_paths? req.ip end end throttle_or_track(rack_attack, 'throttle_authenticated_protected_paths_api', Gitlab::Throttle.protected_paths_options) do |req| - if req.post? && - req.api_request? && - req.protected_path? && - Gitlab::Throttle.protected_paths_enabled? + if req.throttle_authenticated_protected_paths_api? req.throttled_user_id([:api]) end end throttle_or_track(rack_attack, 'throttle_authenticated_protected_paths_web', Gitlab::Throttle.protected_paths_options) do |req| - if req.post? && - req.web_request? && - req.protected_path? && - Gitlab::Throttle.protected_paths_enabled? + if req.throttle_authenticated_protected_paths_web? req.throttled_user_id([:api, :rss, :ics]) end end + throttle_or_track(rack_attack, 'throttle_unauthenticated_packages_api', Gitlab::Throttle.unauthenticated_packages_api_options) do |req| + if req.throttle_unauthenticated_packages_api? + req.ip + end + end + + throttle_or_track(rack_attack, 'throttle_authenticated_packages_api', Gitlab::Throttle.authenticated_packages_api_options) do |req| + if req.throttle_authenticated_packages_api? + req.throttled_user_id([:api]) + end + end + rack_attack.safelist('throttle_bypass_header') do |req| Gitlab::Throttle.bypass_header.present? && req.get_header(Gitlab::Throttle.bypass_header) == '1' @@ -173,4 +171,4 @@ module Gitlab end end end -::Gitlab::RackAttack.prepend_if_ee('::EE::Gitlab::RackAttack') +::Gitlab::RackAttack.prepend_mod_with('Gitlab::RackAttack') diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb index bd6d2e016b4..7fee6a1b43d 100644 --- a/lib/gitlab/rack_attack/request.rb +++ b/lib/gitlab/rack_attack/request.rb @@ -58,6 +58,57 @@ module Gitlab path =~ protected_paths_regex end + def throttle_unauthenticated? + !should_be_skipped? && + !throttle_unauthenticated_packages_api? && + Gitlab::Throttle.settings.throttle_unauthenticated_enabled && + unauthenticated? + end + + def throttle_authenticated_api? + api_request? && + !throttle_authenticated_packages_api? && + Gitlab::Throttle.settings.throttle_authenticated_api_enabled + end + + def throttle_authenticated_web? + web_request? && + Gitlab::Throttle.settings.throttle_authenticated_web_enabled + end + + def throttle_unauthenticated_protected_paths? + post? && + !should_be_skipped? && + protected_path? && + Gitlab::Throttle.protected_paths_enabled? && + unauthenticated? + end + + def throttle_authenticated_protected_paths_api? + post? && + api_request? && + protected_path? && + Gitlab::Throttle.protected_paths_enabled? + end + + def throttle_authenticated_protected_paths_web? + post? && + web_request? && + protected_path? && + Gitlab::Throttle.protected_paths_enabled? + end + + def throttle_unauthenticated_packages_api? + packages_api_path? && + Gitlab::Throttle.settings.throttle_unauthenticated_packages_api_enabled && + unauthenticated? + end + + def throttle_authenticated_packages_api? + packages_api_path? && + Gitlab::Throttle.settings.throttle_authenticated_packages_api_enabled + end + private def authenticated_user_id(request_formats) @@ -75,7 +126,11 @@ module Gitlab def protected_paths_regex Regexp.union(protected_paths.map { |path| /\A#{Regexp.escape(path)}/ }) end + + def packages_api_path? + path =~ ::Gitlab::Regex::Packages::API_PATH_REGEX + end end end end -::Gitlab::RackAttack::Request.prepend_if_ee('::EE::Gitlab::RackAttack::Request') +::Gitlab::RackAttack::Request.prepend_mod_with('Gitlab::RackAttack::Request') diff --git a/lib/gitlab/redis/boolean.rb b/lib/gitlab/redis/boolean.rb index 9b0b20fc2be..cd0877c5b13 100644 --- a/lib/gitlab/redis/boolean.rb +++ b/lib/gitlab/redis/boolean.rb @@ -50,7 +50,7 @@ module Gitlab # @return [String] the encoded boolean # @raise [NotABooleanError] if the value isn't true or false def encode(value) - raise NotABooleanError.new(value) unless bool?(value) + raise NotABooleanError, value unless bool?(value) [LABEL, to_string(value)].join(DELIMITER) end @@ -61,11 +61,11 @@ module Gitlab # @return [Boolean] true or false # @raise [NotAnEncodedBooleanStringError] if the provided value isn't an encoded boolean def decode(value) - raise NotAnEncodedBooleanStringError.new(value.class) unless value.is_a?(String) + raise NotAnEncodedBooleanStringError, value.class unless value.is_a?(String) label, bool_str = *value.split(DELIMITER, 2) - raise NotAnEncodedBooleanStringError.new(label) unless label == LABEL + raise NotAnEncodedBooleanStringError, label unless label == LABEL from_string(bool_str) end @@ -99,7 +99,7 @@ module Gitlab end def from_string(str) - raise NotAnEncodedBooleanStringError.new(str) unless [TRUE_STR, FALSE_STR].include?(str) + raise NotAnEncodedBooleanStringError, str unless [TRUE_STR, FALSE_STR].include?(str) str == TRUE_STR end diff --git a/lib/gitlab/redis/hll.rb b/lib/gitlab/redis/hll.rb index 010a6b59da5..0d04545688b 100644 --- a/lib/gitlab/redis/hll.rb +++ b/lib/gitlab/redis/hll.rb @@ -46,7 +46,7 @@ module Gitlab def validate_key!(key) return if KEY_REGEX.match?(key) - raise KeyFormatError.new("Invalid key format. #{key} key should have changeable parts in curly braces. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands") + raise KeyFormatError, "Invalid key format. #{key} key should have changeable parts in curly braces. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands" end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 6f80c7d439f..94ab67ef08a 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -142,7 +142,7 @@ module Gitlab def fetch_config return false unless self.class._raw_config - yaml = YAML.load(self.class._raw_config) + yaml = YAML.safe_load(self.class._raw_config, aliases: true) # If the file has content but it's invalid YAML, `load` returns false if yaml diff --git a/lib/gitlab/reference_counter.rb b/lib/gitlab/reference_counter.rb index c2fa2e1330a..f41e42b9e9c 100644 --- a/lib/gitlab/reference_counter.rb +++ b/lib/gitlab/reference_counter.rb @@ -84,7 +84,7 @@ module Gitlab Gitlab::Redis::SharedState.with { |redis| yield(redis) } true - rescue => e + rescue StandardError => e Gitlab::AppLogger.warn("GitLab: An unexpected error occurred in writing to Redis: #{e}") false diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 488ba04f87c..ccb4f6e1097 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -6,6 +6,8 @@ module Gitlab CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt conan_sources.tgz conan_export.tgz].freeze CONAN_PACKAGE_FILES = %w[conaninfo.txt conanmanifest.txt conan_package.tgz].freeze + API_PATH_REGEX = %r{^/api/v\d+/(projects/[^/]+/|groups?/[^/]+/-/)?packages/[A-Za-z]+}.freeze + def conan_package_reference_regex @conan_package_reference_regex ||= %r{\A[A-Za-z0-9]+\z}.freeze end @@ -75,6 +77,10 @@ module Gitlab /x.freeze end + def terraform_module_package_name_regex + @terraform_module_package_name_regex ||= %r{\A[-a-z0-9]+\/[-a-z0-9]+\z}.freeze + end + def pypi_version_regex # See the official regex: https://github.com/pypa/packaging/blob/16.7/packaging/version.py#L159 @@ -123,6 +129,18 @@ module Gitlab @debian_component_regex ||= %r{#{debian_distribution_regex}}.freeze end + def helm_channel_regex + @helm_channel_regex ||= %r{\A[-\.\_a-zA-Z0-9]+\z}.freeze + end + + def helm_package_regex + @helm_package_regex ||= %r{#{helm_channel_regex}}.freeze + end + + def helm_version_regex + @helm_version_regex ||= %r{#{prefixed_semver_regex}}.freeze + end + def unbounded_semver_regex # See the official regex: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string @@ -135,7 +153,7 @@ module Gitlab end def semver_regex - @semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) + @semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options).freeze end # These partial semver regexes are intended for use in composing other @@ -235,7 +253,7 @@ module Gitlab # used as a routing constraint. # def container_registry_tag_regex - @container_registry_tag_regex ||= /[\w][\w.-]{0,127}/ + @container_registry_tag_regex ||= /\w[\w.-]{0,127}/ end def environment_name_regex_chars diff --git a/lib/gitlab/relative_positioning.rb b/lib/gitlab/relative_positioning.rb index e2cbe4b2de0..c2a73b7cfe5 100644 --- a/lib/gitlab/relative_positioning.rb +++ b/lib/gitlab/relative_positioning.rb @@ -13,7 +13,9 @@ module Gitlab MIN_GAP = 2 NoSpaceLeft = Class.new(StandardError) + InvalidPosition = Class.new(StandardError) IllegalRange = Class.new(ArgumentError) + IssuePositioningDisabled = Class.new(StandardError) def self.range(lhs, rhs) if lhs && rhs diff --git a/lib/gitlab/relative_positioning/item_context.rb b/lib/gitlab/relative_positioning/item_context.rb index 8f5495ece5e..1e738aef9b0 100644 --- a/lib/gitlab/relative_positioning/item_context.rb +++ b/lib/gitlab/relative_positioning/item_context.rb @@ -129,6 +129,14 @@ module Gitlab neighbour(sib) end + def at_position(position) + item = scoped_items.find_by(relative_position: position) + + raise InvalidPosition, 'No item found at the specified position' if item.nil? + + neighbour(item) + end + def shift_left move_sequence_before(true) object.reset_relative_position diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 46c84107e0f..42b94d5cf3b 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -110,4 +110,4 @@ module Gitlab end end -Gitlab::RepoPath.singleton_class.prepend_if_ee('EE::Gitlab::RepoPath::ClassMethods') +Gitlab::RepoPath.singleton_class.prepend_mod_with('Gitlab::RepoPath::ClassMethods') diff --git a/lib/gitlab/repository_size_checker.rb b/lib/gitlab/repository_size_checker.rb index 0ed31176dd8..2afc5e8d668 100644 --- a/lib/gitlab/repository_size_checker.rb +++ b/lib/gitlab/repository_size_checker.rb @@ -56,4 +56,4 @@ module Gitlab end end -Gitlab::RepositorySizeChecker.prepend_if_ee('EE::Gitlab::RepositorySizeChecker') +Gitlab::RepositorySizeChecker.prepend_mod_with('Gitlab::RepositorySizeChecker') diff --git a/lib/gitlab/repository_url_builder.rb b/lib/gitlab/repository_url_builder.rb index a2d0d50d20b..ed9a298ee8c 100644 --- a/lib/gitlab/repository_url_builder.rb +++ b/lib/gitlab/repository_url_builder.rb @@ -10,7 +10,7 @@ module Gitlab when :http http_url(path) else - raise NotImplementedError.new("No URL builder defined for protocol #{protocol}") + raise NotImplementedError, "No URL builder defined for protocol #{protocol}" end end diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb index 7050aee3847..acdf8d4541f 100644 --- a/lib/gitlab/request_profiler/middleware.rb +++ b/lib/gitlab/request_profiler/middleware.rb @@ -90,7 +90,7 @@ module Gitlab File.open(file_path, 'wb') do |file| yield(file) end - rescue + rescue StandardError FileUtils.rm(file_path) end end diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb index a555bf1d812..65c7f4b39e5 100644 --- a/lib/gitlab/route_map.rb +++ b/lib/gitlab/route_map.rb @@ -7,7 +7,7 @@ module Gitlab def initialize(data) begin entries = YAML.safe_load(data) - rescue + rescue StandardError raise FormatError, 'Route map is not valid YAML' end diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index cad127922df..fd9fb8ab7e2 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -30,7 +30,7 @@ module Gitlab rescue URI::InvalidURIError => e # If url is invalid, raise custom error, # which can be ignored by monitoring tools. - raise ActionController::RoutingError.new(e.message) + raise ActionController::RoutingError, e.message end end diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index 968ef06b085..b0bcea0ca69 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -26,13 +26,9 @@ module Gitlab if matches.one? matches.first elsif matches.none? - raise UnknownProcessError.new( - "Failed to identify runtime for process #{Process.pid} (#{$0})" - ) + raise UnknownProcessError, "Failed to identify runtime for process #{Process.pid} (#{$0})" else - raise AmbiguousProcessError.new( - "Ambiguous runtime #{matches} for process #{Process.pid} (#{$0})" - ) + raise AmbiguousProcessError, "Ambiguous runtime #{matches} for process #{Process.pid} (#{$0})" end end @@ -91,7 +87,7 @@ module Gitlab def max_threads threads = 1 # main thread - if puma? + if puma? && Puma.respond_to?(:cli_config) threads += Puma.cli_config.options[:max_threads] elsif sidekiq? # An extra thread for the poller in Sidekiq Cron: diff --git a/lib/gitlab/sanitizers/exif.rb b/lib/gitlab/sanitizers/exif.rb index eec50deb61e..f607aff9d29 100644 --- a/lib/gitlab/sanitizers/exif.rb +++ b/lib/gitlab/sanitizers/exif.rb @@ -71,7 +71,7 @@ module Gitlab relation.find_each(**find_params) do |upload| clean(upload.retrieve_uploader, dry_run: dry_run) sleep sleep_time if sleep_time - rescue => err + rescue StandardError => err logger.error "failed to sanitize #{upload_ref(upload)}: #{err.message}" logger.debug err.backtrace.join("\n ") end diff --git a/lib/gitlab/search/parsed_query.rb b/lib/gitlab/search/parsed_query.rb index 5d5d407c172..a397ce935cb 100644 --- a/lib/gitlab/search/parsed_query.rb +++ b/lib/gitlab/search/parsed_query.rb @@ -50,11 +50,11 @@ module Gitlab when :including then including when :excluding then excluding else - raise ArgumentError.new(type) + raise ArgumentError, type end end end end end -Gitlab::Search::ParsedQuery.prepend_if_ee('EE::Gitlab::Search::ParsedQuery') +Gitlab::Search::ParsedQuery.prepend_mod_with('Gitlab::Search::ParsedQuery') diff --git a/lib/gitlab/search_context.rb b/lib/gitlab/search_context.rb index 0323220690a..04ef2be87f8 100644 --- a/lib/gitlab/search_context.rb +++ b/lib/gitlab/search_context.rb @@ -164,4 +164,4 @@ module Gitlab end end -Gitlab::SearchContext::Builder.prepend_ee_mod +Gitlab::SearchContext::Builder.prepend_mod diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index d0beb74c289..678c0b396ef 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -236,4 +236,4 @@ module Gitlab end end -Gitlab::SearchResults.prepend_if_ee('EE::Gitlab::SearchResults') +Gitlab::SearchResults.prepend_mod_with('Gitlab::SearchResults') diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 3419989c110..d26e1a34a9f 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -92,7 +92,7 @@ module Gitlab Gitlab::Git::Repository.new(storage, "#{disk_path}.git", nil, nil).rename("#{new_disk_path}.git") true - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, path: disk_path, new_path: new_disk_path, storage: storage) false @@ -115,7 +115,7 @@ module Gitlab Gitlab::Git::Repository.new(storage, "#{disk_path}.git", nil, nil).remove true - rescue => e + rescue StandardError => e Gitlab::AppLogger.warn("Repository does not exist: #{e} at: #{disk_path}.git") Gitlab::ErrorTracking.track_exception(e, path: disk_path, storage: storage) diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index 78d45b5f3f0..16a0619daf6 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -21,8 +21,18 @@ module Gitlab # invalid class name. We keep it in the YAML file for safety, just # in case anything does get scheduled to run there. DEFAULT_WORKERS = { - '_' => DummyWorker.new('default', weight: 1, tags: []), - 'ActionMailer::MailDeliveryJob' => DummyWorker.new('mailers', feature_category: :issue_tracking, urgency: 'low', weight: 2, tags: []) + '_' => DummyWorker.new( + queue: 'default', + weight: 1, tags: [] + ), + 'ActionMailer::MailDeliveryJob' => DummyWorker.new( + name: 'ActionMailer::MailDeliveryJob', + queue: 'mailers', + feature_category: :issue_tracking, + urgency: 'low', + weight: 2, + tags: [] + ) }.transform_values { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false) }.freeze class << self diff --git a/lib/gitlab/sidekiq_config/dummy_worker.rb b/lib/gitlab/sidekiq_config/dummy_worker.rb index 7568840410b..ef0dce0cf84 100644 --- a/lib/gitlab/sidekiq_config/dummy_worker.rb +++ b/lib/gitlab/sidekiq_config/dummy_worker.rb @@ -4,9 +4,9 @@ module Gitlab module SidekiqConfig # For queues that don't have explicit workers - default and mailers class DummyWorker - attr_accessor :queue - ATTRIBUTE_METHODS = { + queue: :queue, + name: :name, feature_category: :get_feature_category, has_external_dependencies: :worker_has_external_dependencies?, urgency: :get_urgency, @@ -16,8 +16,7 @@ module Gitlab tags: :get_tags }.freeze - def initialize(queue, attributes = {}) - @queue = queue + def initialize(attributes = {}) @attributes = attributes end diff --git a/lib/gitlab/sidekiq_config/worker.rb b/lib/gitlab/sidekiq_config/worker.rb index 46fa0aa5be1..aea4209f631 100644 --- a/lib/gitlab/sidekiq_config/worker.rb +++ b/lib/gitlab/sidekiq_config/worker.rb @@ -6,10 +6,9 @@ module Gitlab include Comparable attr_reader :klass - delegate :feature_category_not_owned?, :get_feature_category, :get_tags, - :get_urgency, :get_weight, :get_worker_resource_boundary, - :idempotent?, :queue, :queue_namespace, - :worker_has_external_dependencies?, + delegate :feature_category_not_owned?, :get_feature_category, :get_sidekiq_options, + :get_tags, :get_urgency, :get_weight, :get_worker_resource_boundary, + :idempotent?, :queue, :queue_namespace, :worker_has_external_dependencies?, to: :klass def initialize(klass, ee:) @@ -47,6 +46,7 @@ module Gitlab def to_yaml { name: queue, + worker_name: klass.name, feature_category: get_feature_category, has_external_dependencies: worker_has_external_dependencies?, urgency: get_urgency, @@ -64,6 +64,10 @@ module Gitlab def queue_and_weight [queue, get_weight] end + + def retries + get_sidekiq_options['retry'] + end end end end diff --git a/lib/gitlab/sidekiq_config/worker_matcher.rb b/lib/gitlab/sidekiq_config/worker_matcher.rb index fe5ac10c65a..d615d5ecba4 100644 --- a/lib/gitlab/sidekiq_config/worker_matcher.rb +++ b/lib/gitlab/sidekiq_config/worker_matcher.rb @@ -10,6 +10,7 @@ module Gitlab QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w:#{QUERY_CONCATENATE_OPERATOR}]+)}.freeze QUERY_PREDICATES = { + worker_name: :to_s, feature_category: :to_sym, has_external_dependencies: lambda { |value| value == 'true' }, name: :to_s, @@ -50,7 +51,7 @@ module Gitlab def predicate_for_term(term) match = term.match(QUERY_TERM_REGEX) - raise InvalidTerm.new("Invalid term: #{term}") unless match + raise InvalidTerm, "Invalid term: #{term}" unless match _, lhs, op, rhs = *match @@ -66,14 +67,14 @@ module Gitlab else # This is unreachable because InvalidTerm will be raised instead, but # keeping it allows to guard against that changing in future. - raise UnknownOperator.new("Unknown operator: #{op}") + raise UnknownOperator, "Unknown operator: #{op}" end end def predicate_factory(lhs, values) values_block = QUERY_PREDICATES[lhs.to_sym] - raise UnknownPredicate.new("Unknown predicate: #{lhs}") unless values_block + raise UnknownPredicate, "Unknown predicate: #{lhs}" unless values_block lambda do |queue| comparator = Array(queue[lhs.to_sym]).to_set diff --git a/lib/gitlab/sidekiq_config/worker_router.rb b/lib/gitlab/sidekiq_config/worker_router.rb new file mode 100644 index 00000000000..946296a24d3 --- /dev/null +++ b/lib/gitlab/sidekiq_config/worker_router.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqConfig + class WorkerRouter + InvalidRoutingRuleError = Class.new(StandardError) + RuleEvaluator = Struct.new(:matcher, :queue_name) + + def self.queue_name_from_worker_name(worker_klass) + base_queue_name = + worker_klass.name + .delete_prefix('Gitlab::') + .delete_suffix('Worker') + .underscore + .tr('/', '_') + [worker_klass.queue_namespace, base_queue_name].compact.join(':') + end + + def self.global + @global_worker_router ||= new(::Gitlab.config.sidekiq.routing_rules) + rescue InvalidRoutingRuleError, ::Gitlab::SidekiqConfig::WorkerMatcher::UnknownPredicate => e + ::Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + + @global_worker_router = new([]) + end + + # call-seq: + # router = WorkerRouter.new([ + # ["resource_boundary=cpu", 'cpu_boundary'], + # ["feature_category=pages", nil], + # ["feature_category=source_code_management", ''], + # ["*", "default"] + # ]) + # router.route(ACpuBoundaryWorker) # Return "cpu_boundary" + # router.route(JustAPagesWorker) # Return "just_a_pages_worker" + # router.route(PostReceive) # Return "post_receive" + # router.route(RandomWorker) # Return "default" + # + # This class is responsible for routing a Sidekiq worker to a certain + # queue defined in the input routing rules. The input routing rules, as + # described above, is an order-matter array of tuples [query, queue_name]. + # + # - The query syntax is the same as the "queue selector" detailedly + # denoted in doc/administration/operations/extra_sidekiq_processes.md. + # + # - The queue_name must be a valid Sidekiq queue name. If the queue name + # is nil, or an empty string, the worker is routed to the queue generated + # by the name of the worker instead. + # + # Rules are evaluated from first to last, and as soon as we find a match + # for a given worker we stop processing for that worker (first match + # wins). If the worker doesn't match any rule, it falls back the queue + # name generated from the worker name + # + # For further information, please visit: + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1016 + # + def initialize(routing_rules) + @rule_evaluators = parse_routing_rules(routing_rules) + end + + def route(worker_klass) + # A medium representation to ensure the backward-compatibility of + # WorkerMatcher + worker_metadata = generate_worker_metadata(worker_klass) + @rule_evaluators.each do |evaluator| + if evaluator.matcher.match?(worker_metadata) + return evaluator.queue_name.presence || queue_name_from_worker_name(worker_klass) + end + end + + queue_name_from_worker_name(worker_klass) + end + + private + + def parse_routing_rules(routing_rules) + raise InvalidRoutingRuleError, 'The set of routing rule must be an array' unless routing_rules.is_a?(Array) + + routing_rules.map do |rule_tuple| + raise InvalidRoutingRuleError, "Routing rule `#{rule_tuple.inspect}` is invalid" unless valid_routing_rule?(rule_tuple) + + selector, destination_queue = rule_tuple + RuleEvaluator.new( + ::Gitlab::SidekiqConfig::WorkerMatcher.new(selector), + destination_queue + ) + end + end + + def valid_routing_rule?(rule_tuple) + rule_tuple.is_a?(Array) && rule_tuple.length == 2 + end + + def generate_worker_metadata(worker_klass) + # The ee indicator here is insignificant and irrelevant to the matcher. + # Plus, it's not easy to determine whether a worker is **only** + # available in EE. + ::Gitlab::SidekiqConfig::Worker.new(worker_klass, ee: false).to_yaml + end + + def queue_name_from_worker_name(worker_klass) + self.class.queue_name_from_worker_name(worker_klass) + end + end + end +end diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index 8793a672693..113076a6a75 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -73,7 +73,7 @@ module Gitlab begin sleep(CHECK_INTERVAL_SECONDS) restart_sidekiq unless rss_within_range? - rescue => e + rescue StandardError => e log_exception(e, __method__) rescue Exception => e # rubocop:disable Lint/RescueException log_exception(e, __method__ ) @@ -249,7 +249,7 @@ module Gitlab def get_job_options(job, key, default) job[:worker_class].sidekiq_options.fetch(key, default) - rescue + rescue StandardError default end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index b1fb3771c78..87fb36d04e9 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -30,7 +30,7 @@ module Gitlab Sidekiq.logger.warn log_job_done(job, started_time, base_payload, job_exception.cause || job_exception) raise - rescue => job_exception + rescue StandardError => job_exception Sidekiq.logger.warn log_job_done(job, started_time, base_payload, job_exception) raise @@ -39,7 +39,7 @@ module Gitlab private def add_instrumentation_keys!(job, output_payload) - output_payload.merge!(job[:instrumentation].stringify_keys) + output_payload.merge!(job[:instrumentation].stringify_keys) if job[:instrumentation] end def add_logging_extras!(job, output_payload) @@ -70,6 +70,8 @@ module Gitlab message = base_message(payload) + payload['database_chosen'] = job[:database_chosen] if job[:database_chosen] + if job_exception payload['message'] = "#{message}: fail: #{payload['duration_s']} sec" payload['job_status'] = 'fail' diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index 563a105484d..c5b980769f0 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -44,4 +44,4 @@ module Gitlab end end -Gitlab::SidekiqMiddleware.singleton_class.prepend_if_ee('EE::Gitlab::SidekiqMiddleware') +Gitlab::SidekiqMiddleware.singleton_class.prepend_mod_with('Gitlab::SidekiqMiddleware') diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index f5fee8050ac..474afffcf93 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -119,4 +119,4 @@ module Gitlab end end -Gitlab::SidekiqMiddleware::ServerMetrics.prepend_if_ee('EE::Gitlab::SidekiqMiddleware::ServerMetrics') +Gitlab::SidekiqMiddleware::ServerMetrics.prepend_mod_with('Gitlab::SidekiqMiddleware::ServerMetrics') diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb b/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb index da6c903ccae..540159e8a72 100644 --- a/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb +++ b/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb @@ -13,7 +13,7 @@ module Gitlab @size = size @size_limit = size_limit - super "#{@worker_class} job exceeds payload size limit (#{size}/#{size_limit})" + super "#{@worker_class} job exceeds payload size limit" end def sentry_extra_data diff --git a/lib/gitlab/sidekiq_migrate_jobs.rb b/lib/gitlab/sidekiq_migrate_jobs.rb new file mode 100644 index 00000000000..62d62bf82c4 --- /dev/null +++ b/lib/gitlab/sidekiq_migrate_jobs.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + class SidekiqMigrateJobs + LOG_FREQUENCY = 1_000 + + attr_reader :sidekiq_set, :logger + + def initialize(sidekiq_set, logger: nil) + @sidekiq_set = sidekiq_set + @logger = logger + end + + # mappings is a hash of WorkerClassName => target_queue_name + def execute(mappings) + source_queues_regex = Regexp.union(mappings.keys) + cursor = 0 + scanned = 0 + migrated = 0 + + estimated_size = Sidekiq.redis { |c| c.zcard(sidekiq_set) } + logger&.info("Processing #{sidekiq_set} set. Estimated size: #{estimated_size}.") + + begin + cursor, jobs = Sidekiq.redis { |c| c.zscan(sidekiq_set, cursor) } + + jobs.each do |(job, score)| + if scanned > 0 && scanned % LOG_FREQUENCY == 0 + logger&.info("In progress. Scanned records: #{scanned}. Migrated records: #{migrated}.") + end + + scanned += 1 + + next unless job.match?(source_queues_regex) + + job_hash = Sidekiq.load_json(job) + destination_queue = mappings[job_hash['class']] + + next unless mappings.has_key?(job_hash['class']) + next if job_hash['queue'] == destination_queue + + job_hash['queue'] = destination_queue + + migrated += migrate_job(job, score, job_hash) + end + end while cursor.to_i != 0 + + logger&.info("Done. Scanned records: #{scanned}. Migrated records: #{migrated}.") + + { + scanned: scanned, + migrated: migrated + } + end + + private + + def migrate_job(job, score, job_hash) + Sidekiq.redis do |connection| + removed = connection.zrem(sidekiq_set, job) + + if removed + connection.zadd(sidekiq_set, score, Sidekiq.dump_json(job_hash)) + + 1 + else + 0 + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 2293e2adee1..623fdd89456 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -66,7 +66,7 @@ module Gitlab def self.num_running(job_ids) responses = self.job_status(job_ids) - responses.select(&:present?).count + responses.count(&:present?) end # Returns the number of jobs that have completed. diff --git a/lib/gitlab/slash_commands/issue_close.rb b/lib/gitlab/slash_commands/issue_close.rb index 5fcc86e91c4..3dad7216983 100644 --- a/lib/gitlab/slash_commands/issue_close.rb +++ b/lib/gitlab/slash_commands/issue_close.rb @@ -29,7 +29,7 @@ module Gitlab private def close_issue(issue:) - Issues::CloseService.new(project, current_user).execute(issue) + Issues::CloseService.new(project: project, current_user: current_user).execute(issue) end def presenter(issue) diff --git a/lib/gitlab/slash_commands/issue_move.rb b/lib/gitlab/slash_commands/issue_move.rb index d2f1f130b38..0612663017c 100644 --- a/lib/gitlab/slash_commands/issue_move.rb +++ b/lib/gitlab/slash_commands/issue_move.rb @@ -29,7 +29,7 @@ module Gitlab return Gitlab::SlashCommands::Presenters::Access.new.not_found end - new_issue = Issues::MoveService.new(project, current_user) + new_issue = Issues::MoveService.new(project: project, current_user: current_user) .execute(old_issue, target_project) presenter(new_issue).present(old_issue) diff --git a/lib/gitlab/slash_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb index 48379031537..99a056c97fc 100644 --- a/lib/gitlab/slash_commands/issue_new.rb +++ b/lib/gitlab/slash_commands/issue_new.rb @@ -33,7 +33,7 @@ module Gitlab private def create_issue(title:, description:) - Issues::CreateService.new(project, current_user, title: title, description: description).execute + Issues::CreateService.new(project: project, current_user: current_user, params: { title: title, description: description }).execute end def presenter(issue) diff --git a/lib/gitlab/slash_commands/presenters/issue_base.rb b/lib/gitlab/slash_commands/presenters/issue_base.rb index 017fb8a62c4..f6f6e3d7fc6 100644 --- a/lib/gitlab/slash_commands/presenters/issue_base.rb +++ b/lib/gitlab/slash_commands/presenters/issue_base.rb @@ -50,4 +50,4 @@ module Gitlab end end -Gitlab::SlashCommands::Presenters::IssueBase.prepend_if_ee('EE::Gitlab::SlashCommands::Presenters::IssueBase') +Gitlab::SlashCommands::Presenters::IssueBase.prepend_mod_with('Gitlab::SlashCommands::Presenters::IssueBase') diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index 41ec19f0da8..581d6b738f3 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -45,4 +45,4 @@ module Gitlab end end -Gitlab::SnippetSearchResults.prepend_if_ee('::EE::Gitlab::SnippetSearchResults') +Gitlab::SnippetSearchResults.prepend_mod_with('Gitlab::SnippetSearchResults') diff --git a/lib/gitlab/spamcheck/client.rb b/lib/gitlab/spamcheck/client.rb new file mode 100644 index 00000000000..6afc21be4e0 --- /dev/null +++ b/lib/gitlab/spamcheck/client.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true +require 'spamcheck' + +module Gitlab + module Spamcheck + class Client + include ::Spam::SpamConstants + DEFAULT_TIMEOUT_SECS = 2 + + VERDICT_MAPPING = { + ::Spamcheck::SpamVerdict::Verdict::ALLOW => ALLOW, + ::Spamcheck::SpamVerdict::Verdict::CONDITIONAL_ALLOW => CONDITIONAL_ALLOW, + ::Spamcheck::SpamVerdict::Verdict::DISALLOW => DISALLOW, + ::Spamcheck::SpamVerdict::Verdict::BLOCK => BLOCK_USER, + ::Spamcheck::SpamVerdict::Verdict::NOOP => NOOP + }.freeze + + ACTION_MAPPING = { + create: ::Spamcheck::Action::CREATE, + update: ::Spamcheck::Action::UPDATE + }.freeze + + def initialize + @endpoint_url = Gitlab::CurrentSettings.current_application_settings.spam_check_endpoint_url + + # remove the `grpc://` as it's only useful to ensure we're expecting to + # connect with Spamcheck + @endpoint_url = @endpoint_url.gsub(%r(^grpc:\/\/), '') + + creds = + if Rails.env.development? || Rails.env.test? + :this_channel_is_insecure + else + GRPC::Core::ChannelCredentials.new + end + + @stub = ::Spamcheck::SpamcheckService::Stub.new(@endpoint_url, creds, + timeout: DEFAULT_TIMEOUT_SECS) + end + + def issue_spam?(spam_issue:, user:, context: {}) + issue = build_issue_protobuf(issue: spam_issue, user: user, context: context) + + response = @stub.check_for_spam_issue(issue, + metadata: { 'authorization' => + Gitlab::CurrentSettings.spam_check_api_key }) + verdict = convert_verdict_to_gitlab_constant(response.verdict) + [verdict, response.extra_attributes.to_h, response.error] + end + + private + + def convert_verdict_to_gitlab_constant(verdict) + VERDICT_MAPPING.fetch(::Spamcheck::SpamVerdict::Verdict.resolve(verdict), verdict) + end + + def build_issue_protobuf(issue:, user:, context:) + issue_pb = ::Spamcheck::Issue.new + issue_pb.title = issue.spam_title || '' + issue_pb.description = issue.spam_description || '' + issue_pb.created_at = convert_to_pb_timestamp(issue.created_at) if issue.created_at + issue_pb.updated_at = convert_to_pb_timestamp(issue.updated_at) if issue.updated_at + issue_pb.user_in_project = user.authorized_project?(issue.project) + issue_pb.project = build_project_protobuf(issue) + issue_pb.action = ACTION_MAPPING.fetch(context.fetch(:action)) if context.has_key?(:action) + issue_pb.user = build_user_protobuf(user) + issue_pb + end + + def build_user_protobuf(user) + user_pb = ::Spamcheck::User.new + user_pb.username = user.username + user_pb.org = user.organization || '' + user_pb.created_at = convert_to_pb_timestamp(user.created_at) + + user_pb.emails << build_email(user.email, user.confirmed?) + + user.emails.each do |email| + user_pb.emails << build_email(email.email, email.confirmed?) + end + + user_pb + end + + def build_email(email, verified) + email_pb = ::Spamcheck::User::Email.new + email_pb.email = email + email_pb.verified = verified + email_pb + end + + def build_project_protobuf(issue) + project_pb = ::Spamcheck::Project.new + project_pb.project_id = issue.project_id + project_pb.project_path = issue.project.full_path + project_pb + end + + def convert_to_pb_timestamp(ar_timestamp) + Google::Protobuf::Timestamp.new(seconds: ar_timestamp.to_time.to_i, + nanos: ar_timestamp.to_time.nsec) + end + end + end +end diff --git a/lib/gitlab/stack_prof.rb b/lib/gitlab/stack_prof.rb new file mode 100644 index 00000000000..4b7d93c91ce --- /dev/null +++ b/lib/gitlab/stack_prof.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +# trigger stackprof by sending a SIGUSR2 signal +# +# Docs: https://docs.gitlab.com/ee/development/performance.html#production + +module Gitlab + class StackProf + DEFAULT_FILE_PREFIX = Dir.tmpdir + DEFAULT_TIMEOUT_SEC = 30 + DEFAULT_MODE = :cpu + # Sample interval as a frequency in microseconds (~99hz); appropriate for CPU profiles + DEFAULT_INTERVAL_US = 10_100 + # Sample interval in event occurrences (n = every nth event); appropriate for allocation profiles + DEFAULT_INTERVAL_EVENTS = 100 + + # this is a workaround for sidekiq, which defines its own SIGUSR2 handler. + # by defering to the sidekiq startup event, we get to set up our own + # handler late enough. + # see also: https://github.com/mperham/sidekiq/pull/4653 + def self.install + require 'stackprof' + require 'tmpdir' + + if Gitlab::Runtime.sidekiq? + Sidekiq.configure_server do |config| + config.on :startup do + on_worker_start + end + end + else + Gitlab::Cluster::LifecycleEvents.on_worker_start do + on_worker_start + end + end + end + + def self.on_worker_start + log_event('listening for SIGUSR2 signal') + + # create a pipe in order to propagate signal out of the signal handler + # see also: https://cr.yp.to/docs/selfpipe.html + read, write = IO.pipe + + # create a separate thread that polls for signals on the pipe. + # + # this way we do not execute in signal handler context, which + # lifts restrictions and also serializes the calls in a thread-safe + # manner. + # + # it's very similar to a goroutine and channel design. + # + # another nice benefit of this method is that we can timeout the + # IO.select call, allowing the profile to automatically stop after + # a given interval (by default 30 seconds), avoiding unbounded memory + # growth from a profile that was started and never stopped. + t = Thread.new do + timeout_s = ENV['STACKPROF_TIMEOUT_S']&.to_i || DEFAULT_TIMEOUT_SEC + current_timeout_s = nil + loop do + read.getbyte if IO.select([read], nil, nil, current_timeout_s) + + if ::StackProf.running? + stackprof_file_prefix = ENV['STACKPROF_FILE_PREFIX'] || DEFAULT_FILE_PREFIX + stackprof_out_file = "#{stackprof_file_prefix}/stackprof.#{Process.pid}.#{SecureRandom.hex(6)}.profile" + + log_event( + 'stopping profile', + profile_filename: stackprof_out_file, + profile_timeout_s: timeout_s + ) + + ::StackProf.stop + ::StackProf.results(stackprof_out_file) + current_timeout_s = nil + else + mode = ENV['STACKPROF_MODE']&.to_sym || DEFAULT_MODE + interval = ENV['STACKPROF_INTERVAL']&.to_i + interval ||= (mode == :object ? DEFAULT_INTERVAL_EVENTS : DEFAULT_INTERVAL_US) + + log_event( + 'starting profile', + profile_mode: mode, + profile_interval: interval, + profile_timeout: timeout_s + ) + + ::StackProf.start( + mode: mode, + raw: Gitlab::Utils.to_boolean(ENV['STACKPROF_RAW'] || 'true'), + interval: interval + ) + current_timeout_s = timeout_s + end + end + rescue StandardError => e + log_event("stackprof failed: #{e}") + end + t.abort_on_exception = true + + # in the case of puma, this will override the existing SIGUSR2 signal handler + # that can be used to trigger a restart. + # + # puma cluster has two types of restarts: + # * SIGUSR1: phased restart + # * SIGUSR2: restart + # + # phased restart is not supported in our configuration, because we use + # preload_app. this means we will always perform a normal restart. + # additionally, phased restart is not supported when sending a SIGUSR2 + # directly to a puma worker (as opposed to the master process). + # + # the result is that the behaviour of SIGUSR1 and SIGUSR2 is identical in + # our configuration, and we can always use a SIGUSR1 to perform a restart. + # + # thus, it is acceptable for us to re-appropriate the SIGUSR2 signal, and + # override the puma behaviour. + # + # see also: + # * https://github.com/puma/puma/blob/master/docs/signals.md#puma-signals + # * https://github.com/phusion/unicorn/blob/master/SIGNALS + # * https://github.com/mperham/sidekiq/wiki/Signals + Signal.trap('SIGUSR2') do + write.write('.') + end + end + + def self.log_event(event, labels = {}) + Gitlab::AppJsonLogger.info({ + event: 'stackprof', + message: event, + pid: Process.pid + }.merge(labels.compact)) + end + end +end diff --git a/lib/gitlab/static_site_editor/config/generated_config.rb b/lib/gitlab/static_site_editor/config/generated_config.rb index 0a2cee75af7..1555c3469a5 100644 --- a/lib/gitlab/static_site_editor/config/generated_config.rb +++ b/lib/gitlab/static_site_editor/config/generated_config.rb @@ -42,11 +42,11 @@ module Gitlab end def supported_content? - master_branch? && extension_supported? && file_exists? + branch_supported? && extension_supported? && file_exists? end - def master_branch? - ref == 'master' + def branch_supported? + ref.in?(%w[master main]) end def extension_supported? diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb index 3072210d7c8..ab2e1404cd2 100644 --- a/lib/gitlab/subscription_portal.rb +++ b/lib/gitlab/subscription_portal.rb @@ -9,8 +9,13 @@ module Gitlab def self.subscriptions_url ENV.fetch('CUSTOMER_PORTAL_URL', default_subscriptions_url) end + + def self.payment_form_url + "#{self.subscriptions_url}/payment_forms/cc_validation" + end end end -Gitlab::SubscriptionPortal.prepend_if_jh('JH::Gitlab::SubscriptionPortal') +Gitlab::SubscriptionPortal.prepend_mod Gitlab::SubscriptionPortal::SUBSCRIPTIONS_URL = Gitlab::SubscriptionPortal.subscriptions_url.freeze +Gitlab::SubscriptionPortal::PAYMENT_FORM_URL = Gitlab::SubscriptionPortal.payment_form_url.freeze diff --git a/lib/gitlab/suggestions/suggestion_set.rb b/lib/gitlab/suggestions/suggestion_set.rb index f9a635734a3..53885cdbf19 100644 --- a/lib/gitlab/suggestions/suggestion_set.rb +++ b/lib/gitlab/suggestions/suggestion_set.rb @@ -39,6 +39,10 @@ module Gitlab @file_paths ||= suggestions.map(&:file_path).uniq end + def authors + suggestions.map { |suggestion| suggestion.note.author }.uniq + end + private def first_suggestion diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index db3c058184c..1ceccc64ec0 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -109,7 +109,7 @@ module Gitlab def run_command!(command) output, status = Gitlab::Popen.popen(command) - raise Gitlab::TaskFailedError.new(output) unless status == 0 + raise Gitlab::TaskFailedError, output unless status == 0 output end diff --git a/lib/gitlab/tcp_checker.rb b/lib/gitlab/tcp_checker.rb index f37a044b607..a07a2e8786b 100644 --- a/lib/gitlab/tcp_checker.rb +++ b/lib/gitlab/tcp_checker.rb @@ -30,7 +30,7 @@ module Gitlab end true - rescue => err + rescue StandardError => err @error = err false diff --git a/lib/gitlab/template/gitlab_ci_syntax_yml_template.rb b/lib/gitlab/template/gitlab_ci_syntax_yml_template.rb deleted file mode 100644 index 3bf3a28d3c5..00000000000 --- a/lib/gitlab/template/gitlab_ci_syntax_yml_template.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Template - class GitlabCiSyntaxYmlTemplate < BaseTemplate - class << self - def extension - '.gitlab-ci.yml' - end - - def categories - { - 'General' => '' - } - end - - def base_dir - Rails.root.join('lib/gitlab/ci/syntax_templates') - end - - def finder(project = nil) - Gitlab::Template::Finders::GlobalTemplateFinder.new( - self.base_dir, self.extension, self.categories - ) - end - end - end - end -end diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index 01158cafc4f..e1ca4b5ff6a 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -59,4 +59,4 @@ module Gitlab end end -Gitlab::Template::GitlabCiYmlTemplate.prepend_if_ee('::EE::Gitlab::Template::GitlabCiYmlTemplate') +Gitlab::Template::GitlabCiYmlTemplate.prepend_mod_with('Gitlab::Template::GitlabCiYmlTemplate') diff --git a/lib/gitlab/terraform_registry_token.rb b/lib/gitlab/terraform_registry_token.rb new file mode 100644 index 00000000000..ae7df49835f --- /dev/null +++ b/lib/gitlab/terraform_registry_token.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + class TerraformRegistryToken < JWTToken + class << self + def from_token(token) + new.tap do |terraform_registry_token| + terraform_registry_token['token'] = token.try(:token).presence || token.try(:id).presence + end + end + end + end +end diff --git a/lib/gitlab/throttle.rb b/lib/gitlab/throttle.rb index 520075012e8..8f045021088 100644 --- a/lib/gitlab/throttle.rb +++ b/lib/gitlab/throttle.rb @@ -49,6 +49,20 @@ module Gitlab { limit: limit_proc, period: period_proc } end + def self.unauthenticated_packages_api_options + limit_proc = proc { |req| settings.throttle_unauthenticated_packages_api_requests_per_period } + period_proc = proc { |req| settings.throttle_unauthenticated_packages_api_period_in_seconds.seconds } + + { limit: limit_proc, period: period_proc } + end + + def self.authenticated_packages_api_options + limit_proc = proc { |req| settings.throttle_authenticated_packages_api_requests_per_period } + period_proc = proc { |req| settings.throttle_authenticated_packages_api_period_in_seconds.seconds } + + { limit: limit_proc, period: period_proc } + end + def self.rate_limiting_response_text (settings.rate_limiting_response_text.presence || DEFAULT_RATE_LIMITING_RESPONSE_TEXT) + "\n" end diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb index b15cb85dde0..bfdfb01093f 100644 --- a/lib/gitlab/time_tracking_formatter.rb +++ b/lib/gitlab/time_tracking_formatter.rb @@ -15,7 +15,7 @@ module Gitlab ChronicDuration.parse( string, CUSTOM_DAY_AND_MONTH_LENGTH.merge(default_unit: 'hours')) - rescue + rescue StandardError nil end @@ -30,7 +30,7 @@ module Gitlab format: :short, limit_to_hours: limit_to_hours_setting, weeks: true)) - rescue + rescue StandardError nil end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index b16ae39bcee..5fb360296b7 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -14,7 +14,7 @@ module Gitlab snowplow.event(category, action, label: label, property: property, value: value, context: contexts) product_analytics.event(category, action, label: label, property: property, value: value, context: contexts) - rescue => error + rescue StandardError => error Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action) end diff --git a/lib/gitlab/tracking/docs/helper.rb b/lib/gitlab/tracking/docs/helper.rb new file mode 100644 index 00000000000..81874aac9a5 --- /dev/null +++ b/lib/gitlab/tracking/docs/helper.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + module Docs + # Helper with functions to be used by HAML templates + module Helper + def auto_generated_comment + <<-MARKDOWN.strip_heredoc + --- + stage: Growth + group: Product Intelligence + info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers + --- + + <!--- + This documentation is auto generated by a script. + + Please do not edit this file directly, check generate_metrics_dictionary task on lib/tasks/gitlab/usage_data.rake. + ---> + + <!-- vale gitlab.Spelling = NO --> + MARKDOWN + end + + def render_description(object) + return 'Missing description' unless object.description.present? + + object.description + end + + def render_event_taxonomy(object) + headers = %w[category action label property value] + values = %i[category action label property_description value_description] + values = values.map { |key| backtick(object.attributes[key]) } + values = values.join(" | ") + + [ + "| #{headers.join(" | ")} |", + "#{'|---' * headers.size}|", + "| #{values} |" + ].join("\n") + end + + def md_link_to(anchor_text, url) + "[#{anchor_text}](#{url})" + end + + def render_owner(object) + "Owner: #{backtick(object.product_group)}" + end + + def render_tiers(object) + "Tiers: #{object.tiers.map(&method(:backtick)).join(', ')}" + end + + def render_yaml_definition_path(object) + "YAML definition: #{backtick(object.yaml_path)}" + end + + def backtick(string) + "`#{string}`" + end + end + end + end +end diff --git a/lib/gitlab/tracking/docs/renderer.rb b/lib/gitlab/tracking/docs/renderer.rb new file mode 100644 index 00000000000..184b935c2ba --- /dev/null +++ b/lib/gitlab/tracking/docs/renderer.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + module Docs + class Renderer + include Gitlab::Tracking::Docs::Helper + DICTIONARY_PATH = Rails.root.join('doc', 'development', 'snowplow') + TEMPLATE_PATH = Rails.root.join('lib', 'gitlab', 'tracking', 'docs', 'templates', 'default.md.haml') + + def initialize(event_definitions) + @layout = Haml::Engine.new(File.read(TEMPLATE_PATH)) + @event_definitions = event_definitions.sort + end + + def contents + # Render and remove an extra trailing new line + @contents ||= @layout.render(self, event_definitions: @event_definitions).sub!(/\n(?=\Z)/, '') + end + + def write + filename = DICTIONARY_PATH.join('dictionary.md').to_s + + FileUtils.mkdir_p(DICTIONARY_PATH) + File.write(filename, contents) + + filename + end + end + end + end +end diff --git a/lib/gitlab/tracking/docs/templates/default.md.haml b/lib/gitlab/tracking/docs/templates/default.md.haml new file mode 100644 index 00000000000..568f56590fa --- /dev/null +++ b/lib/gitlab/tracking/docs/templates/default.md.haml @@ -0,0 +1,35 @@ += auto_generated_comment + +:plain + # Event Dictionary + + This file is autogenerated, please do not edit it directly. + + To generate these files from the GitLab repository, run: + + ```shell + bundle exec rake gitlab:snowplow:generate_event_dictionary + ``` + + The Event Dictionary is based on the following event definition YAML files: + + - [`config/events`](https://gitlab.com/gitlab-org/gitlab/-/tree/f9a404301ca22d038e7b9a9eb08d9c1bbd6c4d84/config/events) + - [`ee/config/events`](https://gitlab.com/gitlab-org/gitlab/-/tree/f9a404301ca22d038e7b9a9eb08d9c1bbd6c4d84/ee/config/events) + + ## Event definitions + +\ +- event_definitions.each do |_path, object| + + = "### `#{object.category} #{object.action}`" + \ + = render_event_taxonomy(object) + \ + = render_description(object) + \ + = render_yaml_definition_path(object) + \ + = render_owner(object) + \ + = render_tiers(object) + \ diff --git a/lib/gitlab/tracking/event_definition.rb b/lib/gitlab/tracking/event_definition.rb new file mode 100644 index 00000000000..8f70c8ecab7 --- /dev/null +++ b/lib/gitlab/tracking/event_definition.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + InvalidEventError = Class.new(RuntimeError) + + class EventDefinition + EVENT_SCHEMA_PATH = Rails.root.join('config', 'events', 'schema.json') + BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master' + SCHEMA = ::JSONSchemer.schema(Pathname.new(EVENT_SCHEMA_PATH)) + + attr_reader :path + attr_reader :attributes + + class << self + def paths + @paths ||= [Rails.root.join('config', 'events', '*.yml'), Rails.root.join('ee', 'config', 'events', '*.yml')] + end + + def definitions + paths.each_with_object({}) do |glob_path, definitions| + load_all_from_path!(definitions, glob_path) + end + end + + private + + def load_from_file(path) + definition = File.read(path) + definition = YAML.safe_load(definition) + definition.deep_symbolize_keys! + + self.new(path, definition).tap(&:validate!) + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Tracking::InvalidEventError.new(e.message)) + end + + def load_all_from_path!(definitions, glob_path) + Dir.glob(glob_path).each do |path| + definition = load_from_file(path) + definitions[definition.path] = definition + end + end + end + + def initialize(path, opts = {}) + @path = path + @attributes = opts + end + + def to_h + attributes + end + alias_method :to_dictionary, :to_h + + def yaml_path + path.delete_prefix(Rails.root.to_s) + end + + def validate! + SCHEMA.validate(attributes.stringify_keys).each do |error| + error_message = <<~ERROR_MSG + Error type: #{error['type']} + Data: #{error['data']} + Path: #{error['data_pointer']} + Details: #{error['details']} + Definition file: #{path} + ERROR_MSG + + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Tracking::InvalidEventError.new(error_message)) + end + end + + private + + def method_missing(method, *args) + attributes[method] || super + end + end + end +end diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index da030649f76..7902f96dfa6 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -3,10 +3,12 @@ module Gitlab module Tracking class StandardContext - GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-4' + GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-5' GITLAB_RAILS_SOURCE = 'gitlab-rails' def initialize(namespace: nil, project: nil, user: nil, **extra) + @namespace = namespace + @plan = @namespace&.actual_plan_name @extra = extra end @@ -36,6 +38,7 @@ module Gitlab { environment: environment, source: source, + plan: @plan, extra: @extra } end diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb index 86cd91f0a32..85f0ba1fd25 100644 --- a/lib/gitlab/tree_summary.rb +++ b/lib/gitlab/tree_summary.rb @@ -161,4 +161,4 @@ module Gitlab end end -Gitlab::TreeSummary.prepend_if_ee('::EE::Gitlab::TreeSummary') +Gitlab::TreeSummary.prepend_mod_with('Gitlab::TreeSummary') diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index 706c0925302..09236a7f1f0 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -22,7 +22,7 @@ module Gitlab @regexp = RE2::Regexp.new(pattern, log_errors: false) - raise RegexpError.new(regexp.error) unless regexp.ok? + raise RegexpError, regexp.error unless regexp.ok? end def replace_all(text, rewrite) diff --git a/lib/gitlab/uploads/migration_helper.rb b/lib/gitlab/uploads/migration_helper.rb index b610d2a10c6..deab2cd43a6 100644 --- a/lib/gitlab/uploads/migration_helper.rb +++ b/lib/gitlab/uploads/migration_helper.rb @@ -76,4 +76,4 @@ module Gitlab end end -Gitlab::Uploads::MigrationHelper.prepend_if_ee('EE::Gitlab::Uploads::MigrationHelper') +Gitlab::Uploads::MigrationHelper.prepend_mod_with('Gitlab::Uploads::MigrationHelper') diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index f98c488bbe5..a242f718b16 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -49,7 +49,7 @@ module Gitlab when ::DesignManagement::Design design_url(object, **options) else - raise NotImplementedError.new("No URL builder defined for #{object.inspect}") + raise NotImplementedError, "No URL builder defined for #{object.inspect}" end end # rubocop:enable Metrics/CyclomaticComplexity @@ -127,4 +127,4 @@ module Gitlab end end -::Gitlab::UrlBuilder.prepend_if_ee('EE::Gitlab::UrlBuilder') +::Gitlab::UrlBuilder.prepend_mod_with('Gitlab::UrlBuilder') diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb index 6b185a5a1e9..c2e5d467dbb 100644 --- a/lib/gitlab/usage/docs/helper.rb +++ b/lib/gitlab/usage/docs/helper.rb @@ -18,8 +18,6 @@ module Gitlab Please do not edit this file directly, check generate_metrics_dictionary task on lib/tasks/gitlab/usage_data.rake. ---> - - <!-- vale gitlab.Spelling = NO --> MARKDOWN end diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml index 26f1aa4396d..8911ac2ed1a 100644 --- a/lib/gitlab/usage/docs/templates/default.md.haml +++ b/lib/gitlab/usage/docs/templates/default.md.haml @@ -19,6 +19,10 @@ Each table includes a `milestone`, which corresponds to the GitLab version when the metric was released. + <!-- vale off --> + <!-- Docs linting disabled after this line. --> + <!-- See https://docs.gitlab.com/ee/development/documentation/testing.html#disable-vale-tests --> + ## Metrics Definitions \ diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 9c4255a7c92..ccd2c69e2e7 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -26,11 +26,11 @@ module Gitlab def json_schema_path return '' unless has_json_schema? - "#{BASE_REPO_PATH}/#{attributes[:object_json_schema]}" + "#{BASE_REPO_PATH}/#{attributes[:value_json_schema]}" end def has_json_schema? - attributes[:value_type] == 'object' && attributes[:object_json_schema].present? + attributes[:value_type] == 'object' && attributes[:value_json_schema].present? end def yaml_path @@ -65,6 +65,10 @@ module Gitlab @definitions ||= load_all! end + def all + @all ||= definitions.map { |_key_path, definition| definition } + end + def schemer @schemer ||= ::JSONSchemer.schema(Pathname.new(METRIC_SCHEMA_PATH)) end @@ -87,7 +91,7 @@ module Gitlab definition.deep_symbolize_keys! self.new(path, definition).tap(&:validate!) - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new(e.message)) end @@ -117,4 +121,4 @@ module Gitlab end end -Gitlab::Usage::MetricDefinition.prepend_if_ee('EE::Gitlab::Usage::MetricDefinition') +Gitlab::Usage::MetricDefinition.prepend_mod_with('Gitlab::Usage::MetricDefinition') diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index f77c8cab39c..4c40bfbc06f 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -83,7 +83,7 @@ module Gitlab when UNION_OF_AGGREGATED_METRICS source.calculate_metrics_union(metric_names: aggregation[:events], start_date: start_date, end_date: end_date, recorded_at: recorded_at) when INTERSECTION_OF_AGGREGATED_METRICS - calculate_metrics_intersections(source: source, metric_names: aggregation[:events], start_date: start_date, end_date: end_date) + source.calculate_metrics_intersections(metric_names: aggregation[:events], start_date: start_date, end_date: end_date, recorded_at: recorded_at) else Gitlab::ErrorTracking .track_and_raise_for_dev_exception(UnknownAggregationOperator.new("Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}")) @@ -94,67 +94,6 @@ module Gitlab Gitlab::Utils::UsageData::FALLBACK end - # calculate intersection of 'n' sets based on inclusion exclusion principle https://en.wikipedia.org/wiki/Inclusion%E2%80%93exclusion_principle - # this method will be extracted to dedicated module with https://gitlab.com/gitlab-org/gitlab/-/issues/273391 - def calculate_metrics_intersections(source:, metric_names:, start_date:, end_date:, subset_powers_cache: Hash.new({})) - # calculate power of intersection of all given metrics from inclusion exclusion principle - # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C|) => - # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| - # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => - # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| - - # calculate each components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... - subset_powers_data = subsets_intersection_powers(source, metric_names, start_date, end_date, subset_powers_cache) - - # calculate last component of the equation |A & B & C & D| = .... - |A + B + C + D| - power_of_union_of_all_metrics = begin - subset_powers_cache[metric_names.size][metric_names.join('_+_')] ||= \ - source.calculate_metrics_union(metric_names: metric_names, start_date: start_date, end_date: end_date, recorded_at: recorded_at) - end - - # in order to determine if part of equation (|A & B & C|, |A & B & C & D|), that represents the intersection that we need to calculate, - # is positive or negative in particular equation we need to determine if number of subsets is even or odd. Please take a look at two examples below - # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + |A & B & C| => - # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| - # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => - # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| - subset_powers_size_even = subset_powers_data.size.even? - - # sum all components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... => - sum_of_all_subset_powers = sum_subset_powers(subset_powers_data, subset_powers_size_even) - - # add last component of the equation |A & B & C & D| = sum_of_all_subset_powers - |A + B + C + D| - sum_of_all_subset_powers + (subset_powers_size_even ? power_of_union_of_all_metrics : -power_of_union_of_all_metrics) - end - - def sum_subset_powers(subset_powers_data, subset_powers_size_even) - sum_without_sign = subset_powers_data.to_enum.with_index.sum do |value, index| - (index + 1).odd? ? value : -value - end - - (subset_powers_size_even ? -1 : 1) * sum_without_sign - end - - def subsets_intersection_powers(source, metric_names, start_date, end_date, subset_powers_cache) - subset_sizes = (1...metric_names.size) - - subset_sizes.map do |subset_size| - if subset_size > 1 - # calculate sum of powers of intersection between each subset (with given size) of metrics: #|A + B + C + D| = ... - (|A & B| + |A & C| + .. + |C & D|) - metric_names.combination(subset_size).sum do |metrics_subset| - subset_powers_cache[subset_size][metrics_subset.join('_&_')] ||= - calculate_metrics_intersections(source: source, metric_names: metrics_subset, start_date: start_date, end_date: end_date, subset_powers_cache: subset_powers_cache) - end - else - # calculate sum of powers of each set (metric) alone #|A + B + C + D| = (|A| + |B| + |C| + |D|) - ... - metric_names.sum do |metric| - subset_powers_cache[subset_size][metric] ||= \ - source.calculate_metrics_union(metric_names: metric, start_date: start_date, end_date: end_date, recorded_at: recorded_at) - end - end - end - end - def load_metrics(wildcard) Dir[wildcard].each_with_object([]) do |path, metrics| metrics.push(*load_yaml_from_path(path)) @@ -170,4 +109,4 @@ module Gitlab end end -Gitlab::Usage::Metrics::Aggregates::Aggregate.prepend_if_ee('EE::Gitlab::Usage::Metrics::Aggregates::Aggregate') +Gitlab::Usage::Metrics::Aggregates::Aggregate.prepend_mod_with('Gitlab::Usage::Metrics::Aggregates::Aggregate') diff --git a/lib/gitlab/usage/metrics/aggregates/sources/calculations/intersection.rb b/lib/gitlab/usage/metrics/aggregates/sources/calculations/intersection.rb new file mode 100644 index 00000000000..dabf757c8a7 --- /dev/null +++ b/lib/gitlab/usage/metrics/aggregates/sources/calculations/intersection.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Aggregates + module Sources + module Calculations + module Intersection + def calculate_metrics_intersections(metric_names:, start_date:, end_date:, recorded_at:, subset_powers_cache: Hash.new({})) + # calculate power of intersection of all given metrics from inclusion exclusion principle + # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C|) => + # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| + # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => + # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| + + # calculate each components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... + subset_powers_data = subsets_intersection_powers(metric_names, start_date, end_date, recorded_at, subset_powers_cache) + + # calculate last component of the equation |A & B & C & D| = .... - |A + B + C + D| + power_of_union_of_all_metrics = begin + subset_powers_cache[metric_names.size][metric_names.join('_+_')] ||= \ + calculate_metrics_union(metric_names: metric_names, start_date: start_date, end_date: end_date, recorded_at: recorded_at) + end + + # in order to determine if part of equation (|A & B & C|, |A & B & C & D|), that represents the intersection that we need to calculate, + # is positive or negative in particular equation we need to determine if number of subsets is even or odd. Please take a look at two examples below + # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + |A & B & C| => + # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| + # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => + # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| + subset_powers_size_even = subset_powers_data.size.even? + + # sum all components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... => + sum_of_all_subset_powers = sum_subset_powers(subset_powers_data, subset_powers_size_even) + + # add last component of the equation |A & B & C & D| = sum_of_all_subset_powers - |A + B + C + D| + sum_of_all_subset_powers + (subset_powers_size_even ? power_of_union_of_all_metrics : -power_of_union_of_all_metrics) + end + + private + + def subsets_intersection_powers(metric_names, start_date, end_date, recorded_at, subset_powers_cache) + subset_sizes = (1...metric_names.size) + + subset_sizes.map do |subset_size| + if subset_size > 1 + # calculate sum of powers of intersection between each subset (with given size) of metrics: #|A + B + C + D| = ... - (|A & B| + |A & C| + .. + |C & D|) + metric_names.combination(subset_size).sum do |metrics_subset| + subset_powers_cache[subset_size][metrics_subset.join('_&_')] ||= + calculate_metrics_intersections(metric_names: metrics_subset, start_date: start_date, end_date: end_date, recorded_at: recorded_at, subset_powers_cache: subset_powers_cache) + end + else + # calculate sum of powers of each set (metric) alone #|A + B + C + D| = (|A| + |B| + |C| + |D|) - ... + metric_names.sum do |metric| + subset_powers_cache[subset_size][metric] ||= \ + calculate_metrics_union(metric_names: metric, start_date: start_date, end_date: end_date, recorded_at: recorded_at) + end + end + end + end + + def sum_subset_powers(subset_powers_data, subset_powers_size_even) + sum_without_sign = subset_powers_data.to_enum.with_index.sum do |value, index| + (index + 1).odd? ? value : -value + end + + (subset_powers_size_even ? -1 : 1) * sum_without_sign + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb index a01efbdb1a6..3069afab147 100644 --- a/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb +++ b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb @@ -6,6 +6,7 @@ module Gitlab module Aggregates module Sources class PostgresHll + extend Calculations::Intersection class << self def calculate_metrics_union(metric_names:, start_date:, end_date:, recorded_at:) time_period = start_date && end_date ? (start_date..end_date) : nil diff --git a/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb b/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb index f3a4dcf1e31..009b8e62543 100644 --- a/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb +++ b/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb @@ -8,6 +8,7 @@ module Gitlab UnionNotAvailable = Class.new(AggregatedMetricError) class RedisHll + extend Calculations::Intersection def self.calculate_metrics_union(metric_names:, start_date:, end_date:, recorded_at: nil) union = Gitlab::UsageDataCounters::HLLRedisCounter .calculate_events_union(event_names: metric_names, start_date: start_date, end_date: end_date) diff --git a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb new file mode 100644 index 00000000000..29b44f2bd0a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class BaseMetric + include Gitlab::Utils::UsageData + + attr_reader :time_frame + + def initialize(time_frame:) + @time_frame = time_frame + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_boards_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_boards_metric.rb new file mode 100644 index 00000000000..4e1ba027bca --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_boards_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountBoardsMetric < DatabaseMetric + operation :count + + relation { Board } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_issues_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_issues_metric.rb new file mode 100644 index 00000000000..34247f4f6dd --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_issues_metric.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountIssuesMetric < DatabaseMetric + operation :count + + start { Issue.minimum(:id) } + finish { Issue.maximum(:id) } + + relation { Issue } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_users_creating_issues_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_users_creating_issues_metric.rb new file mode 100644 index 00000000000..c8331ce5b31 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_users_creating_issues_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountUsersCreatingIssuesMetric < DatabaseMetric + operation :distinct_count, column: :author_id + + relation { Issue } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_users_using_approve_quick_action_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_users_using_approve_quick_action_metric.rb new file mode 100644 index 00000000000..9c92f2e9595 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_users_using_approve_quick_action_metric.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountUsersUsingApproveQuickActionMetric < RedisHLLMetric + event_names :i_quickactions_approve + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb new file mode 100644 index 00000000000..f83f90dea03 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class DatabaseMetric < BaseMetric + # Usage Example + # + # class CountUsersCreatingIssuesMetric < DatabaseMetric + # operation :distinct_count, column: :author_id + # + # relation do |database_time_constraints| + # ::Issue.where(database_time_constraints) + # end + # end + class << self + def start(&block) + @metric_start = block + end + + def finish(&block) + @metric_finish = block + end + + def relation(&block) + @metric_relation = block + end + + def operation(symbol, column: nil) + @metric_operation = symbol + @column = column + end + + attr_reader :metric_operation, :metric_relation, :metric_start, :metric_finish, :column + end + + def value + method(self.class.metric_operation) + .call(relation, + self.class.column, + start: self.class.metric_start&.call, + finish: self.class.metric_finish&.call) + end + + def relation + self.class.metric_relation.call.where(time_constraints) + end + + private + + def time_constraints + case time_frame + when '28d' + { created_at: 30.days.ago..2.days.ago } + when 'all' + {} + when 'none' + nil + else + raise "Unknown time frame: #{time_frame} for DatabaseMetric" + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb new file mode 100644 index 00000000000..7c97cc37d17 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class GenericMetric < BaseMetric + # Usage example + # + # class UuidMetric < GenericMetric + # value do + # Gitlab::CurrentSettings.uuid + # end + # end + class << self + def value(&block) + @metric_value = block + end + + attr_reader :metric_value + end + + def value + alt_usage_data do + self.class.metric_value.call + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/hostname_metric.rb b/lib/gitlab/usage/metrics/instrumentations/hostname_metric.rb new file mode 100644 index 00000000000..3364c330cca --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/hostname_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class HostnameMetric < GenericMetric + value do + Gitlab.config.gitlab.host + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb new file mode 100644 index 00000000000..140d56f0d42 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class RedisHLLMetric < BaseMetric + # Usage example + # + # class CountUsersVisitingAnalyticsValuestreamMetric < RedisHLLMetric + # event_names :g_analytics_valuestream + # end + class << self + def event_names(events = nil) + @metric_events = events + end + + attr_reader :metric_events + end + + def value + redis_usage_data do + event_params = time_constraints.merge(event_names: self.class.metric_events) + + Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(**event_params) + end + end + + private + + def time_constraints + case time_frame + when '28d' + { start_date: 4.weeks.ago.to_date, end_date: Date.current } + when '7d' + { start_date: 7.days.ago.to_date, end_date: Date.current } + else + raise "Unknown time frame: #{time_frame} for TimeConstraint" + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/uuid_metric.rb b/lib/gitlab/usage/metrics/instrumentations/uuid_metric.rb new file mode 100644 index 00000000000..58547b5383a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/uuid_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class UuidMetric < GenericMetric + value do + Gitlab::CurrentSettings.uuid + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/key_path_processor.rb b/lib/gitlab/usage/metrics/key_path_processor.rb new file mode 100644 index 00000000000..dbe574d5838 --- /dev/null +++ b/lib/gitlab/usage/metrics/key_path_processor.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + class KeyPathProcessor + class << self + def process(key_path, value) + unflatten(key_path.split('.'), value) + end + + private + + def unflatten(keys, value) + loop do + value = { keys.pop.to_sym => value } + + break if keys.blank? + end + + value + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index b36ca38cd64..b1ba529d4a4 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -98,7 +98,6 @@ module Gitlab ci_external_pipelines: count(::Ci::Pipeline.external), ci_pipeline_config_auto_devops: count(::Ci::Pipeline.auto_devops_source), ci_pipeline_config_repository: count(::Ci::Pipeline.repository_source), - ci_runners: count(::Ci::Runner), ci_triggers: count(::Ci::Trigger), ci_pipeline_schedules: count(::Ci::PipelineSchedule), auto_devops_enabled: count(::ProjectAutoDevops.enabled), @@ -164,7 +163,6 @@ module Gitlab projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)), projects_with_tracing_enabled: count(ProjectTracingSetting), projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), - projects_with_alerts_service_enabled: count(Service.active.where(type: 'AlertsService')), projects_with_alerts_created: distinct_count(::AlertManagement::Alert, :project_id), projects_with_enabled_alert_integrations: distinct_count(::AlertManagement::HttpIntegration.active, :project_id), projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id), @@ -186,12 +184,14 @@ module Gitlab merge_requests: count(MergeRequest), notes: count(Note) }.merge( + runners_usage, services_usage, usage_counters, user_preferences_usage, ingress_modsecurity_usage, container_expiration_policies_usage, - service_desk_counts + service_desk_counts, + email_campaign_counts ).tap do |data| data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) end @@ -199,6 +199,18 @@ module Gitlab end # rubocop: enable Metrics/AbcSize + def runners_usage + { + ci_runners: count(::Ci::Runner), + ci_runners_instance_type_active: count(::Ci::Runner.instance_type.active), + ci_runners_group_type_active: count(::Ci::Runner.group_type.active), + ci_runners_project_type_active: count(::Ci::Runner.project_type.active), + ci_runners_instance_type_active_online: count(::Ci::Runner.instance_type.active.online), + ci_runners_group_type_active_online: count(::Ci::Runner.group_type.active.online), + ci_runners_project_type_active_online: count(::Ci::Runner.project_type.active.online) + } + end + def snowplow_event_counts(time_period) return {} unless report_snowplow_events? @@ -243,7 +255,8 @@ module Gitlab { settings: { ldap_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth::Ldap::Config.encrypted_secrets.active? }, - operating_system: alt_usage_data(fallback: nil) { operating_system } + operating_system: alt_usage_data(fallback: nil) { operating_system }, + gitaly_apdex: alt_usage_data { gitaly_apdex } } } end @@ -414,13 +427,15 @@ module Gitlab def services_usage # rubocop: disable UsageData/LargeTable: - Service.available_services_names.each_with_object({}) do |service_name, response| - response["projects_#{service_name}_active".to_sym] = count(Service.active.where.not(project: nil).where(type: "#{service_name}_service".camelize)) - response["groups_#{service_name}_active".to_sym] = count(Service.active.where.not(group: nil).where(type: "#{service_name}_service".camelize)) - response["templates_#{service_name}_active".to_sym] = count(Service.active.where(template: true, type: "#{service_name}_service".camelize)) - response["instances_#{service_name}_active".to_sym] = count(Service.active.where(instance: true, type: "#{service_name}_service".camelize)) - response["projects_inheriting_#{service_name}_active".to_sym] = count(Service.active.where.not(project: nil).where.not(inherit_from_id: nil).where(type: "#{service_name}_service".camelize)) - response["groups_inheriting_#{service_name}_active".to_sym] = count(Service.active.where.not(group: nil).where.not(inherit_from_id: nil).where(type: "#{service_name}_service".camelize)) + Integration.available_services_names(include_dev: false).each_with_object({}) do |service_name, response| + service_type = Integration.service_name_to_type(service_name) + + response["projects_#{service_name}_active".to_sym] = count(Integration.active.where.not(project: nil).where(type: service_type)) + response["groups_#{service_name}_active".to_sym] = count(Integration.active.where.not(group: nil).where(type: service_type)) + response["templates_#{service_name}_active".to_sym] = count(Integration.active.where(template: true, type: service_type)) + response["instances_#{service_name}_active".to_sym] = count(Integration.active.where(instance: true, type: service_type)) + response["projects_inheriting_#{service_name}_active".to_sym] = count(Integration.active.where.not(project: nil).where.not(inherit_from_id: nil).where(type: service_type)) + response["groups_inheriting_#{service_name}_active".to_sym] = count(Integration.active.where.not(group: nil).where.not(inherit_from_id: nil).where(type: service_type)) end.merge(jira_usage, jira_import_usage) # rubocop: enable UsageData/LargeTable: end @@ -435,18 +450,10 @@ module Gitlab projects_jira_dvcs_server_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled(cloud: false)) } - # rubocop: disable UsageData/LargeTable: - JiraService.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services| - counts = services.group_by do |service| - # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 - service_url = service.data_fields&.url || (service.properties && service.properties['url']) - service_url&.include?('.atlassian.net') ? :cloud : :server - end + jira_service_data_hash = jira_service_data + results[:projects_jira_server_active] = jira_service_data_hash[:projects_jira_server_active] + results[:projects_jira_cloud_active] = jira_service_data_hash[:projects_jira_cloud_active] - results[:projects_jira_server_active] += counts[:server].size if counts[:server] - results[:projects_jira_cloud_active] += counts[:cloud].size if counts[:cloud] - end - # rubocop: enable UsageData/LargeTable: results rescue ActiveRecord::StatementInvalid { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK } @@ -570,7 +577,11 @@ module Gitlab projects_with_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: true))), projects_without_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: [false, nil]))), remote_mirrors: distinct_count(::Project.with_remote_mirrors.where(time_period), :creator_id), - snippets: distinct_count(::Snippet.where(time_period), :author_id) + snippets: distinct_count(::Snippet.where(time_period), :author_id), + suggestions: distinct_count(::Note.with_suggestions.where(time_period), + :author_id, + start: minimum_id(::User), + finish: maximum_id(::User)) }.tap do |h| if time_period.present? h[:merge_requests_users] = merge_requests_users(time_period) @@ -597,7 +608,7 @@ module Gitlab unique_users_all_imports: unique_users_all_imports(time_period), bulk_imports: { gitlab: DEPRECATED_VALUE, - gitlab_v1: count(::BulkImport.where(time_period, source_type: :gitlab)) + gitlab_v1: count(::BulkImport.where(**time_period, source_type: :gitlab)) }, project_imports: project_imports(time_period), issue_imports: issue_imports(time_period), @@ -767,6 +778,16 @@ module Gitlab private + def gitaly_apdex + with_prometheus_client(verify: false, fallback: FALLBACK) do |client| + result = client.query('avg_over_time(gitlab_usage_ping:gitaly_apdex:ratio_avg_over_time_5m[1w])').first + + break FALLBACK unless result + + result['value'].last.to_f + end + end + def aggregated_metrics @aggregated_metrics ||= ::Gitlab::Usage::Metrics::Aggregates::Aggregate.new(recorded_at) end @@ -825,6 +846,28 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord + def email_campaign_counts + # rubocop:disable UsageData/LargeTable + sent_emails = count(Users::InProductMarketingEmail.group(:track, :series)) + clicked_emails = count(Users::InProductMarketingEmail.where.not(cta_clicked_at: nil).group(:track, :series)) + + series_amount = Namespaces::InProductMarketingEmailsService::INTERVAL_DAYS.count + + Users::InProductMarketingEmail.tracks.keys.each_with_object({}) do |track, result| + # rubocop: enable UsageData/LargeTable: + 0.upto(series_amount - 1).map do |series| + # When there is an error with the query and it's not the Hash we expect, we return what we got from `count`. + sent_count = sent_emails.is_a?(Hash) ? sent_emails.fetch([track, series], 0) : sent_emails + clicked_count = clicked_emails.is_a?(Hash) ? clicked_emails.fetch([track, series], 0) : clicked_emails + + result["in_product_marketing_email_#{track}_#{series}_sent"] = sent_count + result["in_product_marketing_email_#{track}_#{series}_cta_clicked"] = clicked_count + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + def unique_visit_service strong_memoize(:unique_visit_service) do ::Gitlab::Analytics::UniqueVisits.new @@ -955,4 +998,4 @@ module Gitlab end end -Gitlab::UsageData.prepend_if_ee('EE::Gitlab::UsageData') +Gitlab::UsageData.prepend_mod_with('Gitlab::UsageData') diff --git a/lib/gitlab/usage_data/topology.rb b/lib/gitlab/usage_data/topology.rb index 7f7854c3eb1..b823d6cc2bf 100644 --- a/lib/gitlab/usage_data/topology.rb +++ b/lib/gitlab/usage_data/topology.rb @@ -47,7 +47,7 @@ module Gitlab nodes: topology_node_data(client) }.compact end - rescue => e + rescue StandardError => e @failures << CollectionFailure.new('other', e.class.to_s) {} @@ -183,7 +183,7 @@ module Gitlab @failures << CollectionFailure.new(query_name, 'empty_result') fallback - rescue => e + rescue StandardError => e @failures << CollectionFailure.new(query_name, e.class.to_s) fallback end diff --git a/lib/gitlab/usage_data_counters/counter_events/package_events.yml b/lib/gitlab/usage_data_counters/counter_events/package_events.yml index e1648245f3f..dd66a40a48f 100644 --- a/lib/gitlab/usage_data_counters/counter_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/counter_events/package_events.yml @@ -47,3 +47,6 @@ - i_package_tag_delete_package - i_package_tag_pull_package - i_package_tag_push_package +- i_package_terraform_module_delete_package +- i_package_terraform_module_pull_package +- i_package_terraform_module_push_package diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb index bef3fc7b504..bc0126cd893 100644 --- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -50,7 +50,6 @@ module Gitlab private def track_unique_action(action, author, time) - return unless Feature.enabled?(:track_editor_edit_actions, default_enabled: true) return unless author Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id, time: time) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index a8691169fb8..833eebd5d04 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -132,7 +132,7 @@ module Gitlab return unless feature_enabled?(event) Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: expiry(event)) - rescue => e + rescue StandardError => e # Ignore any exceptions unless is dev or test env # The application flow should not be blocked by erros in tracking Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) @@ -232,8 +232,8 @@ module Gitlab # Compose the key in order to store events daily or weekly def redis_key(event, time, context = '') - raise UnknownEvent.new("Unknown event #{event[:name]}") unless known_events_names.include?(event[:name].to_s) - raise UnknownAggregation.new("Use :daily or :weekly aggregation") unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym) + raise UnknownEvent, "Unknown event #{event[:name]}" unless known_events_names.include?(event[:name].to_s) + raise UnknownAggregation, "Use :daily or :weekly aggregation" unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym) key = apply_slot(event) key = apply_time_aggregation(key, time, event) @@ -277,4 +277,4 @@ module Gitlab end end -Gitlab::UsageDataCounters::HLLRedisCounter.prepend_if_ee('EE::Gitlab::UsageDataCounters::HLLRedisCounter') +Gitlab::UsageDataCounters::HLLRedisCounter.prepend_mod_with('Gitlab::UsageDataCounters::HLLRedisCounter') diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index 6f5f878501f..083de402175 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -154,4 +154,4 @@ module Gitlab end end -Gitlab::UsageDataCounters::IssueActivityUniqueCounter.prepend_if_ee('EE::Gitlab::UsageDataCounters::IssueActivityUniqueCounter') +Gitlab::UsageDataCounters::IssueActivityUniqueCounter.prepend_mod_with('Gitlab::UsageDataCounters::IssueActivityUniqueCounter') diff --git a/lib/gitlab/usage_data_counters/known_events/analytics.yml b/lib/gitlab/usage_data_counters/known_events/analytics.yml new file mode 100644 index 00000000000..e4f20b61901 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/analytics.yml @@ -0,0 +1,85 @@ +- name: users_viewing_analytics_group_devops_adoption + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: i_analytics_dev_ops_adoption + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: i_analytics_dev_ops_score + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: p_analytics_merge_request + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: i_analytics_instance_statistics + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: g_analytics_contribution + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: g_analytics_insights + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: g_analytics_issues + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: g_analytics_productivity + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: g_analytics_valuestream + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: p_analytics_pipelines + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: p_analytics_code_reviews + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: p_analytics_valuestream + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: p_analytics_insights + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: p_analytics_issues + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: p_analytics_repo + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: i_analytics_cohorts + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index 18c5dc73de2..cc89fbd5caf 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -3,204 +3,219 @@ redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_mr_diffs - name: i_code_review_user_single_file_diffs redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_single_file_diffs - name: i_code_review_mr_single_file_diffs redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_mr_single_file_diffs - name: i_code_review_user_toggled_task_item_status redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_toggled_task_item_status - name: i_code_review_user_create_mr redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_mr - name: i_code_review_user_close_mr redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_close_mr - name: i_code_review_user_reopen_mr redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_reopen_mr - name: i_code_review_user_approve_mr redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_approve_mr - name: i_code_review_user_unapprove_mr redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_unapprove_mr - name: i_code_review_user_resolve_thread redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_resolve_thread - name: i_code_review_user_unresolve_thread redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_unresolve_thread - name: i_code_review_edit_mr_title redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_edit_mr_title - name: i_code_review_edit_mr_desc redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_edit_mr_desc - name: i_code_review_user_merge_mr redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_merge_mr - name: i_code_review_user_create_mr_comment redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_mr_comment - name: i_code_review_user_edit_mr_comment redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_edit_mr_comment - name: i_code_review_user_remove_mr_comment redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_remove_mr_comment - name: i_code_review_user_create_review_note redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_review_note - name: i_code_review_user_publish_review redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_publish_review - name: i_code_review_user_create_multiline_mr_comment redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_multiline_mr_comment - name: i_code_review_user_edit_multiline_mr_comment redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_edit_multiline_mr_comment - name: i_code_review_user_remove_multiline_mr_comment redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_remove_multiline_mr_comment - name: i_code_review_user_add_suggestion redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_add_suggestion - name: i_code_review_user_apply_suggestion redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_apply_suggestion - name: i_code_review_user_assigned redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_assigned - name: i_code_review_user_marked_as_draft redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_marked_as_draft - name: i_code_review_user_unmarked_as_draft redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_unmarked_as_draft - name: i_code_review_user_review_requested redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_review_requested - name: i_code_review_user_approval_rule_added redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_approval_rule_added - name: i_code_review_user_approval_rule_deleted redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_approval_rule_deleted - name: i_code_review_user_approval_rule_edited redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_approval_rule_edited - name: i_code_review_user_vs_code_api_request redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_vs_code_api_request - name: i_code_review_user_create_mr_from_issue redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_mr_from_issue - name: i_code_review_user_mr_discussion_locked redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_mr_discussion_locked - name: i_code_review_user_mr_discussion_unlocked redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_mr_discussion_unlocked - name: i_code_review_user_time_estimate_changed redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_time_estimate_changed - name: i_code_review_user_time_spent_changed redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_time_spent_changed - name: i_code_review_user_assignees_changed redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_assignees_changed - name: i_code_review_user_reviewers_changed redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_reviewers_changed - name: i_code_review_user_milestone_changed redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_milestone_changed - name: i_code_review_user_labels_changed redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_labels_changed +# Diff settings events +- name: i_code_review_click_single_file_mode_setting + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_click_file_browser_setting + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_click_whitespace_setting + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_diff_view_inline + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_diff_view_parallel + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_file_browser_tree_view + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_file_browser_list_view + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_diff_show_whitespace + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_diff_hide_whitespace + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_diff_single_file + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_diff_multiple_files + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 077864032e8..f2504396cc4 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -24,92 +24,6 @@ category: compliance redis_slot: compliance aggregation: weekly - feature_flag: usage_data_a_compliance_audit_events_api -# Analytics category -- name: g_analytics_contribution - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: g_analytics_insights - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: g_analytics_issues - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: g_analytics_productivity - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: g_analytics_valuestream - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: p_analytics_pipelines - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: p_analytics_code_reviews - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: p_analytics_valuestream - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: p_analytics_insights - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: p_analytics_issues - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: p_analytics_repo - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: i_analytics_cohorts - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: i_analytics_dev_ops_score - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: i_analytics_dev_ops_adoption - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: g_analytics_merge_request - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: p_analytics_merge_request - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: i_analytics_instance_statistics - category: analytics - redis_slot: analytics - aggregation: weekly feature_flag: track_unique_visits - name: g_edit_by_web_ide category: ide_edit @@ -139,17 +53,14 @@ category: search redis_slot: search aggregation: weekly - feature_flag: search_track_unique_users - name: i_search_advanced category: search redis_slot: search aggregation: weekly - feature_flag: search_track_unique_users - name: i_search_paid category: search redis_slot: search aggregation: weekly - feature_flag: search_track_unique_users - name: wiki_action category: source_code aggregation: daily @@ -175,52 +86,42 @@ redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_alert_status_changed - name: incident_management_alert_assigned redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_alert_assigned - name: incident_management_alert_todo redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_alert_todo - name: incident_management_incident_created redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_created - name: incident_management_incident_reopened redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_reopened - name: incident_management_incident_closed redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_closed - name: incident_management_incident_assigned redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_assigned - name: incident_management_incident_todo redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_todo - name: incident_management_incident_comment redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_comment - name: incident_management_incident_zoom_meeting redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_zoom_meeting - name: incident_management_incident_published redis_slot: incident_management category: incident_management @@ -230,23 +131,19 @@ redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_relate - name: incident_management_incident_unrelate redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_unrelate - name: incident_management_incident_change_confidential redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_change_confidential # Incident management alerts - name: incident_management_alert_create_incident redis_slot: incident_management category: incident_management_alerts aggregation: weekly - feature_flag: usage_data_incident_management_alert_create_incident # Incident management on-call - name: i_incident_management_oncall_notification_sent redis_slot: incident_management diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml index 1c765bb1830..adc5ba36ad7 100644 --- a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml +++ b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml @@ -24,45 +24,35 @@ category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - name: i_ecosystem_slack_service_push_notification category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - name: i_ecosystem_slack_service_deployment_notification category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - name: i_ecosystem_slack_service_wiki_page_notification category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - name: i_ecosystem_slack_service_merge_request_notification category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - name: i_ecosystem_slack_service_note_notification category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - name: i_ecosystem_slack_service_tag_push_notification category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - name: i_ecosystem_slack_service_confidential_note_notification category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - name: i_ecosystem_slack_service_confidential_issue_notification category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - diff --git a/lib/gitlab/usage_data_counters/known_events/epic_board_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_board_events.yml new file mode 100644 index 00000000000..281db441829 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/epic_board_events.yml @@ -0,0 +1,22 @@ +# Epic board events +# +# We are using the same slot of issue events 'project_management' for +# epic events to allow data aggregation. +# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405 +- name: g_project_management_users_creating_epic_boards + category: epic_boards_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epic_boards_activity + +- name: g_project_management_users_viewing_epic_boards + category: epic_boards_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epic_boards_activity + +- name: g_project_management_users_updating_epic_board_names + category: epic_boards_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epic_boards_activity diff --git a/lib/gitlab/usage_data_counters/known_events/epic_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_events.yml index 80460dbe4d2..d1864cd569b 100644 --- a/lib/gitlab/usage_data_counters/known_events/epic_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/epic_events.yml @@ -9,6 +9,20 @@ aggregation: daily feature_flag: track_epics_activity +# content change events + +- name: project_management_users_unchecking_epic_task + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: project_management_users_checking_epic_task + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + - name: g_project_management_users_updating_epic_titles category: epics_usage redis_slot: project_management @@ -41,6 +55,20 @@ aggregation: daily feature_flag: track_epics_activity +# emoji + +- name: g_project_management_users_awarding_epic_emoji + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_removing_epic_emoji + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + # start date events - name: g_project_management_users_setting_epic_start_date_as_fixed @@ -81,6 +109,8 @@ aggregation: daily feature_flag: track_epics_activity +# relationships + - name: g_project_management_epic_issue_added category: epics_usage redis_slot: project_management @@ -99,6 +129,12 @@ aggregation: daily feature_flag: track_epics_activity +- name: g_project_management_users_updating_epic_parent + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + - name: g_project_management_epic_closed category: epics_usage redis_slot: project_management @@ -140,3 +176,9 @@ redis_slot: project_management aggregation: daily feature_flag: track_epics_activity + +- name: g_project_management_epic_cross_referenced + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml index b7e583003c8..d8ad2b538d6 100644 --- a/lib/gitlab/usage_data_counters/known_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -95,3 +95,11 @@ category: user_packages aggregation: weekly redis_slot: package +- name: i_package_terraform_module_deploy_token + category: deploy_token_packages + aggregation: weekly + redis_slot: package +- name: i_package_terraform_module_user + category: user_packages + aggregation: weekly + redis_slot: package diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index 0fe65afb237..c1eabb352f7 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -3,334 +3,267 @@ category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_assign_single category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_assign_multiple category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_assign_self category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_assign_reviewer category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_award category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_board_move category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_child_epic category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_clear_weight category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_clone category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_close category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_confidential category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_copy_metadata_merge_request category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_copy_metadata_issue category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_create_merge_request category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_done category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_draft category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_due category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_duplicate category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_epic category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_estimate category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_iteration category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_label category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_lock category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_merge category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_milestone category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_move category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_parent_epic category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_promote category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_publish category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_reassign category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_reassign_reviewer category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_rebase category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_relabel category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_relate category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_child_epic category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_due_date category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_epic category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_estimate category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_iteration category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_milestone category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_parent_epic category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_time_spent category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_zoom category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_reopen category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_shrug category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_spend_subtract category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_spend_add category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_submit_review category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_subscribe category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_tableflip category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_tag category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_target_branch category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_title category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_todo category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_unassign_specific category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_unassign_all category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_unassign_reviewer category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_unlabel_specific category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_unlabel_all category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_unlock category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_unsubscribe category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_weight category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_wip category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_zoom category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_invite_email_single category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_invite_email_multiple category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions diff --git a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb index eae42bdc4a1..8b9ca0fc220 100644 --- a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb +++ b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb @@ -4,17 +4,27 @@ module Gitlab module UsageDataCounters class KubernetesAgentCounter < BaseCounter PREFIX = 'kubernetes_agent' - KNOWN_EVENTS = %w[gitops_sync].freeze + KNOWN_EVENTS = %w[gitops_sync k8s_api_proxy_request].freeze class << self - def increment_gitops_sync(incr) - raise ArgumentError, 'must be greater than or equal to zero' if incr < 0 + def increment_event_counts(events) + validate!(events) - # rather then hitting redis for this no-op, we return early - # note: redis returns the increment, so we mimic this here - return 0 if incr == 0 + events.each do |event, incr| + # rather then hitting redis for this no-op, we return early + next if incr == 0 - increment_by(redis_key(:gitops_sync), incr) + increment_by(redis_key(event), incr) + end + end + + private + + def validate!(events) + events.each do |event, incr| + raise ArgumentError, "unknown event #{event}" unless event.in?(KNOWN_EVENTS) + raise ArgumentError, "#{event} count must be greater than or equal to zero" if incr < 0 + end end end end diff --git a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb index ed3df7dcf75..557179ad57a 100644 --- a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb @@ -7,7 +7,6 @@ module Gitlab # Tracks the quick action with name `name`. # `args` is expected to be a single string, will be split internally when necessary. def track_unique_action(name, args:, user:) - return unless Feature.enabled?(:usage_data_track_quickactions, default_enabled: :yaml) return unless user args ||= '' diff --git a/lib/gitlab/usage_data_metrics.rb b/lib/gitlab/usage_data_metrics.rb new file mode 100644 index 00000000000..e181da01229 --- /dev/null +++ b/lib/gitlab/usage_data_metrics.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + class UsageDataMetrics + class << self + # Build the Usage Ping JSON payload from metrics YAML definitions which have instrumentation class set + def uncached_data + ::Gitlab::Usage::MetricDefinition.all.map do |definition| + instrumentation_class = definition.attributes[:instrumentation_class] + + if instrumentation_class.present? + metric_value = "Gitlab::Usage::Metrics::Instrumentations::#{instrumentation_class}".constantize.new(time_frame: definition.attributes[:time_frame]).value + + metric_payload(definition.key_path, metric_value) + else + {} + end + end.reduce({}, :deep_merge) + end + + private + + def metric_payload(key_path, value) + ::Gitlab::Usage::Metrics::KeyPathProcessor.process(key_path, value) + end + end + end +end diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb index 1f72bf4ce26..bc72a96a468 100644 --- a/lib/gitlab/usage_data_non_sql_metrics.rb +++ b/lib/gitlab/usage_data_non_sql_metrics.rb @@ -25,10 +25,17 @@ module Gitlab SQL_METRIC_DEFAULT end - def maximum_id(model) + def maximum_id(model, column = nil) end - def minimum_id(model) + def minimum_id(model, column = nil) + end + + def jira_service_data + { + projects_jira_server_active: 0, + projects_jira_cloud_active: 0 + } end end end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index c0dfae88fc7..1c776501fdb 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -25,6 +25,27 @@ module Gitlab relation.select(relation.all.table[column].sum).to_sql end + # rubocop: disable CodeReuse/ActiveRecord + def histogram(relation, column, buckets:, bucket_size: buckets.size) + count_grouped = relation.group(column).select(Arel.star.count.as('count_grouped')) + cte = Gitlab::SQL::CTE.new(:count_cte, count_grouped) + + bucket_segments = bucket_size - 1 + width_bucket = Arel::Nodes::NamedFunction + .new('WIDTH_BUCKET', [cte.table[:count_grouped], buckets.first, buckets.last, bucket_segments]) + .as('buckets') + + query = cte + .table + .project(width_bucket, cte.table[:count]) + .group('buckets') + .order('buckets') + .with(cte.to_arel) + + query.to_sql + end + # rubocop: enable CodeReuse/ActiveRecord + # For estimated distinct count use exact query instead of hll # buckets query, because it can't be used to obtain estimations without # supplementary ruby code present in Gitlab::Database::PostgresHll::BatchDistinctCounter @@ -36,10 +57,21 @@ module Gitlab 'SELECT ' + args.map {|arg| "(#{arg})" }.join(' + ') end - def maximum_id(model) + def maximum_id(model, column = nil) + end + + def minimum_id(model, column = nil) + end + + def jira_service_data + { + projects_jira_server_active: 0, + projects_jira_cloud_active: 0 + } end - def minimum_id(model) + def epics_deepest_relationship_level + { epics_deepest_relationship_level: 0 } end private diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index c1a57566640..d70e5c3594c 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -16,7 +16,7 @@ module Gitlab path_regex = /(\A(\.{1,2})\z|\A\.\.[\/\\]|[\/\\]\.\.\z|[\/\\]\.\.[\/\\]|\n)/ if path.match?(path_regex) - raise PathTraversalAttackError.new('Invalid path') + raise PathTraversalAttackError, 'Invalid path' end path diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index c92865636d0..39670a835a6 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -43,12 +43,12 @@ module Gitlab instance_method_defined?(parent, method_name) end - raise NotImplementedError.new("#{klass}\##{method_name} doesn't exist!") unless overridden_parent + raise NotImplementedError, "#{klass}\##{method_name} doesn't exist!" unless overridden_parent super_method_arity = find_direct_method(overridden_parent, method_name).arity unless arity_compatible?(sub_method_arity, super_method_arity) - raise NotImplementedError.new("#{subject}\##{method_name} has arity of #{sub_method_arity}, but #{overridden_parent}\##{method_name} has arity of #{super_method_arity}") + raise NotImplementedError, "#{subject}\##{method_name} has arity of #{sub_method_arity}, but #{overridden_parent}\##{method_name} has arity of #{super_method_arity}" end end diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index efa2f7a943f..b1ccdcb1df0 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -121,7 +121,7 @@ module Gitlab count_grouped = relation.group(column).select(Arel.star.count.as('count_grouped')) cte = Gitlab::SQL::CTE.new(:count_cte, count_grouped) - # For example, 9 segements gives 10 buckets + # For example, 9 segments gives 10 buckets bucket_segments = bucket_size - 1 width_bucket = Arel::Nodes::NamedFunction @@ -171,7 +171,7 @@ module Gitlab else value end - rescue + rescue StandardError fallback end @@ -188,7 +188,7 @@ module Gitlab return fallback unless client yield client - rescue + rescue StandardError fallback end @@ -210,20 +210,54 @@ module Gitlab Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values) end - def maximum_id(model) - key = :"#{model.name.downcase}_maximum_id" + def maximum_id(model, column = nil) + key = :"#{model.name.downcase.gsub('::', '_')}_maximum_id" + column_to_read = column || :id + strong_memoize(key) do - model.maximum(:id) + model.maximum(column_to_read) end end - def minimum_id(model) - key = :"#{model.name.downcase}_minimum_id" + # rubocop: disable UsageData/LargeTable: + def jira_service_data + data = { + projects_jira_server_active: 0, + projects_jira_cloud_active: 0 + } + + # rubocop: disable CodeReuse/ActiveRecord + JiraService.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services| + counts = services.group_by do |service| + # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 + service_url = service.data_fields&.url || (service.properties && service.properties['url']) + service_url&.include?('.atlassian.net') ? :cloud : :server + end + + data[:projects_jira_server_active] += counts[:server].size if counts[:server] + data[:projects_jira_cloud_active] += counts[:cloud].size if counts[:cloud] + end + + data + end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: enable UsageData/LargeTable: + + def minimum_id(model, column = nil) + key = :"#{model.name.downcase.gsub('::', '_')}_minimum_id" + column_to_read = column || :id + strong_memoize(key) do - model.minimum(:id) + model.minimum(column_to_read) end end + def epics_deepest_relationship_level + # rubocop: disable UsageData/LargeTable + { epics_deepest_relationship_level: ::Epic.deepest_relationship_level.to_i } + # rubocop: enable UsageData/LargeTable + end + private def prometheus_client(verify:) @@ -237,7 +271,7 @@ module Gitlab api_url = "#{scheme}://#{server_address}" client = Gitlab::PrometheusClient.new(api_url, allow_local_requests: true, verify: verify) break client if client.ready? - rescue + rescue StandardError nil end end diff --git a/lib/gitlab/verify/batch_verifier.rb b/lib/gitlab/verify/batch_verifier.rb index fc114a4e9dd..71d106db742 100644 --- a/lib/gitlab/verify/batch_verifier.rb +++ b/lib/gitlab/verify/batch_verifier.rb @@ -24,11 +24,11 @@ module Gitlab end def name - raise NotImplementedError.new + raise NotImplementedError end def describe(_object) - raise NotImplementedError.new + raise NotImplementedError end private @@ -39,7 +39,7 @@ module Gitlab def verify(object) local?(object) ? verify_local(object) : verify_remote(object) - rescue => err + rescue StandardError => err failure(object, err.inspect) end @@ -77,27 +77,27 @@ module Gitlab # This should return an ActiveRecord::Relation suitable for calling #in_batches on def all_relation - raise NotImplementedError.new + raise NotImplementedError end # Should return true if the object is stored locally def local?(_object) - raise NotImplementedError.new + raise NotImplementedError end # The checksum we expect the object to have def expected_checksum(_object) - raise NotImplementedError.new + raise NotImplementedError end # The freshly-recalculated checksum of the object def actual_checksum(_object) - raise NotImplementedError.new + raise NotImplementedError end # Be sure to perform a hard check of the remote object (don't just check DB value) def remote_object_exists?(object) - raise NotImplementedError.new + raise NotImplementedError end end end diff --git a/lib/gitlab/view/presenter/delegated.rb b/lib/gitlab/view/presenter/delegated.rb index 4a90ab758fb..d14f8cc4e5e 100644 --- a/lib/gitlab/view/presenter/delegated.rb +++ b/lib/gitlab/view/presenter/delegated.rb @@ -11,7 +11,7 @@ module Gitlab attributes.each do |key, value| if subject.respond_to?(key) - raise CannotOverrideMethodError.new("#{subject} already respond to #{key}!") + raise CannotOverrideMethodError, "#{subject} already respond to #{key}!" end define_singleton_method(key) { value } diff --git a/lib/gitlab/web_ide/config/entry/global.rb b/lib/gitlab/web_ide/config/entry/global.rb index 2c67c7d02d4..2939095fd0f 100644 --- a/lib/gitlab/web_ide/config/entry/global.rb +++ b/lib/gitlab/web_ide/config/entry/global.rb @@ -30,4 +30,4 @@ module Gitlab end end -::Gitlab::WebIde::Config::Entry::Global.prepend_if_ee('EE::Gitlab::WebIde::Config::Entry::Global') +::Gitlab::WebIde::Config::Entry::Global.prepend_mod_with('Gitlab::WebIde::Config::Entry::Global') diff --git a/lib/gitlab/webpack/manifest.rb b/lib/gitlab/webpack/manifest.rb index 9c967d99e3a..b73c2ebb578 100644 --- a/lib/gitlab/webpack/manifest.rb +++ b/lib/gitlab/webpack/manifest.rb @@ -102,13 +102,13 @@ module Gitlab rescue OpenSSL::SSL::SSLError, EOFError => e ssl_status = Gitlab.config.webpack.dev_server.https ? ' over SSL' : '' raise ManifestLoadError.new("Could not connect to webpack-dev-server at #{uri}#{ssl_status}.\n\nIs SSL enabled? Check that settings in `gitlab.yml` and webpack-dev-server match.", e) - rescue => e + rescue StandardError => e raise ManifestLoadError.new("Could not load manifest from webpack-dev-server at #{uri}.\n\nIs webpack-dev-server running? Try running `gdk status webpack` or `gdk tail webpack`.", e) end def load_static_manifest File.read(static_manifest_path) - rescue => e + rescue StandardError => e raise ManifestLoadError.new("Could not load compiled manifest from #{static_manifest_path}.\n\nHave you run `rake gitlab:assets:compile`?", e) end diff --git a/lib/gitlab/x509/signature.rb b/lib/gitlab/x509/signature.rb index edff1540cb3..c83213e973b 100644 --- a/lib/gitlab/x509/signature.rb +++ b/lib/gitlab/x509/signature.rb @@ -72,7 +72,7 @@ module Gitlab pkcs7_text = pkcs7_text.sub('-----END SIGNED MESSAGE-----', '-----END PKCS7-----') OpenSSL::PKCS7.new(pkcs7_text) - rescue + rescue StandardError nil end end @@ -87,7 +87,7 @@ module Gitlab def valid_signature? p7.verify([], cert_store, signed_text, OpenSSL::PKCS7::NOVERIFY) - rescue + rescue StandardError nil end @@ -104,7 +104,7 @@ module Gitlab else nil end - rescue + rescue StandardError nil end diff --git a/lib/gitlab/x509/tag.rb b/lib/gitlab/x509/tag.rb index 48582c17764..ad85b200130 100644 --- a/lib/gitlab/x509/tag.rb +++ b/lib/gitlab/x509/tag.rb @@ -23,7 +23,7 @@ module Gitlab def signature_text @raw_tag.message.slice(@raw_tag.message.index("-----BEGIN SIGNED MESSAGE-----")..-1) - rescue + rescue StandardError nil end |